Bump version to 0.3.0 and add pre-commit configuration; refactor RepoConverter and enhance ignore logic

Author: mathiasesn

.pre-commit-config.yaml

@@ -0,0 +1,14 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+      - id: end-of-file-fixer
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    # Ruff version.
+    rev: v0.6.2
+    hooks:
+      # Run the linter.
+      - id: ruff
+        args: [ --fix ]
+      # Run the formatter.
+      - id: ruff-format

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "repo-context"
-version = "0.2.0"
+version = "0.3.0"
 description = "Convert Git repositories into LLM-friendly context format"
 authors = [{ name = "Mathias Nielsen", email = "[email protected]" }]
 maintainers = [{ name = "Mathias Nielsen", email = "[email protected]" }]
@@ -26,4 +26,4 @@ repo-context = "repo_context.cli:main"
 
 [build-system]
 requires = ["hatchling"]
-build-backend = "hatchling.build"
+build-backend = "hatchling.build"

repo_context/__init__.py

@@ -14,6 +14,7 @@
     handlers=[RichHandler(console=console, rich_tracebacks=True)],
 )
 
-from repo_context.repo_converter import RepoConverter  # noqa: E402
+from repo_context.converter import RepoConverter  # noqa: E402
+from repo_context.structure import RepoStructure  # noqa: E402
 
-__all__ = ["RepoConverter"]
+__all__ = ["RepoConverter", "RepoStructure"]

repo_context/cli.py

@@ -3,7 +3,7 @@
 from pathlib import Path
 from urllib.parse import urlparse
 
-from repo_context.repo_converter import RepoConverter
+from repo_context.converter import RepoConverter
 
 logger = logging.getLogger("repo_context.cli")
 
@@ -60,18 +60,27 @@ def main():
     converter = RepoConverter(ignore_patterns=ignore_patterns)
 
     try:
+        # Clone or use local repository
         if urlparse(args.source).scheme:
             logger.info(f"Cloning repository from {args.source}")
             repo_path, _ = converter.clone_repo(args.source)
         else:
             repo_path = Path(args.source)
 
+        # Convert repository to context
         context = converter.convert(repo_path, max_file_lines=args.max_file_lines)
+        fname = repo_path.stem
 
-        for i, c in enumerate(context):
-            output_path = Path(f"{args.output}/context_{i}.md")
-            output_path.write_text(c)
+        # Write context to files
+        if len(context) == 1:
+            output_path = Path(f"{args.output}/{fname}.md")
+            output_path.write_text(context[0])
             logger.info(f"Context written to {output_path}")
+        else:
+            for i, c in enumerate(context):
+                output_path = Path(f"{args.output}/{fname}_{i}.md")
+                output_path.write_text(c)
+                logger.info(f"Context written to {output_path}")
 
     except Exception as e:
         logger.error(f"Error: {e}")

repo_context/repo_converter.py renamed to repo_context/converter.py

@@ -9,6 +9,7 @@
 
 from repo_context.ignore import EXTENSIONS, FILES, PATTERNS
 from repo_context.utils import should_ignore
+from repo_context.structure import RepoStructure
 
 logger = logging.getLogger("repo_context.repo_converter")
 
@@ -20,11 +21,25 @@ def __init__(
         max_file_size: int = 1_000_000,
         max_workers: int | None = None,
     ) -> None:
+        """
+        Initialize the converter with specified parameters.
+
+        Args:
+            ignore_patterns (list[str] | None, optional): A list of patterns to ignore. Defaults to None.
+            max_file_size (int, optional): The maximum file size to process in bytes. Defaults to 1,000,000.
+            max_workers (int | None, optional): The maximum number of worker threads to use. Defaults to the number of CPU cores.
+
+        Attributes:
+            ignore_patterns (list[str]): The list of patterns to ignore.
+            max_file_size (int): The maximum file size to process in bytes.
+            max_workers (int): The maximum number of worker threads to use.
+            structure (RepoStructure): The repository structure initialized with the ignore patterns.
+        """
         self.ignore_patterns = ignore_patterns or []
         self.max_file_size = max_file_size
         self.max_workers = max_workers or cpu_count()
-
         self.ignore_patterns += FILES + EXTENSIONS + PATTERNS
+        self.structure = RepoStructure(ignore_patterns=self.ignore_patterns)
 
     def clone_repo(self, url: str) -> Path:
         """Clone a repository from URL to temporary directory.
@@ -98,14 +113,22 @@ def convert(self, repo_path: Path, max_file_lines: int | None = None) -> list[st
         if not repo_path.exists():
             raise FileNotFoundError(f"Repository path {repo_path} does not exist")
 
+        context = []
+
+        # Get structure of the repository
+        tree_structure = self.structure.create_tree_structure(repo_path)
+        if tree_structure:
+            context.append(tree_structure)
+
+        # Get all files in the repository
         with logging_redirect_tqdm():
             file_paths = [
                 (str(p), str(repo_path))
                 for p in tqdm(repo_path.rglob("*"), ncols=120)
                 if self._is_valid_file(p)
             ]
 
-        context = []
+        # Process files in parallel
         with Pool(self.max_workers) as pool:
             with logging_redirect_tqdm():
                 with tqdm(

repo_context/ignore.py

@@ -8,6 +8,8 @@
     "uv.lock",
     "poetry.lock",
     ".dockerignore",
+    ".coverage",
+    ".pre-commit-config.yaml",
 ]
 
 EXTENSIONS = [
@@ -27,6 +29,11 @@
     "*.pyo",
     "*.pyd",
     ".DS_Store",
+    "*.zip",
+    "*.far",
+    "*.fst",
+    "*.tsv",
+    "*.csv",
 ]
 
 PATTERNS = [
@@ -44,4 +51,5 @@
     "publish",
     "tests",
     "test",
+    ".ruff_cache",
 ]

repo_context/structure.py

@@ -1,7 +1,6 @@
 import logging
 from pathlib import Path
 
-from repo_context.ignore import EXTENSIONS, FILES, PATTERNS
 from repo_context.utils import should_ignore
 
 logger = logging.getLogger("repo_context.structure")
@@ -10,30 +9,25 @@
 class RepoStructure:
     def __init__(self, ignore_patterns: list[str] | None = None) -> None:
         self.ignore_patterns = ignore_patterns or []
-        self.ignore_patterns += FILES + EXTENSIONS + PATTERNS
 
     def generate_tree(
         self,
         directory: Path,
         prefix: str = "",
         is_last: bool = True,
-        ignore_patterns: list[str] | None = None,
     ) -> list[str]:
         """
         Recursively generate a tree structure of the directory.
 
         Args:
-            directory: Path object pointing to the directory
-            prefix: Prefix for the current line (used for recursion)
-            is_last: Boolean indicating if this is the last item in current directory
-            ignore_patterns: List of patterns to ignore
+            directory (Path): Path object pointing to the directory
+            prefix (str): Prefix for the current line (used for recursion). default: ""
+            is_last (bool): Boolean indicating if this is the last item in current directory. default: True
+            ignore_patterns (list[str] | None): List of patterns to ignore. default: None
 
         Returns:
-            List[str]: Lines of the tree structure
+            list[str]: Lines of the tree structure
         """
-        if ignore_patterns is None:
-            ignore_patterns = []
-
         if not directory.is_dir():
             logger.error(f"'{directory}' is not a valid directory")
             return []
@@ -42,65 +36,47 @@ def generate_tree(
         items = [
             item
             for item in sorted(directory.iterdir())
-            if not should_ignore(item.name, ignore_patterns)
+            if not should_ignore(item.name, self.ignore_patterns)
         ]
 
         for i, item in enumerate(items):
             is_last_item = i == len(items) - 1
-            connector = "??? " if is_last_item else "??? "
+            connector = "└── " if is_last_item else "├── "
 
             tree_lines.append(f"{prefix}{connector}{item.name}")
 
             if item.is_dir():
-                extension = "    " if is_last_item else "?   "
+                extension = "    " if is_last_item else "│   "
                 tree_lines.extend(
                     self.generate_tree(
                         item,
                         prefix + extension,
                         is_last_item,
-                        ignore_patterns,
                     )
                 )
 
         return tree_lines
 
-    def create_tree_structure(
-        self,
-        path: str,
-        output_file: str | None = None,
-        ignore_patterns: list[str] | None = None,
-    ) -> None:
+    def create_tree_structure(self, path: str) -> str:
         """
         Create and display/save a tree structure of the specified directory.
 
         Args:
             path: Path to the directory
-            output_file: Optional file path to save the tree structure
-            ignore_patterns: List of patterns to ignore
+
+        Returns:
+            str: The tree structure
         """
         directory = Path(path)
         if not directory.exists():
-            logger.error(f"Directory '{path}' does not exist")
-            return
+            raise FileNotFoundError(f"Directory '{path}' does not exist")
 
         logger.info(f"Generating tree structure for: {directory.absolute()}")
 
-        tree_lines = ["Directory Structure:", directory.name]
-        tree_lines.extend(
-            self.generate_tree(directory, ignore_patterns=ignore_patterns or [])
-        )
+        tree_lines = ["# Directory Structure", directory.name]
+        tree_lines.extend(self.generate_tree(directory))
 
         # Join lines with newlines
-        tree_structure = "\n".join(tree_lines)
-
-        # Print to console
-        logger.info(tree_structure)
-
-        # Save to file if specified
-        if output_file:
-            output_path = Path(output_file)
-            try:
-                output_path.write_text(tree_structure)
-                logger.info(f"Tree structure saved to: {output_path.absolute()}")
-            except Exception as e:
-                logger.error(f"Failed to save tree structure: {e}")
+        tree_structure = "\n".join(tree_lines) + "\n"
+
+        return tree_structure

repo_context/utils.py

@@ -31,6 +31,9 @@ def should_ignore(path: Path, ignore_patterns: list[str]) -> bool:
     Returns:
         True if path should be ignored
     """
+    if not isinstance(path, Path):
+        path = Path(path)
+
     fname = path.name
     path_str = str(path)
     relative_path = get_relative_path(path)

tests/test_repo_converter.py renamed to tests/test_converter.py

@@ -46,29 +46,6 @@ def test_clone_repo_invalid_url(converter):
         converter.clone_repo("invalid_url")
 
 
-def test_should_ignore():
-    converter = RepoConverter()
-
-    assert converter.should_ignore(Path(".gitignore"))
-    assert converter.should_ignore(Path("some/path/.gitignore"))
-
-    assert converter.should_ignore(Path("image.png"))
-    assert converter.should_ignore(Path("deep/path/image.png"))
-
-    assert converter.should_ignore(Path(".git/config"))
-    assert converter.should_ignore(Path("some/path/.git/config"))
-
-    assert not converter.should_ignore(Path("regular.txt"))
-    assert not converter.should_ignore(Path("src/main.py"))
-
-
-def test_should_ignore_with_ignore_patterns():
-    converter = RepoConverter(ignore_patterns=["*.pyc", "test/*"])
-    assert converter.should_ignore(Path("file.pyc"))
-    assert converter.should_ignore(Path("test/file.py"))
-    assert not converter.should_ignore(Path("src/file.py"))
-
-
 def test_is_valid_file(converter, temp_repo):
     assert converter._is_valid_file(temp_repo / "file.txt")
     assert not converter._is_valid_file(temp_repo / "large.txt")
@@ -88,8 +65,6 @@ def test_convert(converter, temp_repo):
     result = converter.convert(temp_repo)[0]
     assert "file.txt" in result
     assert "test content" in result
-    assert "empty.txt" not in result
-    assert "large.txt" not in result
     assert "test.ignored" not in result