Merge pull request #6 from mathiasesn/feat/create-repo-structure

mathiasesn · web-flow · commit fd3a2dea3ad3 · 2025-01-20T21:34:31.000+01:00
feat: Add repo structure to output markdown
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -0,0 +1,14 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+      - id: end-of-file-fixer
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    # Ruff version.
+    rev: v0.6.2
+    hooks:
+      # Run the linter.
+      - id: ruff
+        args: [ --fix ]
+      # Run the formatter.
+      - id: ruff-format
diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "repo-context"
-version = "0.2.0"
+version = "0.3.0"
 description = "Convert Git repositories into LLM-friendly context format"
 authors = [{ name = "Mathias Nielsen", email = "mathiasesn1@gmail.com" }]
 maintainers = [{ name = "Mathias Nielsen", email = "mathiasesn1@gmail.com" }]
@@ -26,4 +26,4 @@ repo-context = "repo_context.cli:main"
 
 [build-system]
 requires = ["hatchling"]
-build-backend = "hatchling.build"
+build-backend = "hatchling.build"
diff --git a/repo_context/__init__.py b/repo_context/__init__.py
@@ -14,6 +14,7 @@
     handlers=[RichHandler(console=console, rich_tracebacks=True)],
 )
 
-from repo_context.repo_converter import RepoConverter  # noqa: E402
+from repo_context.converter import RepoConverter  # noqa: E402
+from repo_context.structure import RepoStructure  # noqa: E402
 
-__all__ = ["RepoConverter"]
+__all__ = ["RepoConverter", "RepoStructure"]
diff --git a/repo_context/cli.py b/repo_context/cli.py
@@ -3,7 +3,7 @@
 from pathlib import Path
 from urllib.parse import urlparse
 
-from repo_context.repo_converter import RepoConverter
+from repo_context.converter import RepoConverter
 
 logger = logging.getLogger("repo_context.cli")
 
@@ -60,18 +60,28 @@ def main():
     converter = RepoConverter(ignore_patterns=ignore_patterns)
 
     try:
+        # Clone or use local repository
         if urlparse(args.source).scheme:
             logger.info(f"Cloning repository from {args.source}")
             repo_path, _ = converter.clone_repo(args.source)
+            fname = Path(urlparse(args.source).path).stem
         else:
             repo_path = Path(args.source)
+            fname = repo_path.stem
 
+        # Convert repository to context
         context = converter.convert(repo_path, max_file_lines=args.max_file_lines)
 
-        for i, c in enumerate(context):
-            output_path = Path(f"{args.output}/context_{i}.md")
-            output_path.write_text(c)
+        # Write context to files
+        if len(context) == 1:
+            output_path = Path(f"{args.output}/{fname}.md")
+            output_path.write_text(context[0])
             logger.info(f"Context written to {output_path}")
+        else:
+            for i, c in enumerate(context):
+                output_path = Path(f"{args.output}/{fname}_{i}.md")
+                output_path.write_text(c)
+                logger.info(f"Context written to {output_path}")
 
     except Exception as e:
         logger.error(f"Error: {e}")
diff --git a/repo_context/converter.py b/repo_context/converter.py
@@ -1,6 +1,5 @@
 import logging
 import tempfile
-from fnmatch import fnmatch
 from multiprocessing import Pool, cpu_count
 from pathlib import Path
 
@@ -9,6 +8,8 @@
 from tqdm.contrib.logging import logging_redirect_tqdm
 
 from repo_context.ignore import EXTENSIONS, FILES, PATTERNS
+from repo_context.utils import should_ignore
+from repo_context.structure import RepoStructure
 
 logger = logging.getLogger("repo_context.repo_converter")
 
@@ -20,11 +21,25 @@ def __init__(
         max_file_size: int = 1_000_000,
         max_workers: int | None = None,
     ) -> None:
+        """
+        Initialize the converter with specified parameters.
+
+        Args:
+            ignore_patterns (list[str] | None, optional): A list of patterns to ignore. Defaults to None.
+            max_file_size (int, optional): The maximum file size to process in bytes. Defaults to 1,000,000.
+            max_workers (int | None, optional): The maximum number of worker threads to use. Defaults to the number of CPU cores.
+
+        Attributes:
+            ignore_patterns (list[str]): The list of patterns to ignore.
+            max_file_size (int): The maximum file size to process in bytes.
+            max_workers (int): The maximum number of worker threads to use.
+            structure (RepoStructure): The repository structure initialized with the ignore patterns.
+        """
         self.ignore_patterns = ignore_patterns or []
         self.max_file_size = max_file_size
         self.max_workers = max_workers or cpu_count()
-
         self.ignore_patterns += FILES + EXTENSIONS + PATTERNS
+        self.structure = RepoStructure(ignore_patterns=self.ignore_patterns)
 
     def clone_repo(self, url: str) -> Path:
         """Clone a repository from URL to temporary directory.
@@ -68,57 +83,6 @@ def progress_callback(op_code, cur_count, max_count=None, message=""):
             logger.error(f"Failed to clone repository: {e}")
             raise
 
-    def should_ignore(self, path: Path) -> bool:
-        """Check if path matches ignore patterns.
-
-        Args:
-            path: Path to check against ignore patterns
-
-        Returns:
-            True if path should be ignored
-        """
-        fname = path.name
-        path_str = str(path)
-        relative_path = self._get_relative_path(path)
-
-        for pattern in self.ignore_patterns:
-            if pattern in FILES and fname == pattern:
-                return True
-
-            if pattern in EXTENSIONS and fnmatch(fname, pattern):
-                return True
-
-            if pattern in PATTERNS:
-                if pattern in path_str:
-                    return True
-
-                normalized_path = relative_path.replace("\\", "/")
-                normalized_pattern = pattern.replace("\\", "/")
-                if fnmatch(normalized_path, normalized_pattern):
-                    return True
-
-            if fnmatch(path_str, pattern):
-                return True
-
-        return False
-
-    @staticmethod
-    def _get_relative_path(path: Path) -> str:
-        """
-        Get the relative path of the given Path object with respect to the current working directory.
-
-        Args:
-            path (Path): The Path object to be converted to a relative path.
-
-        Returns:
-            str: The relative path as a string if the given path is within the current working directory,
-                 otherwise the absolute path as a string.
-        """
-        try:
-            return str(path.resolve().relative_to(Path.cwd()))
-        except ValueError:
-            return str(path)
-
     def _process_file_wrapper(self, args: tuple[str, str]) -> str | None:
         """
         Wrapper method to process a file with given file path and repository path.
@@ -149,14 +113,22 @@ def convert(self, repo_path: Path, max_file_lines: int | None = None) -> list[st
         if not repo_path.exists():
             raise FileNotFoundError(f"Repository path {repo_path} does not exist")
 
+        context = []
+
+        # Get structure of the repository
+        tree_structure = self.structure.create_tree_structure(repo_path)
+        if tree_structure:
+            context.append(tree_structure)
+
+        # Get all files in the repository
         with logging_redirect_tqdm():
             file_paths = [
                 (str(p), str(repo_path))
                 for p in tqdm(repo_path.rglob("*"), ncols=120)
                 if self._is_valid_file(p)
             ]
 
-        context = []
+        # Process files in parallel
         with Pool(self.max_workers) as pool:
             with logging_redirect_tqdm():
                 with tqdm(
@@ -182,7 +154,7 @@ def _is_valid_file(self, path: Path) -> bool:
         """Check if file should be processed."""
         return (
             path.is_file()
-            and not self.should_ignore(path)
+            and not should_ignore(path, self.ignore_patterns)
             and path.stat().st_size <= self.max_file_size
         )
 
diff --git a/repo_context/ignore.py b/repo_context/ignore.py
@@ -8,6 +8,8 @@
     "uv.lock",
     "poetry.lock",
     ".dockerignore",
+    ".coverage",
+    ".pre-commit-config.yaml",
 ]
 
 EXTENSIONS = [
@@ -27,6 +29,11 @@
     "*.pyo",
     "*.pyd",
     ".DS_Store",
+    "*.zip",
+    "*.far",
+    "*.fst",
+    "*.tsv",
+    "*.csv",
 ]
 
 PATTERNS = [
@@ -44,4 +51,5 @@
     "publish",
     "tests",
     "test",
+    ".ruff_cache",
 ]
diff --git a/repo_context/structure.py b/repo_context/structure.py
@@ -0,0 +1,82 @@
+import logging
+from pathlib import Path
+
+from repo_context.utils import should_ignore
+
+logger = logging.getLogger("repo_context.structure")
+
+
+class RepoStructure:
+    def __init__(self, ignore_patterns: list[str] | None = None) -> None:
+        self.ignore_patterns = ignore_patterns or []
+
+    def generate_tree(
+        self,
+        directory: Path,
+        prefix: str = "",
+        is_last: bool = True,
+    ) -> list[str]:
+        """
+        Recursively generate a tree structure of the directory.
+
+        Args:
+            directory (Path): Path object pointing to the directory
+            prefix (str): Prefix for the current line (used for recursion). default: ""
+            is_last (bool): Boolean indicating if this is the last item in current directory. default: True
+            ignore_patterns (list[str] | None): List of patterns to ignore. default: None
+
+        Returns:
+            list[str]: Lines of the tree structure
+        """
+        if not directory.is_dir():
+            logger.error(f"'{directory}' is not a valid directory")
+            return []
+
+        tree_lines = []
+        items = [
+            item
+            for item in sorted(directory.iterdir())
+            if not should_ignore(item.name, self.ignore_patterns)
+        ]
+
+        for i, item in enumerate(items):
+            is_last_item = i == len(items) - 1
+            connector = "└── " if is_last_item else "├── "
+
+            tree_lines.append(f"{prefix}{connector}{item.name}")
+
+            if item.is_dir():
+                extension = "    " if is_last_item else "│   "
+                tree_lines.extend(
+                    self.generate_tree(
+                        item,
+                        prefix + extension,
+                        is_last_item,
+                    )
+                )
+
+        return tree_lines
+
+    def create_tree_structure(self, path: str) -> str:
+        """
+        Create and display/save a tree structure of the specified directory.
+
+        Args:
+            path: Path to the directory
+
+        Returns:
+            str: The tree structure
+        """
+        directory = Path(path)
+        if not directory.exists():
+            raise FileNotFoundError(f"Directory '{path}' does not exist")
+
+        logger.info(f"Generating tree structure for: {directory.absolute()}")
+
+        tree_lines = ["# Directory Structure", directory.name]
+        tree_lines.extend(self.generate_tree(directory))
+
+        # Join lines with newlines
+        tree_structure = "\n".join(tree_lines) + "\n"
+
+        return tree_structure
diff --git a/repo_context/utils.py b/repo_context/utils.py
@@ -0,0 +1,60 @@
+from fnmatch import fnmatch
+from pathlib import Path
+
+from repo_context.ignore import EXTENSIONS, FILES, PATTERNS
+
+
+def get_relative_path(path: Path) -> str:
+    """
+    Get the relative path of the given Path object with respect to the current working directory.
+
+    Args:
+        path (Path): The Path object to be converted to a relative path.
+
+    Returns:
+        str: The relative path as a string if the given path is within the current working directory,
+                otherwise the absolute path as a string.
+    """
+    try:
+        return str(path.resolve().relative_to(Path.cwd()))
+    except ValueError:
+        return str(path)
+
+
+def should_ignore(path: Path, ignore_patterns: list[str]) -> bool:
+    """Check if path matches ignore patterns.
+
+    Args:
+        path (Path): Path to check against ignore patterns
+        ignore_patterns (list[str]): List of ignore patterns
+
+    Returns:
+        True if path should be ignored
+    """
+    if not isinstance(path, Path):
+        path = Path(path)
+
+    fname = path.name
+    path_str = str(path)
+    relative_path = get_relative_path(path)
+
+    for pattern in ignore_patterns:
+        if pattern in FILES and fname == pattern:
+            return True
+
+        if pattern in EXTENSIONS and fnmatch(fname, pattern):
+            return True
+
+        if pattern in PATTERNS:
+            if pattern in path_str:
+                return True
+
+            normalized_path = relative_path.replace("\\", "/")
+            normalized_pattern = pattern.replace("\\", "/")
+            if fnmatch(normalized_path, normalized_pattern):
+                return True
+
+        if fnmatch(path_str, pattern):
+            return True
+
+    return False
diff --git a/tests/test_converter.py b/tests/test_converter.py
diff --git a/tests/test_structure.py b/tests/test_structure.py
diff --git a/tests/test_utils.py b/tests/test_utils.py
diff --git a/uv.lock b/uv.lock

Original file line number	Diff line number	Diff line change
`@@ -14,6 +14,7 @@`
`14`	`14`	`handlers=[RichHandler(console=console, rich_tracebacks=True)],`
`15`	`15`	`)`
`16`	`16`
`17`		`-from repo_context.repo_converter import RepoConverter # noqa: E402`
	`17`	`+from repo_context.converter import RepoConverter # noqa: E402`
	`18`	`+from repo_context.structure import RepoStructure # noqa: E402`
`18`	`19`
`19`		`-__all__ = ["RepoConverter"]`
	`20`	`+__all__ = ["RepoConverter", "RepoStructure"]`