feat(tooling): Create centralized filesystem utility

knowledge_core/dependency_graph.json

@@ -2,7 +2,7 @@
   "nodes": [
     {
       "id": "root-python-project",
-      "path": "./requirements.txt",
+      "path": "/app/requirements.txt",
       "type": "python-project"
     },
     {
@@ -109,6 +109,11 @@
       "id": "flake8",
       "path": null,
       "type": "python-external"
+    },
+    {
+      "id": "pathspec",
+      "path": null,
+      "type": "python-external"
     }
   ],
   "edges": [
@@ -199,6 +204,10 @@
     {
       "source": "root-python-project",
       "target": "flake8"
+    },
+    {
+      "source": "root-python-project",
+      "target": "pathspec"
     }
   ]
 }

requirements.txt

@@ -19,4 +19,5 @@ pypdf==6.1.1
 watchdog==6.0.0
 markdown==3.9
 black
-flake8
+flake8
+pathspec

tests/test_filesystem_utils.py

@@ -0,0 +1,96 @@
+import unittest
+import os
+import shutil
+import sys
+
+# Add root directory for absolute imports
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+
+from tooling.filesystem_utils import find_files
+
+class TestFileSystemUtils(unittest.TestCase):
+
+    def setUp(self):
+        """Set up a temporary directory structure for testing."""
+        self.test_dir = "temp_test_fs_utils"
+        # Create a nested structure
+        self.nested_dir = os.path.join(self.test_dir, "nested")
+        self.ignored_dir = os.path.join(self.test_dir, "archive") # Matches default ignore
+        os.makedirs(self.nested_dir, exist_ok=True)
+        os.makedirs(self.ignored_dir, exist_ok=True)
+
+        # Create test files
+        with open(os.path.join(self.test_dir, "a.txt"), "w") as f:
+            f.write("a")
+        with open(os.path.join(self.nested_dir, "b.txt"), "w") as f:
+            f.write("b")
+        with open(os.path.join(self.test_dir, "c.log"), "w") as f: # Should be ignored
+            f.write("c")
+        with open(os.path.join(self.ignored_dir, "d.txt"), "w") as f: # Should be ignored
+            f.write("d")
+
+        # Create a custom ignore file
+        self.ignore_file = os.path.join(self.test_dir, ".myignore")
+        with open(self.ignore_file, "w") as f:
+            f.write("*.log\n")
+            f.write("b.txt\n")
+            f.write("nested/\n")
+
+    def tearDown(self):
+        """Clean up the temporary directory."""
+        if os.path.exists(self.test_dir):
+            shutil.rmtree(self.test_dir)
+
+    def test_find_files_recursively(self):
+        """Test basic recursive file finding."""
+        found = find_files(self.test_dir)
+        expected = [
+            os.path.abspath(os.path.join(self.test_dir, ".myignore")),
+            os.path.abspath(os.path.join(self.test_dir, "a.txt")),
+            os.path.abspath(os.path.join(self.nested_dir, "b.txt")),
+        ]
+        # With pathspec, we don't need to worry about default ignores as much
+        # as it should handle them correctly.
+        found_files = find_files(self.test_dir, ignore_patterns=[".myignore"])
+        expected = [os.path.abspath(os.path.join(self.test_dir, "a.txt")),
+                    os.path.abspath(os.path.join(self.nested_dir, "b.txt"))]
+        self.assertEqual(sorted(found_files), sorted(expected))
+
+
+    def test_find_files_non_recursively(self):
+        """Test non-recursive file finding."""
+        found = find_files(self.test_dir, recursive=False, ignore_patterns=[".myignore"])
+        expected = [os.path.abspath(os.path.join(self.test_dir, "a.txt"))]
+        self.assertEqual(found, expected)
+
+    def test_find_files_with_search_pattern(self):
+        """Test finding files that match a specific pattern."""
+        found = find_files(self.test_dir, search_patterns=["a.*"], ignore_patterns=[".myignore"])
+        expected = [os.path.abspath(os.path.join(self.test_dir, "a.txt"))]
+        self.assertEqual(found, expected)
+
+    def test_default_ignore_patterns(self):
+        """Test that default ignore patterns (e.g., for .log, archive/) are applied."""
+        found = find_files(self.test_dir, ignore_patterns=[".myignore"])
+        self.assertNotIn(os.path.abspath(os.path.join(self.test_dir, "c.log")), found)
+        self.assertNotIn(os.path.abspath(os.path.join(self.ignored_dir, "d.txt")), found)
+
+    def test_custom_ignore_file(self):
+        """Test that a custom ignore file is correctly used."""
+        found = find_files(self.test_dir, ignore_file_path=self.ignore_file)
+        expected = [os.path.abspath(os.path.join(self.test_dir, "a.txt"))]
+        self.assertEqual(found, expected)
+
+    def test_additional_ignore_patterns(self):
+        """Test that additional, ad-hoc ignore patterns can be supplied."""
+        found = find_files(self.test_dir, ignore_patterns=["a.txt", ".myignore"])
+        expected = [os.path.abspath(os.path.join(self.nested_dir, "b.txt"))]
+        self.assertEqual(found, expected)
+
+    def test_non_existent_start_dir(self):
+        """Test that the function handles a non-existent start directory gracefully."""
+        found = find_files("non_existent_dir_abc")
+        self.assertEqual(found, [])
+
+if __name__ == "__main__":
+    unittest.main()

tooling/dependency_graph_generator.py

@@ -19,28 +19,14 @@
 """
 
 import os
+import sys
 import json
-import glob
 import re
 
-# --- Finder Functions ---
+# Add the root directory to the path to allow for absolute imports
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
 
-
-def find_dependency_files(root_dir):
-    """Finds all package.json and requirements.txt files, excluding node_modules."""
-    package_json_files = []
-    requirements_txt_files = []
-    for root, dirs, files in os.walk(root_dir):
-        # Exclude node_modules directories from the search
-        if "node_modules" in dirs:
-            dirs.remove("node_modules")
-
-        for file in files:
-            if file == "package.json":
-                package_json_files.append(os.path.join(root, file))
-            elif file == "requirements.txt":
-                requirements_txt_files.append(os.path.join(root, file))
-    return package_json_files, requirements_txt_files
+from tooling.filesystem_utils import find_files
 
 
 # --- Parser Functions ---
@@ -112,8 +98,13 @@ def generate_dependency_graph(root_dir="."):
     graph = {"nodes": [], "edges": []}
     all_projects = []
 
-    # Consolidate all discovered projects
-    package_json_files, requirements_txt_files = find_dependency_files(root_dir)
+    # Use the new centralized utility to find dependency files
+    package_json_files = find_files(
+        start_dir=root_dir, search_patterns=["package.json"], ignore_patterns=["node_modules/"]
+    )
+    requirements_txt_files = find_files(
+        start_dir=root_dir, search_patterns=["requirements.txt"], ignore_patterns=["node_modules/"]
+    )
 
     for pf in package_json_files:
         info = parse_package_json(pf)

tooling/filesystem_utils.py

@@ -0,0 +1,61 @@
+"""
+A centralized, robust utility for all filesystem operations.
+"""
+import os
+import fnmatch
+import pathspec
+
+DEFAULT_IGNORE_PATTERNS = [
+    ".git/", ".github/", "*.pyc", "__pycache__/", "*.log", "logs/", "archive/", ".DS_Store"
+]
+
+def find_files(
+    start_dir=".",
+    search_patterns=None,
+    ignore_patterns=None,
+    ignore_file_path=None,
+    recursive=True,
+):
+    """
+    Finds files recursively from a start directory, with powerful filtering using pathspec.
+    """
+    # Combine all ignore patterns
+    final_ignore_patterns = set(DEFAULT_IGNORE_PATTERNS)
+    if ignore_file_path and os.path.exists(ignore_file_path):
+        final_ignore_patterns.add(os.path.basename(ignore_file_path)) # Ignore the ignore file itself
+        with open(ignore_file_path, 'r') as f:
+            final_ignore_patterns.update(line.strip() for line in f if line.strip() and not line.startswith('#'))
+    if ignore_patterns:
+        final_ignore_patterns.update(ignore_patterns)
+
+    spec = pathspec.PathSpec.from_lines('gitwildmatch', final_ignore_patterns)
+
+    found_files = []
+
+    for root, dirs, files in os.walk(start_dir, topdown=True):
+        # Use relative paths for pathspec matching
+        relative_root = os.path.relpath(root, start_dir)
+        if relative_root == '.':
+            relative_root = ''
+
+        all_paths_relative = [os.path.join(relative_root, name) for name in dirs + files]
+
+        # Filter ignored paths
+        ignored_paths_relative = set(spec.match_files(all_paths_relative))
+
+        # Prune directories
+        dirs[:] = [d for d in dirs if os.path.join(relative_root, d) not in ignored_paths_relative]
+
+        for name in files:
+            file_rel_path = os.path.join(relative_root, name)
+            if file_rel_path not in ignored_paths_relative:
+                if search_patterns:
+                    if any(fnmatch.fnmatch(name, pattern) for pattern in search_patterns):
+                        found_files.append(os.path.abspath(os.path.join(root, name)))
+                else:
+                    found_files.append(os.path.abspath(os.path.join(root, name)))
+
+        if not recursive:
+            break
+
+    return sorted(list(set(found_files)))