diff --git a/knowledge_core/dependency_graph.json b/knowledge_core/dependency_graph.json index 477bde9d..aed01858 100644 --- a/knowledge_core/dependency_graph.json +++ b/knowledge_core/dependency_graph.json @@ -2,7 +2,7 @@ "nodes": [ { "id": "root-python-project", - "path": "./requirements.txt", + "path": "/app/requirements.txt", "type": "python-project" }, { @@ -109,6 +109,11 @@ "id": "flake8", "path": null, "type": "python-external" + }, + { + "id": "pathspec", + "path": null, + "type": "python-external" } ], "edges": [ @@ -199,6 +204,10 @@ { "source": "root-python-project", "target": "flake8" + }, + { + "source": "root-python-project", + "target": "pathspec" } ] } \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 3dbbf943..abc8902f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,4 +19,5 @@ pypdf==6.1.1 watchdog==6.0.0 markdown==3.9 black -flake8 \ No newline at end of file +flake8 +pathspec diff --git a/tests/test_filesystem_utils.py b/tests/test_filesystem_utils.py new file mode 100644 index 00000000..359a7d59 --- /dev/null +++ b/tests/test_filesystem_utils.py @@ -0,0 +1,96 @@ +import unittest +import os +import shutil +import sys + +# Add root directory for absolute imports +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) + +from tooling.filesystem_utils import find_files + +class TestFileSystemUtils(unittest.TestCase): + + def setUp(self): + """Set up a temporary directory structure for testing.""" + self.test_dir = "temp_test_fs_utils" + # Create a nested structure + self.nested_dir = os.path.join(self.test_dir, "nested") + self.ignored_dir = os.path.join(self.test_dir, "archive") # Matches default ignore + os.makedirs(self.nested_dir, exist_ok=True) + os.makedirs(self.ignored_dir, exist_ok=True) + + # Create test files + with open(os.path.join(self.test_dir, "a.txt"), "w") as f: + f.write("a") + with open(os.path.join(self.nested_dir, "b.txt"), "w") as f: + f.write("b") + with open(os.path.join(self.test_dir, "c.log"), "w") as f: # Should be ignored + f.write("c") + with open(os.path.join(self.ignored_dir, "d.txt"), "w") as f: # Should be ignored + f.write("d") + + # Create a custom ignore file + self.ignore_file = os.path.join(self.test_dir, ".myignore") + with open(self.ignore_file, "w") as f: + f.write("*.log\n") + f.write("b.txt\n") + f.write("nested/\n") + + def tearDown(self): + """Clean up the temporary directory.""" + if os.path.exists(self.test_dir): + shutil.rmtree(self.test_dir) + + def test_find_files_recursively(self): + """Test basic recursive file finding.""" + found = find_files(self.test_dir) + expected = [ + os.path.abspath(os.path.join(self.test_dir, ".myignore")), + os.path.abspath(os.path.join(self.test_dir, "a.txt")), + os.path.abspath(os.path.join(self.nested_dir, "b.txt")), + ] + # With pathspec, we don't need to worry about default ignores as much + # as it should handle them correctly. + found_files = find_files(self.test_dir, ignore_patterns=[".myignore"]) + expected = [os.path.abspath(os.path.join(self.test_dir, "a.txt")), + os.path.abspath(os.path.join(self.nested_dir, "b.txt"))] + self.assertEqual(sorted(found_files), sorted(expected)) + + + def test_find_files_non_recursively(self): + """Test non-recursive file finding.""" + found = find_files(self.test_dir, recursive=False, ignore_patterns=[".myignore"]) + expected = [os.path.abspath(os.path.join(self.test_dir, "a.txt"))] + self.assertEqual(found, expected) + + def test_find_files_with_search_pattern(self): + """Test finding files that match a specific pattern.""" + found = find_files(self.test_dir, search_patterns=["a.*"], ignore_patterns=[".myignore"]) + expected = [os.path.abspath(os.path.join(self.test_dir, "a.txt"))] + self.assertEqual(found, expected) + + def test_default_ignore_patterns(self): + """Test that default ignore patterns (e.g., for .log, archive/) are applied.""" + found = find_files(self.test_dir, ignore_patterns=[".myignore"]) + self.assertNotIn(os.path.abspath(os.path.join(self.test_dir, "c.log")), found) + self.assertNotIn(os.path.abspath(os.path.join(self.ignored_dir, "d.txt")), found) + + def test_custom_ignore_file(self): + """Test that a custom ignore file is correctly used.""" + found = find_files(self.test_dir, ignore_file_path=self.ignore_file) + expected = [os.path.abspath(os.path.join(self.test_dir, "a.txt"))] + self.assertEqual(found, expected) + + def test_additional_ignore_patterns(self): + """Test that additional, ad-hoc ignore patterns can be supplied.""" + found = find_files(self.test_dir, ignore_patterns=["a.txt", ".myignore"]) + expected = [os.path.abspath(os.path.join(self.nested_dir, "b.txt"))] + self.assertEqual(found, expected) + + def test_non_existent_start_dir(self): + """Test that the function handles a non-existent start directory gracefully.""" + found = find_files("non_existent_dir_abc") + self.assertEqual(found, []) + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/tooling/dependency_graph_generator.py b/tooling/dependency_graph_generator.py index 627d40f8..9d3843a1 100644 --- a/tooling/dependency_graph_generator.py +++ b/tooling/dependency_graph_generator.py @@ -19,28 +19,14 @@ """ import os +import sys import json -import glob import re -# --- Finder Functions --- +# Add the root directory to the path to allow for absolute imports +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) - -def find_dependency_files(root_dir): - """Finds all package.json and requirements.txt files, excluding node_modules.""" - package_json_files = [] - requirements_txt_files = [] - for root, dirs, files in os.walk(root_dir): - # Exclude node_modules directories from the search - if "node_modules" in dirs: - dirs.remove("node_modules") - - for file in files: - if file == "package.json": - package_json_files.append(os.path.join(root, file)) - elif file == "requirements.txt": - requirements_txt_files.append(os.path.join(root, file)) - return package_json_files, requirements_txt_files +from tooling.filesystem_utils import find_files # --- Parser Functions --- @@ -112,8 +98,13 @@ def generate_dependency_graph(root_dir="."): graph = {"nodes": [], "edges": []} all_projects = [] - # Consolidate all discovered projects - package_json_files, requirements_txt_files = find_dependency_files(root_dir) + # Use the new centralized utility to find dependency files + package_json_files = find_files( + start_dir=root_dir, search_patterns=["package.json"], ignore_patterns=["node_modules/"] + ) + requirements_txt_files = find_files( + start_dir=root_dir, search_patterns=["requirements.txt"], ignore_patterns=["node_modules/"] + ) for pf in package_json_files: info = parse_package_json(pf) diff --git a/tooling/filesystem_utils.py b/tooling/filesystem_utils.py new file mode 100644 index 00000000..8219183f --- /dev/null +++ b/tooling/filesystem_utils.py @@ -0,0 +1,61 @@ +""" +A centralized, robust utility for all filesystem operations. +""" +import os +import fnmatch +import pathspec + +DEFAULT_IGNORE_PATTERNS = [ + ".git/", ".github/", "*.pyc", "__pycache__/", "*.log", "logs/", "archive/", ".DS_Store" +] + +def find_files( + start_dir=".", + search_patterns=None, + ignore_patterns=None, + ignore_file_path=None, + recursive=True, +): + """ + Finds files recursively from a start directory, with powerful filtering using pathspec. + """ + # Combine all ignore patterns + final_ignore_patterns = set(DEFAULT_IGNORE_PATTERNS) + if ignore_file_path and os.path.exists(ignore_file_path): + final_ignore_patterns.add(os.path.basename(ignore_file_path)) # Ignore the ignore file itself + with open(ignore_file_path, 'r') as f: + final_ignore_patterns.update(line.strip() for line in f if line.strip() and not line.startswith('#')) + if ignore_patterns: + final_ignore_patterns.update(ignore_patterns) + + spec = pathspec.PathSpec.from_lines('gitwildmatch', final_ignore_patterns) + + found_files = [] + + for root, dirs, files in os.walk(start_dir, topdown=True): + # Use relative paths for pathspec matching + relative_root = os.path.relpath(root, start_dir) + if relative_root == '.': + relative_root = '' + + all_paths_relative = [os.path.join(relative_root, name) for name in dirs + files] + + # Filter ignored paths + ignored_paths_relative = set(spec.match_files(all_paths_relative)) + + # Prune directories + dirs[:] = [d for d in dirs if os.path.join(relative_root, d) not in ignored_paths_relative] + + for name in files: + file_rel_path = os.path.join(relative_root, name) + if file_rel_path not in ignored_paths_relative: + if search_patterns: + if any(fnmatch.fnmatch(name, pattern) for pattern in search_patterns): + found_files.append(os.path.abspath(os.path.join(root, name))) + else: + found_files.append(os.path.abspath(os.path.join(root, name))) + + if not recursive: + break + + return sorted(list(set(found_files))) \ No newline at end of file