From bb8379665f34c7f2ae779ae314482f7875c0a929 Mon Sep 17 00:00:00 2001 From: Nicholas Denny Date: Wed, 29 Jul 2026 16:34:41 -0500 Subject: [PATCH 1/5] fix: preserve deep Windows paths across corpus pipeline --- .github/workflows/ci.yml | 34 ++++ CHANGELOG.md | 9 + graphify/build.py | 31 +-- graphify/cache.py | 145 ++++++++------ graphify/cargo_introspect.py | 15 +- graphify/cli.py | 127 +++++++----- graphify/detect.py | 187 +++++++++--------- graphify/diagnostics.py | 11 +- graphify/extract.py | 119 ++++++------ graphify/extractors/apex.py | 3 +- graphify/extractors/bash.py | 26 ++- graphify/extractors/blade.py | 3 +- graphify/extractors/dart.py | 12 +- graphify/extractors/dm.py | 26 ++- graphify/extractors/elixir.py | 3 +- graphify/extractors/engine.py | 3 +- graphify/extractors/fortran.py | 26 ++- graphify/extractors/go.py | 3 +- graphify/extractors/json_config.py | 4 +- graphify/extractors/julia.py | 3 +- graphify/extractors/markdown.py | 5 +- graphify/extractors/objc.py | 3 +- graphify/extractors/pascal.py | 5 +- graphify/extractors/pascal_forms.py | 5 +- graphify/extractors/powershell.py | 5 +- graphify/extractors/razor.py | 3 +- graphify/extractors/resolution.py | 171 ++++++++-------- graphify/extractors/rust.py | 3 +- graphify/extractors/sln.py | 5 +- graphify/extractors/sql.py | 3 +- graphify/extractors/terraform.py | 3 +- graphify/extractors/verilog.py | 3 +- graphify/extractors/zig.py | 3 +- graphify/file_slice.py | 8 +- graphify/google_workspace.py | 74 +++++-- graphify/llm.py | 34 ++-- graphify/manifest_ingest.py | 5 +- graphify/mcp_ingest.py | 4 +- graphify/paths.py | 289 ++++++++++++++++++++++++++-- graphify/security.py | 22 ++- graphify/symbol_resolution.py | 15 +- graphify/transcribe.py | 25 ++- graphify/watch.py | 175 +++++++++-------- tests/test_atomic_writes.py | 33 +++- tests/test_cpp_preprocess.py | 7 +- tests/test_long_path_hashing.py | 49 +++-- tests/test_windows_long_paths.py | 239 +++++++++++++++++++++++ 47 files changed, 1398 insertions(+), 588 deletions(-) create mode 100644 tests/test_windows_long_paths.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 548b40f22..3d8d53733 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -78,6 +78,40 @@ jobs: uv run --frozen graphify --help uv run --frozen graphify install + platform-paths: + # Keep the filesystem boundary honest on every supported desktop OS. The + # Windows run creates and extracts a real 300+ character source path; the + # same tests verify that the adapter remains a no-op on Linux and macOS. + name: Filesystem paths (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + + steps: + - uses: actions/checkout@v6 + + - name: Install uv + uses: astral-sh/setup-uv@v8.1.0 + with: + python-version: "3.12" + + - name: Install dependencies + run: uv sync --frozen + + - name: Exercise logical and extended path boundaries + run: >- + uv run --frozen pytest + tests/test_windows_long_paths.py + tests/test_long_path_hashing.py + tests/test_paths.py + tests/test_atomic_writes.py + tests/test_google_workspace.py + tests/test_cargo_introspect.py + tests/test_cpp_preprocess.py + -q --tb=short + security-scan: # The dev deps include bandit and pip-audit. Run them in CI so a new # HIGH-severity finding or vulnerable dependency is caught on the PR that diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ffa5cf55..02e94b779 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ Full release notes with details on each version: [GitHub Releases](https://github.com/safishamsi/graphify/releases) +## Unreleased + +- Fix: Windows scans no longer skip deeply nested local or UNC files when their paths + cross the legacy `MAX_PATH` boundary. Filesystem calls now use extended-length paths + only at the I/O boundary while graph IDs, manifests, cache keys, and diagnostics retain + ordinary paths; discovery, hashing, extraction, incremental rebuilds, and document/media + readers share the same cross-platform adapter. A focused Ubuntu/macOS/Windows CI matrix + exercises 300+ character paths and verifies that the adapter remains a no-op off Windows. + ## 0.9.30 (2026-07-29) - Fix: pin `mcp` below 2.0 so a fresh `graphifyy[mcp]` / `graphifyy[all]` install works again (#2277, #2279, #2291). The `mcp` 2.0.0 major dropped the `mcp.types.AnyUrl` re-export and the `Server` decorator-registration API that `graphify/serve.py` uses, so an unpinned resolve broke `graphify-mcp` on every new install with an `ImportError`. The `mcp` and `all` extras now require `mcp>=1,<2` (resolving to 1.29.0) and `starlette>=1.3.1,<2`. Adapting to the mcp 2.x API is tracked as a follow-up. diff --git a/graphify/build.py b/graphify/build.py index 4ea30a45e..9ff7ae6d7 100644 --- a/graphify/build.py +++ b/graphify/build.py @@ -30,7 +30,12 @@ from pathlib import Path import networkx as nx from .ids import make_id, normalize_id as _normalize_id -from .paths import default_graph_json as _default_graph_json +from .paths import ( + default_graph_json as _default_graph_json, + path_exists as _path_exists, + read_text as _read_text, + resolve_path as _resolve_path, +) from .validate import validate_extraction @@ -178,7 +183,7 @@ def _norm_source_file(p: str | None, root: str | None = None) -> str | None: # matching. Only the slow path resolves, so the common lexical match # stays filesystem-free. try: - p = Path(p).resolve().relative_to(Path(root).resolve()).as_posix() + p = _resolve_path(p).relative_to(_resolve_path(root)).as_posix() except (ValueError, OSError): pass return p @@ -202,7 +207,7 @@ def _abs_identity(p: str | None, root: str | None = None) -> str | None: if not pp.is_absolute() and root: pp = Path(root) / q try: - return pp.resolve().as_posix() + return _resolve_path(pp).as_posix() except OSError: return pp.as_posix() @@ -292,14 +297,14 @@ def _infer_merge_root(graph_path: Path) -> str | None: """ try: marker = graph_path.parent / ".graphify_root" - if marker.exists(): - recorded = marker.read_text(encoding="utf-8").strip() + if _path_exists(marker): + recorded = _read_text(marker, encoding="utf-8").strip() if recorded: - return str(Path(recorded).resolve()) + return str(_resolve_path(recorded)) except OSError: pass try: - return str(graph_path.parent.parent.resolve()) + return str(_resolve_path(graph_path.parent.parent)) except Exception: return None @@ -544,7 +549,7 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat root: if given, absolute source_file paths from semantic subagents are made relative to root so all nodes share a consistent path key (#932). """ - _root = str(Path(root).resolve()) if root else None + _root = str(_resolve_path(root)) if root else None # NetworkX <= 3.1 serialised edges as "links"; remap to "edges" for compatibility. if "edges" not in extraction and "links" in extraction: extraction = dict(extraction, edges=extraction["links"]) @@ -1140,12 +1145,12 @@ def _load_existing_graph(graph_path: Path) -> "tuple[list, list, list] | None": exists but cannot be parsed — callers must refuse to overwrite rather than silently replace a possibly-recoverable graph. """ - if not graph_path.exists(): + if not _path_exists(graph_path): return None from graphify.security import check_graph_file_size_cap check_graph_file_size_cap(graph_path) try: - data = json.loads(graph_path.read_text(encoding="utf-8")) + data = json.loads(_read_text(graph_path, encoding="utf-8")) except (json.JSONDecodeError, OSError) as exc: raise RuntimeError( f"Cannot read {graph_path} for incremental merge: {exc}. " @@ -1196,7 +1201,7 @@ def merge_raw_extraction( existing_nodes, existing_edges, existing_hyperedges = loaded _eff_root = ( - str(Path(root).resolve()) if root is not None + str(_resolve_path(root)) if root is not None else _infer_merge_root(graph_path) ) @@ -1293,7 +1298,7 @@ def build_merge( # absolute deleted-file paths never matched the relative node keys and their # nodes survived as ghosts). _eff_root = ( - str(Path(root).resolve()) if root is not None + str(_resolve_path(root)) if root is not None else _infer_merge_root(graph_path) ) @@ -1434,7 +1439,7 @@ def _prune_match(sf: "str | None") -> bool: # Safety check: refuse to shrink the graph silently (#479) # Skip when dedup or prune_sources is active — shrinkage is intentional there. - if graph_path.exists() and not dedup and not prune_sources: + if _path_exists(graph_path) and not dedup and not prune_sources: existing_n = len(existing_nodes) new_n = G.number_of_nodes() if new_n < existing_n: diff --git a/graphify/cache.py b/graphify/cache.py index 354f5213c..0c268b736 100644 --- a/graphify/cache.py +++ b/graphify/cache.py @@ -15,7 +15,22 @@ # shared-output setups. Accepts a relative name ("graphify-out-feature") or an # absolute path ("/shared/graphify-out"). Single source of truth in graphify.paths # (#1423); re-exported here as _GRAPHIFY_OUT for the existing call sites. -from graphify.paths import GRAPHIFY_OUT as _GRAPHIFY_OUT +from graphify.paths import ( + GRAPHIFY_OUT as _GRAPHIFY_OUT, + glob_paths as _glob_paths, + io_path as _io_path, + iterdir_path as _iterdir_path, + logical_path as _logical_path, + make_dirs as _make_dirs, + path_exists as _path_exists, + path_is_dir as _path_is_dir, + path_is_file as _path_is_file, + path_stat as _path_stat, + read_bytes as _read_bytes, + read_text as _read_text, + resolve_path as _resolve_path, + unlink_path as _unlink_path, +) # AST cache entries are the output of graphify's own extractor code, so they # are only valid for the version that wrote them: keying purely on file @@ -47,18 +62,18 @@ def _cleanup_stale_ast_entries(ast_base: Path, current_dir: Path) -> None: if key in _cleaned_ast_dirs: return _cleaned_ast_dirs.add(key) - if not ast_base.is_dir(): + if not _path_is_dir(ast_base): return import shutil - for child in ast_base.iterdir(): + for child in _iterdir_path(ast_base): if child == current_dir: continue try: - if child.is_dir() and child.name.startswith("v"): - shutil.rmtree(child, ignore_errors=True) + if _path_is_dir(child) and child.name.startswith("v"): + shutil.rmtree(_io_path(child), ignore_errors=True) elif child.suffix == ".json": - child.unlink() + _unlink_path(child) except OSError: pass @@ -100,7 +115,7 @@ def prompt_fingerprint(prompt: "str | Path") -> str: like a prompt change and re-bill extraction. """ if isinstance(prompt, Path): - text = prompt.read_text(encoding="utf-8", errors="replace") + text = _read_text(prompt, encoding="utf-8", errors="replace") else: text = prompt normalized = "\n".join( @@ -131,7 +146,7 @@ class #1939 is about, so the two are not inferred from each other. if prompt_file is not None: prompt = Path(prompt_file) try: - st = prompt.stat() + st = _path_stat(prompt) memo_key = (str(prompt), st.st_size, st.st_mtime_ns) if memo_key in _prompt_fp_cache: return _prompt_fp_cache[memo_key] @@ -231,7 +246,7 @@ def _stat_key_to_absolute(key: str, anchor: Path) -> str: def _stat_index_file(root: Path) -> Path: _out = Path(_GRAPHIFY_OUT) - base = _out if _out.is_absolute() else Path(root).resolve() / _out + base = _out if _out.is_absolute() else _resolve_path(root) / _out return base / "cache" / "stat-index.json" @@ -246,13 +261,13 @@ def _ensure_stat_index(root: Path, cache_root: "Path | None" = None) -> None: # in-memory keys stay absolute, but the on-disk index stores in-anchor keys # relative so a moved/cloned corpus still hits (#2199) — same load/save # re-anchoring the detect manifest uses. - _stat_index_root = Path(cache_root if cache_root is not None else root).resolve() - _stat_index_anchor = Path(root).resolve() + _stat_index_root = _resolve_path(cache_root if cache_root is not None else root) + _stat_index_anchor = _resolve_path(root) p = _stat_index_file(_stat_index_root) _stat_index = {} - if p.exists(): + if _path_exists(p): try: - raw = json.loads(p.read_text(encoding="utf-8")) + raw = json.loads(_read_text(p, encoding="utf-8")) if isinstance(raw, dict): for k, v in raw.items(): if not isinstance(k, str): @@ -283,19 +298,23 @@ def _flush_stat_index() -> None: on_disk: dict[str, dict] = {} for k, v in _stat_index.items(): try: - if not os.path.exists(k): + if not _path_exists(k): continue except OSError: continue dk = _stat_key_to_relative(k, _stat_index_anchor) if _stat_index_anchor is not None else k on_disk[dk] = v try: - p.parent.mkdir(parents=True, exist_ok=True) - fd, tmp = tempfile.mkstemp(dir=p.parent, prefix="stat-index.", suffix=".tmp") + _make_dirs(p.parent, exist_ok=True) + fd, tmp = tempfile.mkstemp( + dir=_io_path(p.parent), + prefix="stat-index.", + suffix=".tmp", + ) try: os.write(fd, json.dumps(on_disk, separators=(",", ":")).encode()) os.close(fd) - os.replace(tmp, p) + os.replace(tmp, _io_path(p)) except Exception: try: os.close(fd) @@ -315,9 +334,7 @@ def _normalize_path(path: Path) -> Path: import sys if sys.platform != "win32": return path - s = str(path) - if s.startswith("\\\\?\\"): - s = s[4:] # strip extended-length prefix \\?\ + s = _logical_path(path) return Path(os.path.normcase(s)) @@ -338,7 +355,7 @@ def file_hash(path: Path, root: Path = Path("."), cache_root: "Path | None" = No global _stat_index_dirty p = _normalize_path(Path(path)) root = _normalize_path(Path(root)) - if not p.is_file(): + if not _path_is_file(p): raise IsADirectoryError(f"file_hash requires a file, got: {p}") # The stat index is a cache artifact, so it must follow the cache location @@ -346,7 +363,7 @@ def file_hash(path: Path, root: Path = Path("."), cache_root: "Path | None" = No # graphify-out/cache/stat-index.json inside the analyzed source tree even when # the AST cache itself is redirected to CWD (#1774 completion). _ensure_stat_index(root, cache_root=cache_root) - resolved = p.resolve() + resolved = _resolve_path(p) abs_key = str(resolved) # The salt is the path component that enters the digest (relative to root, or # the absolute-path fallback). The stat-index memo MUST be keyed by it too: @@ -356,13 +373,13 @@ def file_hash(path: Path, root: Path = Path("."), cache_root: "Path | None" = No # and poisoning the persisted stat-index across runs (#1989). Store one digest # per salt so alternating roots don't force re-reads. try: - salt = resolved.relative_to(Path(root).resolve()).as_posix().lower() + salt = resolved.relative_to(_resolve_path(root)).as_posix().lower() except ValueError: salt = resolved.as_posix().lower() st: "os.stat_result | None" = None try: - st = p.stat() + st = _path_stat(p) entry = _stat_index.get(abs_key) if (isinstance(entry, dict) and entry.get("size") == st.st_size @@ -377,7 +394,7 @@ def file_hash(path: Path, root: Path = Path("."), cache_root: "Path | None" = No except OSError: pass - raw = p.read_bytes() + raw = _read_bytes(p) content = _body_content(raw) if p.suffix.lower() == ".md" else raw h = hashlib.sha256() h.update(content) @@ -421,10 +438,10 @@ def cached_word_count(path: Path, root: Path, compute, cache_root: "Path | None" p = _normalize_path(Path(path)) root = _normalize_path(Path(root)) _ensure_stat_index(root, cache_root=cache_root) - abs_key = str(p.resolve()) + abs_key = str(_resolve_path(p)) st: "os.stat_result | None" = None try: - st = p.stat() + st = _path_stat(p) entry = _stat_index.get(abs_key) if (entry and entry.get("size") == st.st_size @@ -465,7 +482,7 @@ def _relativize_source_files_in(payload: dict, root: Path) -> None: :func:`graphify.detect._to_relative_for_storage`. """ try: - root_resolved = Path(root).resolve() + root_resolved = _resolve_path(root) except OSError: return # raw_calls (#: Pascal/Delphi cross-file inherited-call resolution) carries @@ -579,11 +596,11 @@ def _portability_anchors(path: "str | Path", root: "str | Path") -> tuple[list[s from graphify.ids import normalize_id try: - root_resolved = Path(root).resolve() + root_resolved = _resolve_path(root) except OSError: return [], "", [], "" try: - path_resolved = Path(path).resolve() + path_resolved = _resolve_path(path) except (OSError, RuntimeError): path_resolved = Path(path) try: @@ -719,7 +736,7 @@ def _absolutize_source_files_in(payload: dict, root: Path) -> None: absolute ``source_file`` values pass through unchanged. """ try: - root_resolved = Path(root).resolve() + root_resolved = _resolve_path(root) except OSError: return for bucket in ("nodes", "edges", "hyperedges", "raw_calls"): @@ -758,14 +775,14 @@ def cache_dir(root: Path = Path("."), kind: str = "ast", vintage live. """ _out = Path(_GRAPHIFY_OUT) - base = _out if _out.is_absolute() else Path(root).resolve() / _out + base = _out if _out.is_absolute() else _resolve_path(root) / _out d = base / "cache" / kind if kind == "ast": d = d / f"v{_EXTRACTOR_VERSION}" _cleanup_stale_ast_entries(d.parent, d) elif prompt_fp: d = d / f"p{prompt_fp}" - d.mkdir(parents=True, exist_ok=True) + _make_dirs(d, exist_ok=True) return d @@ -813,13 +830,13 @@ def load_cached(path: Path, root: Path = Path("."), kind: str = "ast", prompt_fp = _resolve_prompt_fp(prompt, prompt_file) entry = cache_dir(location, kind, prompt_fp) / f"{h}.json" legacy_hit = False - if prompt_fp and not entry.exists() and allow_legacy: + if prompt_fp and not _path_exists(entry) and allow_legacy: legacy = cache_dir(location, kind) / f"{h}.json" - if legacy.exists(): + if _path_exists(legacy): entry, legacy_hit = legacy, True - if entry.exists(): + if _path_exists(entry): try: - result = json.loads(entry.read_text(encoding="utf-8")) + result = json.loads(_read_text(entry, encoding="utf-8")) except (json.JSONDecodeError, OSError): return None # A ``partial`` entry was produced from a truncated LLM response and @@ -874,7 +891,7 @@ def save_cached(path: Path, result: dict, root: Path = Path("."), kind: str = "a IsADirectoryError from aborting the whole batch. """ p = Path(path) - if not p.is_file(): + if not _path_is_file(p): return # Relativize source_file fields against ``root`` before write so the # cache file on disk is portable across machines and checkout @@ -905,17 +922,21 @@ def save_cached(path: Path, result: dict, root: Path = Path("."), kind: str = "a location = cache_root if cache_root is not None else root target_dir = cache_dir(location, kind, _resolve_prompt_fp(prompt, prompt_file)) entry = target_dir / f"{h}.json" - fd, tmp_path = tempfile.mkstemp(dir=target_dir, prefix=f"{h}.", suffix=".tmp") + fd, tmp_path = tempfile.mkstemp( + dir=_io_path(target_dir), + prefix=f"{h}.", + suffix=".tmp", + ) try: os.write(fd, json.dumps(on_disk).encode()) os.close(fd) try: - os.replace(tmp_path, entry) + os.replace(tmp_path, _io_path(entry)) except PermissionError: # Windows: os.replace can fail with WinError 5 if the target is # briefly locked. Fall back to copy-then-delete. import shutil - shutil.copy2(tmp_path, entry) + shutil.copy2(tmp_path, _io_path(entry)) os.unlink(tmp_path) except Exception: try: @@ -931,38 +952,38 @@ def save_cached(path: Path, result: dict, root: Path = Path("."), kind: str = "a def cached_files(root: Path = Path(".")) -> set[str]: """Return set of file hashes that have a valid cache entry (any kind).""" - base = Path(root).resolve() / _GRAPHIFY_OUT / "cache" + base = _resolve_path(root) / _GRAPHIFY_OUT / "cache" hashes: set[str] = set() # Legacy flat entries - if base.is_dir(): - hashes.update(p.stem for p in base.glob("*.json")) + if _path_is_dir(base): + hashes.update(p.stem for p in _glob_paths(base, "*.json")) # Namespaced entries, all globbed recursively: ast/ has per-version subdirs, # semantic-deep/ holds --mode deep entries (#1894), and both semantic kinds # have per-prompt-fingerprint subdirs alongside pre-fingerprint flat entries # (#1939). for kind in ("ast", "semantic", "semantic-deep"): d = base / kind - if d.is_dir(): - hashes.update(p.stem for p in d.glob("**/*.json")) + if _path_is_dir(d): + hashes.update(p.stem for p in _glob_paths(d, "**/*.json")) return hashes def clear_cache(root: Path = Path(".")) -> None: """Delete all cache entries (ast/, semantic/, semantic-deep/, and legacy flat entries).""" - base = Path(root).resolve() / _GRAPHIFY_OUT / "cache" + base = _resolve_path(root) / _GRAPHIFY_OUT / "cache" # Legacy flat entries - if base.is_dir(): - for f in base.glob("*.json"): - f.unlink() + if _path_is_dir(base): + for f in _glob_paths(base, "*.json"): + _unlink_path(f) # Namespaced entries, all globbed recursively: ast/ has per-version subdirs, # semantic-deep/ holds --mode deep entries (#1894), and both semantic kinds # have per-prompt-fingerprint subdirs (#1939). for kind in ("ast", "semantic", "semantic-deep"): d = base / kind - if d.is_dir(): - for f in d.glob("**/*.json"): - f.unlink() + if _path_is_dir(d): + for f in _glob_paths(d, "**/*.json"): + _unlink_path(f) def prune_semantic_cache(root: Path, live_hashes: set[str]) -> int: @@ -1002,17 +1023,17 @@ def prune_semantic_cache(root: Path, live_hashes: set[str]) -> int: one doc on a future run, never incorrect output. """ _out = Path(_GRAPHIFY_OUT) - base = _out if _out.is_absolute() else Path(root).resolve() / _out + base = _out if _out.is_absolute() else _resolve_path(root) / _out pruned = 0 for kind in ("semantic", "semantic-deep"): semantic_dir = base / "cache" / kind - if not semantic_dir.is_dir(): + if not _path_is_dir(semantic_dir): continue - for entry in semantic_dir.glob("**/*.json"): + for entry in _glob_paths(semantic_dir, "**/*.json"): if entry.stem in live_hashes: continue try: - entry.unlink() + _unlink_path(entry) pruned += 1 except OSError: pass @@ -1172,7 +1193,7 @@ def save_semantic_cache( from collections import defaultdict kind = "semantic" if mode is None else f"semantic-{mode}" - root_path = Path(root).resolve() + root_path = _resolve_path(root) def _normalized(item: dict) -> dict: """Copy of ``item`` with a portable ``source_file`` (#2197). @@ -1214,7 +1235,7 @@ def resolved_source_path(value: str | Path) -> Path: if not path.is_absolute(): path = root_path / path try: - return path.resolve() + return _resolve_path(path) except (OSError, RuntimeError): # Keep the cache write best-effort for inaccessible paths or a # symlink loop emitted by an untrusted semantic result. @@ -1241,7 +1262,9 @@ def resolved_source_path(value: str | Path) -> Path: def group_skipped(fpath: str) -> bool: """Mirror the write-loop skip condition for one source_file group.""" p = resolved_source_path(fpath) - return not p.is_file() or (allowed_paths is not None and p not in allowed_paths) + return not _path_is_file(p) or ( + allowed_paths is not None and p not in allowed_paths + ) # Dangling-reference pruning (#1916). A node group is skipped by the write # loop below when its source_file is not a real file (ghost path) or is @@ -1298,7 +1321,7 @@ def hyperedge_dangles(h: dict) -> bool: skipped_not_file = 0 for fpath, result in by_file.items(): p = resolved_source_path(fpath) - if p.is_file(): + if _path_is_file(p): if allowed_paths is not None and p not in allowed_paths: warnings.warn( "semantic cache skipped out-of-scope source_file " diff --git a/graphify/cargo_introspect.py b/graphify/cargo_introspect.py index fa03ed309..c5b24e45e 100644 --- a/graphify/cargo_introspect.py +++ b/graphify/cargo_introspect.py @@ -5,6 +5,13 @@ from pathlib import Path from typing import Any +from graphify.paths import ( + glob_paths as _glob_paths, + io_path as _io_path, + path_is_file as _path_is_file, + resolve_path as _resolve_path, +) + _CONFIDENCE_EXTRACTED = "EXTRACTED" @@ -20,7 +27,7 @@ def _load_toml(path: Path) -> dict[str, Any]: "--cargo on Python 3.10 needs tomli. Install with: pip install tomli" ) from None - with path.open("rb") as manifest: + with open(_io_path(path), "rb") as manifest: return tomllib.load(manifest) @@ -37,16 +44,16 @@ def _member_manifest_paths(root: Path, root_data: dict[str, Any]) -> list[Path]: for pattern in members: if not isinstance(pattern, str): continue - for member in sorted(root.glob(pattern)): + for member in sorted(_glob_paths(root, pattern)): manifest = member / "Cargo.toml" - if manifest.is_file() and manifest not in paths: + if _path_is_file(manifest) and manifest not in paths: paths.append(manifest) return paths def introspect_cargo(root: str | Path) -> dict[str, Any]: """Return crate nodes and internal dependency edges from Cargo manifests.""" - root_path = Path(root).resolve() + root_path = _resolve_path(root) root_manifest = root_path / "Cargo.toml" root_data = _load_toml(root_manifest) diff --git a/graphify/cli.py b/graphify/cli.py index dc9ed08cc..79c13c278 100644 --- a/graphify/cli.py +++ b/graphify/cli.py @@ -11,7 +11,17 @@ import re import sys import time -from graphify.paths import GRAPHIFY_OUT as _GRAPHIFY_OUT +from graphify.paths import ( + GRAPHIFY_OUT as _GRAPHIFY_OUT, + make_dirs as _make_dirs, + path_exists as _path_exists, + path_is_file as _path_is_file, + path_stat as _path_stat, + read_text as _read_text, + resolve_path as _resolve_path, + unlink_path as _unlink_path, + write_text as _write_text, +) from pathlib import Path @@ -123,7 +133,7 @@ def _resolve(value: str) -> Path: if not p.is_absolute(): p = root / p try: - return p.resolve() + return _resolve_path(p) except (OSError, RuntimeError): return p @@ -191,19 +201,19 @@ def _stale_graph_sources( """ from graphify.paths import nfc try: - data = json.loads(graph_path.read_text(encoding="utf-8")) + data = json.loads(_read_text(graph_path, encoding="utf-8")) except Exception: return [] if not isinstance(data, dict): return [] try: - root_res = scan_root.resolve() + root_res = _resolve_path(scan_root) except (OSError, RuntimeError): root_res = scan_root # /graphify-out/graph.json — relative source_files may be anchored here. out_base = graph_path.parent.parent try: - out_base = out_base.resolve() + out_base = _resolve_path(out_base) except (OSError, RuntimeError): pass @@ -214,7 +224,7 @@ def _within_root(p: Path) -> bool: except ValueError: pass try: - p.resolve().relative_to(root_res) + _resolve_path(p).relative_to(root_res) return True except (ValueError, OSError, RuntimeError): return False @@ -226,7 +236,7 @@ def _in_seen(p: Path) -> bool: if nfc(str(p)) in seen_nfc: return True try: - return nfc(str(p.resolve())) in seen_nfc + return nfc(str(_resolve_path(p))) in seen_nfc except (OSError, RuntimeError): return False @@ -252,7 +262,7 @@ def _in_seen(p: Path) -> bool: def _provably_excluded(c: Path) -> bool: spellings = [nfc(str(c))] try: - spellings.append(nfc(str(c.resolve()))) + spellings.append(nfc(str(_resolve_path(c)))) except (OSError, RuntimeError): pass for s in spellings: @@ -301,7 +311,7 @@ def _provably_excluded(c: Path) -> bool: alive = [] for c in in_root: try: - if c.exists(): + if _path_exists(c): alive.append(c) except OSError: pass @@ -563,7 +573,7 @@ def _run_hook_guard(kind: str, strict: bool = False) -> None: # candidate that resolves outside that root is out-of-project. root = Path(os.environ.get("CLAUDE_PROJECT_DIR") or os.getcwd()) try: - root = root.resolve() + root = _resolve_path(root) except (OSError, RuntimeError): pass path_vals = [str(t.get("file_path") or ""), str(t.get("path") or "")] @@ -576,7 +586,7 @@ def _run_hook_guard(kind: str, strict: bool = False) -> None: in_project = True # relative -> anchored at cwd == in project break try: - p.resolve().relative_to(root) + _resolve_path(p).relative_to(root) in_project = True break except (ValueError, OSError, RuntimeError): @@ -585,7 +595,7 @@ def _run_hook_guard(kind: str, strict: bool = False) -> None: return # One stat for existence + mtime of the graph. try: - gmtime = os.stat(str(out_path("graph.json"))).st_mtime + gmtime = _path_stat(out_path("graph.json")).st_mtime except OSError: return # #1840 (b): stale-for-target -> soften, never block. The target file @@ -594,11 +604,11 @@ def _run_hook_guard(kind: str, strict: bool = False) -> None: fp = str(t.get("file_path") or "") if fp: try: - stale = os.stat(fp).st_mtime > gmtime + stale = _path_stat(fp).st_mtime > gmtime except OSError: stale = False try: - if out_path("needs_update").exists(): + if _path_exists(out_path("needs_update")): stale = True except Exception: pass @@ -629,16 +639,16 @@ def _target_is_indexed(file_path: str, root: "Path") -> bool: return True try: mp = out_path("manifest.json") - st = mp.stat() + st = _path_stat(mp) if st.st_size > 2_000_000: return True - manifest = json.loads(mp.read_text(encoding="utf-8")) + manifest = json.loads(_read_text(mp, encoding="utf-8")) if not isinstance(manifest, dict) or not manifest: return True p = Path(file_path) rels = set() try: - rels.add(p.resolve().relative_to(root).as_posix()) + rels.add(_resolve_path(p).relative_to(root).as_posix()) except (ValueError, OSError, RuntimeError): pass rels.add(p.name) @@ -1482,7 +1492,7 @@ def dispatch_command(cmd: str) -> None: summary = diagnose_file( graph_path, directed=directed, - root=Path(".").resolve(), + root=_resolve_path(Path(".")), max_examples=max_examples, extract_path=extract_path, ) @@ -1532,7 +1542,7 @@ def dispatch_command(cmd: str) -> None: elif cmd == "watch": watch_path = Path(sys.argv[2]) if len(sys.argv) > 2 else Path(".") - if not watch_path.exists(): + if not _path_exists(watch_path): print(f"error: path not found: {watch_path}", file=sys.stderr) sys.exit(1) from graphify.watch import watch as _watch @@ -1606,7 +1616,7 @@ def dispatch_command(cmd: str) -> None: if watch_path is None: watch_path = Path(".") graph_json = graph_override if graph_override is not None else watch_path / _GRAPHIFY_OUT / "graph.json" - if not graph_json.exists(): + if not _path_exists(graph_json): print( f"error: no graph found at {graph_json} — run /graphify first", file=sys.stderr, @@ -1636,7 +1646,7 @@ def dispatch_command(cmd: str) -> None: except ValueError: _over_cap = True try: - _over_cap_bytes = graph_json.stat().st_size + _over_cap_bytes = _path_stat(graph_json).st_size except OSError: _over_cap_bytes = -1 print( @@ -1644,7 +1654,7 @@ def dispatch_command(cmd: str) -> None: f"falling back to community-aggregation view (node_limit=5000)", file=sys.stderr, ) - _raw = json.loads(graph_json.read_text(encoding="utf-8")) + _raw = json.loads(_read_text(graph_json, encoding="utf-8")) _directed = bool(_raw.get("directed", False)) G = build_from_json(_raw, directed=_directed) print(f"Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges") @@ -1892,11 +1902,11 @@ def dispatch_command(cmd: str) -> None: else: # Try to recover the scan root saved by the last full build saved = Path(_GRAPHIFY_OUT) / ".graphify_root" - if saved.exists(): - watch_path = Path(saved.read_text(encoding="utf-8").strip()) + if _path_exists(saved): + watch_path = Path(_read_text(saved, encoding="utf-8").strip()) else: watch_path = Path(".") - if not watch_path.exists(): + if not _path_exists(watch_path): print(f"error: path not found: {watch_path}", file=sys.stderr) sys.exit(1) from graphify.watch import _rebuild_code @@ -1946,7 +1956,7 @@ def dispatch_command(cmd: str) -> None: sys.exit(1) from graphify.watch import check_update - check_update(Path(sys.argv[2]).resolve()) + check_update(_resolve_path(sys.argv[2])) sys.exit(0) elif cmd == "tree": # Emit a D3 v7 collapsible-tree HTML view of graph.json: @@ -2576,10 +2586,10 @@ def _to_simple(g: "_nx.Graph") -> "_nx.Graph": has_path = True if sys.argv[2].startswith("-"): has_path = False - target = Path(".").resolve() + target = _resolve_path(Path(".")) else: - target = Path(sys.argv[2]).resolve() - if not target.exists(): + target = _resolve_path(Path(sys.argv[2])) + if not _path_exists(target): print(f"error: path not found: {target}", file=sys.stderr) sys.exit(1) @@ -2740,9 +2750,9 @@ def _parse_float(name: str, raw: str) -> float: # Resolve output dir. The user-facing contract is "/graphify-out/" # so a fresh checkout writes graphify-out/ at the project root, matching # the skill.md pipeline. - out_root = (out_dir.resolve() if out_dir else target) + out_root = (_resolve_path(out_dir) if out_dir else target) graphify_out = out_root / _GRAPHIFY_OUT - graphify_out.mkdir(parents=True, exist_ok=True) + _make_dirs(graphify_out, exist_ok=True) # Persist corpus-shaping options so later update/watch/hook rebuilds # use the same file set as the initial extraction (#1886). from graphify.watch import ( @@ -2783,13 +2793,13 @@ def _parse_float(name: str, raw: str) -> float: # and genuinely-deleted sources against the current corpus, so doc/ # paper/image nodes survive a --code-only rebuild instead of being # dropped with the rest of the committed graph. - incremental_mode = existing_graph_path.exists() if has_path else False + incremental_mode = _path_exists(existing_graph_path) if has_path else False # --force: full scan, not the manifest-gated incremental diff — a warm # unchanged tree would otherwise dispatch zero files (#1894). incremental_mode = incremental_mode and not force if force: print("[graphify extract] --force: full re-scan, semantic cache reads skipped") - elif incremental_mode and not manifest_path.exists(): + elif incremental_mode and not _path_exists(manifest_path): print( "[graphify extract] manifest.json missing; using existing " "graph.json as the incremental baseline (all files re-checked; " @@ -3224,7 +3234,7 @@ def _progress(idx: int, total: int, _result: dict) -> None: _abs = Path(_fp) if not _abs.is_absolute(): _abs = Path(target) / _abs - if not _abs.is_file(): + if not _path_is_file(_abs): continue # deleted/missing — leave out so its entry is pruned try: _live_hashes.add(_file_hash(_abs, target, cache_root=out_root)) @@ -3316,7 +3326,7 @@ def _invalidate_file_manifest_for_db_graph() -> None: if has_path: return try: - manifest_path.unlink(missing_ok=True) + _unlink_path(manifest_path, missing_ok=True) except OSError as exc: print(f"error: could not invalidate file manifest: {exc}", file=sys.stderr) sys.exit(1) @@ -3445,8 +3455,10 @@ def _invalidate_file_manifest_for_db_graph() -> None: # relativize deleted-file paths correctly even for a custom --out # (its grandparent-of-graph.json fallback points at the wrong dir # otherwise, and deleted files never prune — #2012/#1571). - (graphify_out / ".graphify_root").write_text( - str(Path(target).resolve()), encoding="utf-8" + _write_text( + graphify_out / ".graphify_root", + str(_resolve_path(target)), + encoding="utf-8", ) except OSError: pass @@ -3579,15 +3591,19 @@ def _invalidate_file_manifest_for_db_graph() -> None: try: # See the --no-cluster path above: persist the scan root so build_merge # can relativize deleted-file paths under a custom --out (#2012/#1571). - (graphify_out / ".graphify_root").write_text( - str(Path(target).resolve()), encoding="utf-8" + _write_text( + graphify_out / ".graphify_root", + str(_resolve_path(target)), + encoding="utf-8", ) except OSError: pass stages.mark("export") if merged.get("output_tokens", 0) > 0: - (graphify_out / ".graphify_semantic_marker").write_text( - json.dumps({"output_tokens": merged["output_tokens"]}), encoding="utf-8" + _write_text( + graphify_out / ".graphify_semantic_marker", + json.dumps({"output_tokens": merged["output_tokens"]}), + encoding="utf-8", ) if global_merge: from graphify.global_graph import global_add as _global_add @@ -3698,19 +3714,20 @@ def _invalidate_file_manifest_for_db_graph() -> None: i += 1 else: i += 1 - files = [f for f in files_from.read_text(encoding="utf-8").splitlines() if f.strip()] + files = [f for f in _read_text(files_from, encoding="utf-8").splitlines() if f.strip()] cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache( files, root, mode=cache_mode, prompt_file=prompt_file ) out = root / _GRAPHIFY_OUT - out.mkdir(parents=True, exist_ok=True) + _make_dirs(out, exist_ok=True) if cached_nodes or cached_edges or cached_hyperedges: - (out / ".graphify_cached.json").write_text( + _write_text( + out / ".graphify_cached.json", json.dumps({"nodes": cached_nodes, "edges": cached_edges, "hyperedges": cached_hyperedges}, ensure_ascii=False), encoding="utf-8", ) - (out / ".graphify_uncached.txt").write_text("\n".join(uncached), encoding="utf-8") + _write_text(out / ".graphify_uncached.txt", "\n".join(uncached), encoding="utf-8") print(f"Cache: {len(files) - len(uncached)} hit, {len(uncached)} miss") elif cmd == "merge-chunks": @@ -3818,8 +3835,16 @@ def _invalidate_file_manifest_for_db_graph() -> None: print("error: --out required", file=sys.stderr) sys.exit(1) empty: dict = {"nodes": [], "edges": [], "hyperedges": []} - cached_data = json.loads(cached_path.read_text(encoding="utf-8")) if cached_path and cached_path.exists() else empty - new_data = json.loads(new_path.read_text(encoding="utf-8")) if new_path and new_path.exists() else empty + cached_data = ( + json.loads(_read_text(cached_path, encoding="utf-8")) + if cached_path and _path_exists(cached_path) + else empty + ) + new_data = ( + json.loads(_read_text(new_path, encoding="utf-8")) + if new_path and _path_exists(new_path) + else empty + ) seen_ids2: set[str] = set() all_nodes: list[dict] = [] for n in cached_data.get("nodes", []) + new_data.get("nodes", []): @@ -3831,12 +3856,16 @@ def _invalidate_file_manifest_for_db_graph() -> None: "edges": cached_data.get("edges", []) + new_data.get("edges", []), "hyperedges": cached_data.get("hyperedges", []) + new_data.get("hyperedges", []), } - out_path2.parent.mkdir(parents=True, exist_ok=True) + _make_dirs(out_path2.parent, exist_ok=True) from graphify.paths import write_json_atomic as _wja _wja(out_path2, merged2, ensure_ascii=False) print(f"Merged: {len(merged2['nodes'])} nodes, {len(merged2['edges'])} edges") - elif Path(cmd).exists() or cmd in (".", "..") or cmd.startswith(("./", "../", "/", "~")): + elif ( + _path_exists(Path(cmd).expanduser()) + or cmd in (".", "..") + or cmd.startswith(("./", "../", "/", "~")) + ): # User ran `graphify ` directly — treat as `graphify extract `. # Common when following the PowerShell note in README (`graphify .`) or # copy-pasting skill invocations without the leading slash. diff --git a/graphify/detect.py b/graphify/detect.py index 0b569e4b7..2e3a1984b 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -15,7 +15,22 @@ convert_google_workspace_file, google_workspace_enabled, ) -from graphify.paths import GRAPHIFY_OUT, GRAPHIFY_OUT_NAME, out_path +from graphify.paths import ( + GRAPHIFY_OUT, + GRAPHIFY_OUT_NAME, + io_path as _os_path, + make_dirs as _make_dirs, + out_path, + path_exists as _path_exists, + path_is_dir as _path_is_dir, + path_is_file as _path_is_file, + path_is_symlink as _path_is_symlink, + path_stat as _path_stat, + read_text as _read_text, + resolve_path as _resolve_path, + walk_path as _walk_path, + write_text as _write_text, +) class FileType(str, Enum): @@ -52,7 +67,7 @@ class FileType(str, Enum): def _file_within_size_cap(path: Path, cap: int = _OFFICE_MAX_RAW_BYTES) -> bool: """True if *path* exists and its on-disk size is within *cap*.""" try: - return path.stat().st_size <= cap + return _path_stat(path).st_size <= cap except OSError: return False @@ -72,7 +87,7 @@ def _zip_within_caps(path: Path) -> bool: if not _file_within_size_cap(path): return False try: - with zipfile.ZipFile(path) as zf: + with zipfile.ZipFile(_os_path(path)) as zf: infos = zf.infolist() compressed = sum(i.compress_size for i in infos) or 1 declared = sum(i.file_size for i in infos) @@ -296,7 +311,7 @@ def _looks_like_paper(path: Path) -> bool: """Heuristic: does this text file read like an academic paper?""" try: # Only scan first 3000 chars for speed - text = path.read_text(encoding="utf-8", errors="ignore")[:3000] + text = _read_text(path, encoding="utf-8", errors="ignore")[:3000] hits = sum(1 for pattern in _PAPER_SIGNALS if pattern.search(text)) return hits >= _PAPER_SIGNAL_THRESHOLD except Exception: @@ -460,7 +475,7 @@ def _shebang_interpreter(path: Path) -> str | None: no shebang / the file is unreadable / parsing fails. """ try: - with path.open("rb") as f: + with open(_os_path(path), "rb") as f: first = f.read(256) if not first.startswith(b"#!"): return None @@ -530,7 +545,7 @@ def extract_pdf_text(path: Path) -> str: return "" try: from pypdf import PdfReader - reader = PdfReader(str(path)) + reader = PdfReader(_os_path(path)) pages = [] for page in reader.pages: text = page.extract_text() @@ -548,7 +563,7 @@ def docx_to_markdown(path: Path) -> str: try: from docx import Document from docx.oxml.ns import qn - doc = Document(str(path)) + doc = Document(_os_path(path)) lines = [] for para in doc.paragraphs: style = para.style.name if para.style else "" @@ -589,7 +604,7 @@ def xlsx_to_markdown(path: Path) -> str: return "" try: import openpyxl - wb = openpyxl.load_workbook(str(path), read_only=True, data_only=True) + wb = openpyxl.load_workbook(_os_path(path), read_only=True, data_only=True) sections = [] for sheet_name in wb.sheetnames: ws = wb[sheet_name] @@ -630,7 +645,7 @@ def _nid(*parts: str) -> str: return {"nodes": [], "edges": []} try: - wb = openpyxl.load_workbook(str(path), read_only=False, data_only=True) + wb = openpyxl.load_workbook(_os_path(path), read_only=False, data_only=True) except Exception: return {"nodes": [], "edges": []} @@ -722,7 +737,7 @@ def convert_office_file(path: Path, out_dir: Path, root: "Path | None" = None) - if not text.strip(): return None - out_dir.mkdir(parents=True, exist_ok=True) + _make_dirs(out_dir, exist_ok=True) # Use a stable name derived from the original path to avoid collisions. # Hash the path RELATIVE to the scan root, not the absolute path: the # absolute form salts the name with the checkout location, so the same @@ -741,12 +756,12 @@ def convert_office_file(path: Path, out_dir: Path, root: "Path | None" = None) - # Default layout: out_dir is //converted. root = out_dir.parent.parent try: - key = path.resolve().relative_to(Path(root).resolve()).as_posix() + key = _resolve_path(path).relative_to(_resolve_path(root)).as_posix() except (ValueError, OSError): # Not under the scan root (custom GRAPHIFY_OUT layouts, --include # sources, direct API callers): keep the previous absolute form rather # than guessing, so behavior is unchanged for those cases. - key = str(path.resolve()) + key = str(_resolve_path(path)) normalized_path = unicodedata.normalize("NFC", key) name_hash = hashlib.sha256(normalized_path.encode()).hexdigest()[:8] out_path = out_dir / f"{path.stem}_{name_hash}.md" @@ -758,12 +773,16 @@ def convert_office_file(path: Path, out_dir: Path, root: "Path | None" = None) - # incremental hash check then correctly picks up. An unchanged source keeps # its (newer-or-equal) sidecar untouched so it never churns (#1226). try: - if out_path.exists() and os.stat(_os_path(out_path)).st_mtime >= os.stat(_os_path(path)).st_mtime: + if ( + _path_exists(out_path) + and _path_stat(out_path).st_mtime >= _path_stat(path).st_mtime + ): return out_path except OSError: - if out_path.exists(): + if _path_exists(out_path): return out_path - out_path.write_text( + _write_text( + out_path, f"\n\n{text}", encoding="utf-8", ) @@ -835,13 +854,19 @@ def _has_venv_markers(d: "Path") -> bool: ``conda-meta/`` (``conda create -p ./env`` writes no pyvenv.cfg). """ try: - if (d / "pyvenv.cfg").is_file(): - return True - if (d / "bin" / "activate").is_file() or (d / "Scripts" / "activate").is_file(): + if _path_is_file(d / "pyvenv.cfg"): return True - if next(d.glob("lib/python*"), None) is not None: + if _path_is_file(d / "bin" / "activate") or _path_is_file( + d / "Scripts" / "activate" + ): return True - if (d / "conda-meta").is_dir(): + try: + with os.scandir(_os_path(d / "lib")) as entries: + if any(entry.name.startswith("python") for entry in entries): + return True + except OSError: + pass + if _path_is_dir(d / "conda-meta"): return True except OSError: pass @@ -866,8 +891,9 @@ def _is_noise_dir(part: str, parent: "Path | None" = None) -> bool: if parent.name in _JS_SNAPSHOT_TEST_ROOTS: return True try: - if next(snap_dir.glob("*.snap"), None) is not None: - return True + with os.scandir(_os_path(snap_dir)) as entries: + if any(entry.name.endswith(".snap") for entry in entries): + return True except OSError: pass return False @@ -912,10 +938,10 @@ def _parse_gitignore_line(raw: str) -> str: def _find_vcs_root(start: Path) -> Path | None: """Walk upward from start; return the first directory containing a VCS marker.""" - current = start.resolve() + current = _resolve_path(start) home = Path.home() while True: - if any((current / m).exists() for m in _VCS_MARKERS): + if any(_path_exists(current / m) for m in _VCS_MARKERS): return current parent = current.parent if parent == current or current == home: @@ -936,33 +962,33 @@ def _git_info_exclude(vcs_root: Path) -> Path | None: """ dot_git = vcs_root / ".git" git_dir: Path | None = None - if dot_git.is_dir(): + if _path_is_dir(dot_git): git_dir = dot_git - elif dot_git.is_file(): + elif _path_is_file(dot_git): try: - content = dot_git.read_text(encoding="utf-8", errors="ignore").strip() + content = _read_text(dot_git, encoding="utf-8", errors="ignore").strip() except OSError: content = "" if content.startswith("gitdir:"): gd = Path(content[len("gitdir:"):].strip()) if not gd.is_absolute(): - gd = (vcs_root / gd).resolve() + gd = _resolve_path(vcs_root / gd) git_dir = gd # A linked worktree's gitdir holds a `commondir` file pointing at the # shared git dir, where info/exclude actually lives. commondir = gd / "commondir" - if commondir.exists(): + if _path_exists(commondir): try: - cd_raw = commondir.read_text(encoding="utf-8", errors="ignore").strip() + cd_raw = _read_text(commondir, encoding="utf-8", errors="ignore").strip() except OSError: cd_raw = "" if cd_raw: cd = Path(cd_raw) - git_dir = cd if cd.is_absolute() else (gd / cd).resolve() + git_dir = cd if cd.is_absolute() else _resolve_path(gd / cd) if git_dir is None: return None exclude = git_dir / "info" / "exclude" - return exclude if exclude.is_file() else None + return exclude if _path_is_file(exclude) else None def _load_dir_own_ignore(d: Path, *, gitignore: bool = True) -> list[tuple[Path, str]]: @@ -984,8 +1010,8 @@ def _load_dir_own_ignore(d: Path, *, gitignore: bool = True) -> list[tuple[Path, patterns: list[tuple[Path, str]] = [] for fname in ((".gitignore", ".graphifyignore") if gitignore else (".graphifyignore",)): ignore_file = d / fname - if ignore_file.exists(): - for raw in ignore_file.read_text(encoding="utf-8-sig", errors="ignore").splitlines(): + if _path_exists(ignore_file): + for raw in _read_text(ignore_file, encoding="utf-8-sig", errors="ignore").splitlines(): line = _parse_gitignore_line(raw) if line: patterns.append((d, line)) @@ -1006,7 +1032,7 @@ def _load_graphifyignore(root: Path, *, gitignore: bool = True) -> list[tuple[Pa scan root are picked up live during the os.walk in `detect()` instead, since they aren't known until the walk reaches them (#1206). """ - root = root.resolve() + root = _resolve_path(root) ceiling = _find_vcs_root(root) or root # Collect ancestor dirs from ceiling down to root (outer → inner) @@ -1027,7 +1053,7 @@ def _load_graphifyignore(root: Path, *, gitignore: bool = True) -> list[tuple[Pa # re-include still override it (#1810). info_exclude = _git_info_exclude(ceiling) if gitignore else None if info_exclude is not None: - for raw in info_exclude.read_text(encoding="utf-8-sig", errors="ignore").splitlines(): + for raw in _read_text(info_exclude, encoding="utf-8-sig", errors="ignore").splitlines(): line = _parse_gitignore_line(raw) if line: patterns.append((ceiling, line)) @@ -1129,7 +1155,7 @@ def _matches(rel: str, p: str, path_relative: bool) -> bool: continue # target outside this pattern's anchor: cannot match if rel_anchor != ".": matched = _matches(rel_anchor, p, path_relative=path_relative) - if matched and directory_only and not target.is_dir(): + if matched and directory_only and not _path_is_dir(target): matched = False if matched: @@ -1163,9 +1189,10 @@ def _auto_follow_symlinks(root: Path) -> bool: explicit opt-in, and out-of-root symlink targets are never indexed. """ try: - for p in root.iterdir(): - if p.is_symlink(): - return True + with os.scandir(_os_path(root)) as entries: + for entry in entries: + if entry.is_symlink(): + return True except (OSError, PermissionError): pass return False @@ -1174,19 +1201,19 @@ def _auto_follow_symlinks(root: Path) -> bool: def _resolves_under_root(path: Path, root: Path) -> bool: """True when ``path`` resolves to a target inside ``root``.""" try: - path.resolve().relative_to(root.resolve()) + _resolve_path(path).relative_to(_resolve_path(root)) except (OSError, RuntimeError, ValueError): return False return True def detect(root: Path, *, follow_symlinks: bool | None = None, google_workspace: bool | None = None, extra_excludes: list[str] | None = None, cache_root: Path | None = None, gitignore: bool = True) -> dict: - root = root.resolve() + root = _resolve_path(root) # .graphifyinclude support was removed (#2112): its loader and matchers had # no consumers, so the file has been a silent no-op since dot directories # became indexed by default (#873). Surface that once per scan so a # leftover allowlist file is not a silent behavior change. - if (root / ".graphifyinclude").is_file(): + if _path_is_file(root / ".graphifyinclude"): import sys as _sys print( "[graphify] WARNING: .graphifyinclude is no longer supported " @@ -1236,7 +1263,7 @@ def _wc(path: Path) -> int: # Always include graphify-out/memory/ - query results filed back into the graph memory_dir = root / GRAPHIFY_OUT / "memory" scan_paths = [root] - if memory_dir.exists(): + if _path_exists(memory_dir): scan_paths.append(memory_dir) seen: set[Path] = set() @@ -1261,14 +1288,20 @@ def _on_walk_error(err: OSError) -> None: ) for scan_root in scan_paths: - in_memory_tree = memory_dir.exists() and str(scan_root).startswith(str(memory_dir)) - for dirpath, dirnames, filenames in os.walk( + in_memory_tree = _path_exists(memory_dir) and str(scan_root).startswith( + str(memory_dir) + ) + for dirpath, dirnames, filenames in _walk_path( scan_root, followlinks=follow_symlinks, onerror=_on_walk_error ): dp = Path(dirpath) - if follow_symlinks and os.path.islink(dirpath): - real = os.path.realpath(dirpath) - parent_real = os.path.realpath(os.path.dirname(dirpath)) + # os.walk must stay in the extended Windows namespace so every + # recursive scandir call can reach long descendants. Convert the + # yielded path back immediately: graph/cache identities must use the + # ordinary UNC/drive spelling, not the transport-only \\?\ prefix. + if follow_symlinks and _path_is_symlink(dp): + real = str(_resolve_path(dp)) + parent_real = str(_resolve_path(dp.parent)) if parent_real == real or parent_real.startswith(real + os.sep): dirnames.clear() continue @@ -1310,7 +1343,7 @@ def _on_walk_error(err: OSError) -> None: safe_dirs: list[str] = [] for d in dirnames: child = dp / d - if child.is_symlink() and not _resolves_under_root(child, root): + if _path_is_symlink(child) and not _resolves_under_root(child, root): skipped_sensitive.append(str(child) + " [symlink target outside scan root]") continue safe_dirs.append(d) @@ -1329,7 +1362,7 @@ def _on_walk_error(err: OSError) -> None: for p in all_files: # For memory dir files, skip hidden/noise filtering - in_memory = memory_dir.exists() and str(p).startswith(str(memory_dir)) + in_memory = _path_exists(memory_dir) and str(p).startswith(str(memory_dir)) if not in_memory: # Skip files inside our own converted/ dir (avoid re-processing sidecars) if str(p).startswith(str(converted_dir)): @@ -1421,38 +1454,10 @@ def _on_walk_error(err: OSError) -> None: "ignored": sorted(ignored), "pruned_noise_dirs": sorted(pruned_noise), "graphifyignore_patterns": len(ignore_patterns), - "scan_root": str(root.resolve()), + "scan_root": str(_resolve_path(root)), } -def _os_path(path: Path) -> str: - r"""Return an OS path string safe for open()/stat() on Windows long paths. - - On win32, paths longer than the legacy MAX_PATH (260 chars) are rejected by - the plain file APIs unless prefixed with the extended-length marker ``\\?\`` - (which also requires a fully-qualified path). Without it, _md5_file / - save_manifest / count_words silently fail to hash deeply-nested files, so - their manifest entry never stabilizes and detect_incremental re-flags them - as changed on every run (#1655). cache._normalize_path strips this prefix - for stable KEYS; this adds it for I/O. Non-win32 and already-prefixed paths - pass through unchanged. - """ - import sys - if sys.platform != "win32": - return str(path) - s = str(path) - if s.startswith("\\\\?\\"): - return s - try: - s = os.path.abspath(s) # \\?\ requires a fully-qualified path - except Exception: - return str(path) - if s.startswith("\\\\"): - # UNC share \\server\share -> \\?\UNC\server\share - return "\\\\?\\UNC\\" + s[2:] - return "\\\\?\\" + s - - def _md5_file(path: Path) -> str: """MD5 of file contents streamed in 64KB chunks — for change detection only.""" import hashlib as _hl @@ -1470,7 +1475,7 @@ def _stat_and_hash(path_str: str) -> tuple[str, float, str] | None: """Stat + MD5 a single file; returns None on OSError (e.g. deleted mid-run).""" try: p = Path(path_str) - return path_str, os.stat(_os_path(p)).st_mtime, _md5_file(p) + return path_str, _path_stat(p).st_mtime, _md5_file(p) except OSError: return None @@ -1511,7 +1516,7 @@ def _to_relative_for_storage(key: str, root: Path) -> str: if not p.is_absolute(): return key try: - base = _nfc(str(Path(root).resolve())) + base = _nfc(str(_resolve_path(root))) rel = os.path.relpath(_nfc(str(p)), base) except (ValueError, OSError): return key # outside root (e.g. Windows cross-drive) @@ -1539,7 +1544,7 @@ def _to_absolute_from_storage(key: str, root: Path) -> str: return str(p) # NFC the joined result so an NFD-resolved root + relative key lands on # the same form load_manifest / detect_incremental compare against. - return _nfc(str(Path(root).resolve() / p)) + return _nfc(str(_resolve_path(root) / p)) def load_manifest( @@ -1559,7 +1564,7 @@ def load_manifest( form still matches a scan that yields the other (#2221). """ try: - raw = json.loads(Path(manifest_path).read_text(encoding="utf-8")) + raw = json.loads(_read_text(manifest_path, encoding="utf-8")) except Exception: return {} if not isinstance(raw, dict): @@ -1628,7 +1633,7 @@ def _path_index(paths: set[str] | list[str] | None) -> set[str] | None: scan_set = _path_index(scan_corpus) clear_set = _path_index(clear_semantic) try: - root_res: Path | None = Path(root).resolve() if root is not None else None + root_res: Path | None = _resolve_path(root) if root is not None else None except (OSError, RuntimeError): root_res = Path(root) if root is not None else None @@ -1636,7 +1641,7 @@ def _in_scan(path_str: str) -> bool: if path_str in scan_set or _nfc(path_str) in scan_set: return True try: - resolved = str(Path(path_str).resolve()) + resolved = str(_resolve_path(path_str)) return resolved in scan_set or _nfc(resolved) in scan_set except (OSError, RuntimeError): return False @@ -1645,7 +1650,7 @@ def _in_clear(path_str: str) -> bool: if path_str in clear_set or _nfc(path_str) in clear_set: return True try: - resolved = str(Path(path_str).resolve()) + resolved = str(_resolve_path(path_str)) return resolved in clear_set or _nfc(resolved) in clear_set except (OSError, RuntimeError): return False @@ -1662,7 +1667,7 @@ def _in_root(path_str: str) -> bool: except ValueError: pass try: - p.resolve().relative_to(root_res) + _resolve_path(p).relative_to(root_res) return True except (ValueError, OSError, RuntimeError): return False @@ -1689,7 +1694,7 @@ def _normalise_entry(entry): if normalised is None: continue try: - if not Path(f).exists(): + if not _path_exists(f): continue except OSError: continue @@ -1803,7 +1808,7 @@ def detect_incremental( # Manifest keys are NFC; scan paths may arrive NFD (#2221). stored = manifest.get(_nfc(f)) try: - current_mtime = os.stat(_os_path(Path(f))).st_mtime + current_mtime = _path_stat(f).st_mtime except Exception: current_mtime = 0 @@ -1859,7 +1864,7 @@ def detect_incremental( if _nfc(f) in current_files: continue try: - alive = Path(f).exists() + alive = _path_exists(f) except OSError: alive = False (excluded_files if alive else deleted_files).append(f) diff --git a/graphify/diagnostics.py b/graphify/diagnostics.py index fcb9a11cf..6ce9405bb 100644 --- a/graphify/diagnostics.py +++ b/graphify/diagnostics.py @@ -11,6 +11,11 @@ import networkx as nx +from graphify.paths import ( + path_exists as _path_exists, + read_text as _read_file_text, +) + _SUPPRESSION_DECL_RE = re.compile(r"^\s*(?Pseen_[A-Za-z0-9_]+)\s*[:=]") _TYPE_TUPLE_RE = re.compile(r"set\[tuple\[(?P[^\]]+)\]\]") @@ -122,7 +127,7 @@ def _tuple_arity_from_annotation(line: str) -> int: def scan_producer_suppression_sites(path: str | Path) -> dict[str, Any]: """Find likely `seen_*` producer-suppression sets in an extractor file.""" source_path = Path(path) - if not source_path.exists(): + if not _path_exists(source_path): return { "path": str(source_path), "total_sites": 0, @@ -131,7 +136,7 @@ def scan_producer_suppression_sites(path: str | Path) -> dict[str, Any]: } sites: list[dict[str, Any]] = [] - lines = source_path.read_text(encoding="utf-8").splitlines() + lines = _read_file_text(source_path, encoding="utf-8").splitlines() for lineno, line in enumerate(lines, start=1): match = _SUPPRESSION_DECL_RE.match(line) if not match: @@ -284,7 +289,7 @@ def _read_json_file(path: str | Path) -> dict[str, Any]: json_path = Path(path) check_graph_file_size_cap(json_path) try: - data = json.loads(json_path.read_text(encoding="utf-8")) + data = json.loads(_read_file_text(json_path, encoding="utf-8")) except (json.JSONDecodeError, OSError) as exc: raise RuntimeError( f"Cannot parse {json_path}: {exc}. " diff --git a/graphify/extract.py b/graphify/extract.py index 516cd374b..eb6f71211 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -56,7 +56,17 @@ from graphify.extractors.verilog import extract_verilog # noqa: F401 from graphify.extractors.zig import extract_zig # noqa: F401 from graphify.security import sanitize_metadata -from graphify.paths import disambiguate_ambiguous_candidates +from graphify.paths import ( + disambiguate_ambiguous_candidates, + iterdir_path as _iterdir_path, + path_exists as _path_exists, + path_is_file as _path_is_file, + path_is_symlink as _path_is_symlink, + read_bytes as _read_file_bytes, + read_text as _read_file_text, + resolve_path as _resolve_path, + walk_path as _walk_path, +) from graphify.extractors.models import LanguageConfig, _JS_CACHE_BYPASS_SUFFIXES, _NamespaceExportFact, _StarExportFact, _SymbolAliasFact, _SymbolDeclarationFact, _SymbolExportFact, _SymbolImportFact, _SymbolResolutionFacts, _SymbolUseFact, _WORKSPACE_PACKAGE_CACHE # noqa: E402,F401 @@ -200,7 +210,7 @@ def _repoint_python_package_imports(paths, all_nodes, all_edges, root) -> None: (ambiguous -> leave dangling, as before). Files whose package root IS the scan root are skipped (ids already coincide).""" try: - root = Path(root).resolve() + root = _resolve_path(root) except OSError: root = Path(root) node_ids = {n.get("id") for n in all_nodes if isinstance(n, dict)} @@ -209,17 +219,17 @@ def _repoint_python_package_imports(paths, all_nodes, all_edges, root) -> None: if p.suffix.lower() not in (".py", ".pyi"): continue try: - rel = Path(p).resolve().relative_to(root) + rel = _resolve_path(p).relative_to(root) except (ValueError, OSError): continue parts = rel.parts if len(parts) < 2: continue # top-level file: scan-root-relative id already matches - d = Path(p).resolve().parent + d = _resolve_path(p).parent levels = 0 # Bounded by the number of dirs between the file and the scan root, so a # pathological `/__init__.py` chain can't loop forever. - while levels < len(parts) - 1 and (d / "__init__.py").is_file(): + while levels < len(parts) - 1 and _path_is_file(d / "__init__.py"): levels += 1 d = d.parent if levels == 0: @@ -373,7 +383,7 @@ def _import_python(node, source: bytes, file_nid: str, stem: str, edges: list, s # and popped before graph.json ships. if target_path is not None: try: - if target_path.is_file(): + if _path_is_file(target_path): edge["target_file"] = str(target_path) except OSError: pass @@ -1086,7 +1096,7 @@ def _extract_python_rationale(path: Path, result: dict) -> None: from tree_sitter import Language, Parser language = Language(tspython.language()) parser = Parser(language) - source = path.read_bytes() + source = _read_file_bytes(path) tree = parser.parse(source) root = tree.root_node except Exception: @@ -1242,7 +1252,7 @@ def _extract_js_rationale(path: Path, result: dict) -> None: Mutates result in-place by appending to result['nodes'] and result['edges']. """ try: - source_text = path.read_text(encoding="utf-8", errors="replace") + source_text = _read_file_text(path, encoding="utf-8", errors="replace") except Exception: return @@ -1352,7 +1362,7 @@ def _emit_rescued_import( ) node_id = _make_id(str(resolved)) stub_source_file = str(resolved) - if resolved is not None and resolved.is_file(): + if resolved is not None and _path_is_file(resolved): resolved_file = resolved else: # Check tsconfig.json path aliases (e.g. "$lib/" -> "src/lib/", @@ -1364,7 +1374,7 @@ def _emit_rescued_import( resolved_alias = _resolve_js_module_path(resolved_alias) node_id = _make_id(str(resolved_alias)) stub_source_file = str(resolved_alias) - if resolved_alias is not None and resolved_alias.is_file(): + if resolved_alias is not None and _path_is_file(resolved_alias): resolved_file = resolved_alias else: # Bare/scoped import (node_modules) - use last segment; @@ -1408,7 +1418,7 @@ def extract_svelte(path: Path) -> dict: result = _extract_generic(path, _JS_CONFIG) try: import re as _re - src = path.read_text(encoding="utf-8", errors="replace") + src = _read_file_text(path, encoding="utf-8", errors="replace") existing_ids = {n["id"] for n in result.get("nodes", [])} # Source file node ID must match the one _extract_generic creates: # _make_id(str(path)) - single arg, no stem prefix. Otherwise the source @@ -1470,7 +1480,7 @@ def extract_astro(path: Path) -> dict: result = _extract_generic(path, _JS_CONFIG) try: import re as _re - src = path.read_text(encoding="utf-8", errors="replace") + src = _read_file_text(path, encoding="utf-8", errors="replace") existing_ids = {n["id"] for n in result.get("nodes", [])} file_node_id = _make_id(str(path)) aliases = _load_tsconfig_aliases(path.parent) @@ -1531,7 +1541,7 @@ def extract_vue(path: Path) -> dict: ``import('…')`` dynamic imports the AST does not edge. """ try: - src = path.read_text(encoding="utf-8", errors="replace") + src = _read_file_text(path, encoding="utf-8", errors="replace") except OSError: return {"nodes": [], "edges": []} @@ -1576,7 +1586,7 @@ def _is_spock_file(path: Path, ts_result: dict) -> bool: import re as _re _SPOCK_FEATURE_RE = _re.compile(r"""^\s*def\s+[\"']""", _re.MULTILINE) try: - return bool(_SPOCK_FEATURE_RE.search(path.read_text(errors="replace"))) + return bool(_SPOCK_FEATURE_RE.search(_read_file_text(path, errors="replace"))) except OSError: return False @@ -1587,7 +1597,7 @@ def _extract_spock_fallback(path: Path, ts_result: dict) -> dict: (which survive reliably) with class and feature-method nodes extracted via regex. """ import re as _re - source = path.read_text(errors="replace") + source = _read_file_text(path, errors="replace") str_path = str(path) stem = _file_stem(path) @@ -3085,7 +3095,7 @@ def extract_lazarus_package(path: Path) -> dict: """ try: import xml.etree.ElementTree as ET - src = path.read_bytes() + src = _read_file_bytes(path) except OSError as e: return {"nodes": [], "edges": [], "error": str(e)} @@ -3192,7 +3202,7 @@ def extract_slnx(path: Path) -> dict: import xml.etree.ElementTree as ET try: - src = path.read_bytes() + src = _read_file_bytes(path) except OSError: return {"nodes": [], "edges": [], "error": f"cannot read {path}"} @@ -3222,7 +3232,7 @@ def extract_slnx(path: Path) -> dict: def _resolve(proj_path: str) -> str: proj_path = proj_path.replace("\\", "/") try: - return str((path.parent / proj_path).resolve()) + return str(_resolve_path(path.parent / proj_path)) except Exception: return proj_path @@ -3271,7 +3281,7 @@ def extract_csproj(path: Path) -> dict: import xml.etree.ElementTree as ET try: - src = path.read_bytes() + src = _read_file_bytes(path) except OSError: return {"nodes": [], "edges": [], "error": f"cannot read {path}"} @@ -3351,7 +3361,7 @@ def find_all(tag: str): continue ref_path_norm = ref_path.replace("\\", "/") try: - abs_ref = str((path.parent / ref_path_norm).resolve()) + abs_ref = str(_resolve_path(path.parent / ref_path_norm)) except Exception: abs_ref = ref_path_norm proj_nid = _make_id(abs_ref) @@ -3486,10 +3496,10 @@ def _xaml_binding_refs(value: str) -> tuple[str | None, str | None]: def _xaml_codebehind_path(path: Path) -> Path | None: expected = path.with_suffix(path.suffix + ".cs") - if expected.exists(): + if _path_exists(expected): return expected try: - for sibling in path.parent.iterdir(): + for sibling in _iterdir_path(path.parent): if sibling.name.casefold() == expected.name.casefold(): return sibling except OSError: @@ -3532,7 +3542,7 @@ def _xaml_codebehind_symbols( # parameter list on method nodes, so we read it from the code-behind source # at the method's recorded line. try: - cb_lines = codebehind.read_text(encoding="utf-8", errors="replace").splitlines() + cb_lines = _read_file_text(codebehind, encoding="utf-8", errors="replace").splitlines() except OSError: cb_lines = [] @@ -3630,16 +3640,16 @@ def _xaml_project_root(path: Path) -> Path: root = path.parent for directory in (path.parent, *path.parent.parents): try: - if any(child.suffix in project_markers for child in directory.iterdir()): + if any(child.suffix in project_markers for child in _iterdir_path(directory)): root = directory break except OSError: continue if _XAML_ACTIVE_EXTRACT_ROOT is None: return root - boundary = _XAML_ACTIVE_EXTRACT_ROOT.resolve() + boundary = _resolve_path(_XAML_ACTIVE_EXTRACT_ROOT) try: - root.resolve().relative_to(boundary) + _resolve_path(root).relative_to(boundary) return root except ValueError: return boundary @@ -3648,7 +3658,7 @@ def _xaml_project_root(path: Path) -> Path: def _xaml_csharp_class_nodes(path: Path) -> dict[str, list[dict]]: from graphify.detect import _is_ignored, _is_noise_dir, _load_graphifyignore root = _xaml_project_root(path) - cache_key = str(root.resolve()) if _XAML_ACTIVE_EXTRACT_ROOT is not None else None + cache_key = str(_resolve_path(root)) if _XAML_ACTIVE_EXTRACT_ROOT is not None else None if cache_key and cache_key in _XAML_CSHARP_CLASS_CACHE: return _XAML_CSHARP_CLASS_CACHE[cache_key] classes: dict[str, list[dict]] = {} @@ -3662,12 +3672,11 @@ def _xaml_csharp_class_nodes(path: Path) -> dict[str, list[dict]]: # scanned millions of paths and effectively hung. A real .NET project sits # well under the cap; a runaway root is bounded to a fast, partial scan # instead of hanging. - import os as _os _DIR_CAP = 20000 cs_files: list[Path] = [] visited = 0 try: - for dirpath, dirnames, filenames in _os.walk(root): + for dirpath, dirnames, filenames in _walk_path(root): dirnames[:] = [ d for d in dirnames if not d.startswith(".") and not _is_noise_dir(d) ] @@ -3718,7 +3727,7 @@ def _xaml_communitytoolkit_members(vm_node: dict) -> tuple[dict[str, dict], list try: # errors="replace" so a non-UTF8 code-behind can't raise UnicodeDecodeError # and abort the whole extract_xaml (matches every other reader here). - lines = Path(source_file).read_text(encoding="utf-8", errors="replace").splitlines() + lines = _read_file_text(Path(source_file), encoding="utf-8", errors="replace").splitlines() except OSError: return {}, [] @@ -3782,7 +3791,7 @@ def extract_xaml(path: Path) -> dict: import xml.etree.ElementTree as ET try: - src = path.read_bytes() + src = _read_file_bytes(path) except OSError: return {"nodes": [], "edges": [], "error": f"cannot read {path}"} @@ -4196,7 +4205,7 @@ def _is_objc_header(path: Path) -> bool: extract_objc while leaving every C/C++ header on its existing extractor. """ try: - head = path.read_bytes()[:256 * 1024] + head = _read_file_bytes(path, limit=256 * 1024) except OSError: return False return any(marker in head for marker in _OBJC_HEADER_MARKERS) @@ -4239,7 +4248,7 @@ def _is_cpp_header(path: Path) -> bool: here and keeps its existing extract_c routing. """ try: - head = path.read_bytes()[:256 * 1024] + head = _read_file_bytes(path, limit=256 * 1024) except OSError: return False return any(marker in head for marker in _CPP_HEADER_MARKERS) @@ -4294,7 +4303,7 @@ def _get_extractor(path: Path) -> Any | None: def _safe_extract_with_xaml_root(extractor, path: Path, root: Path) -> dict: global _XAML_ACTIVE_EXTRACT_ROOT previous_root = _XAML_ACTIVE_EXTRACT_ROOT - _XAML_ACTIVE_EXTRACT_ROOT = root.resolve() + _XAML_ACTIVE_EXTRACT_ROOT = _resolve_path(root) try: return _safe_extract(extractor, path) finally: @@ -4563,7 +4572,7 @@ def extract( root = anchor_root elif cache_root is not None: root = cache_root - root = root.resolve() + root = _resolve_path(root) # #1774: the cache is an OUTPUT, so when no explicit cache_root is given it is # written under the current working directory — never `root` (the inferred @@ -4571,7 +4580,7 @@ def extract( # read-only or foreign corpus. `root` still anchors the content-hash keys, # node ids, symbol resolution, and the XAML project-scan boundary; only the # cache directory's location diverges from it. - cache_location = (cache_root if cache_root is not None else Path(".")).resolve() + cache_location = _resolve_path(cache_root if cache_root is not None else Path(".")) total = len(paths) # Phase 1: separate cached hits from uncached work @@ -4763,7 +4772,7 @@ def _portable_out_of_root_sf(p: Path) -> str: _remap_seen: set[Path] = set() for _p in paths: try: - _remap_seen.add(_p.resolve()) + _remap_seen.add(_resolve_path(_p)) except (OSError, RuntimeError): pass for _e in all_edges: @@ -4772,7 +4781,7 @@ def _portable_out_of_root_sf(p: Path) -> str: continue _raw_tp = Path(_tf) try: - _tp = _raw_tp.resolve() + _tp = _resolve_path(_raw_tp) except (OSError, RuntimeError): continue if _tp in _remap_seen: @@ -4798,7 +4807,7 @@ def _portable_out_of_root_sf(p: Path) -> str: # target that does not actually exist on disk stays dangling, # exactly as before. try: - if _tp.is_file(): + if _path_is_file(_tp): ext_new_id = _make_id("ext", _portable_out_of_root_sf(_tp)) id_remap[_make_id(str(_tp))] = ext_new_id if _raw_tp != _tp: @@ -4818,7 +4827,7 @@ def _portable_out_of_root_sf(p: Path) -> str: pass continue try: - if not _tp.is_file(): + if not _path_is_file(_tp): # Speculatively-resolved target that doesn't exist (e.g. an # import of a not-yet-created sibling): keep its raw id # dangling, exactly as before, so no false canonical edge is @@ -4843,7 +4852,7 @@ def _portable_out_of_root_sf(p: Path) -> str: rel = path.relative_to(root) except ValueError: try: - rel = path.resolve().relative_to(root) + rel = _resolve_path(path).relative_to(root) except ValueError: continue new_id = _file_node_id(rel) @@ -4852,14 +4861,14 @@ def _portable_out_of_root_sf(p: Path) -> str: # Also register the absolute-resolved form of the file-level id so # alias/workspace import targets (resolved via .resolve()) remap to # canonical instead of orphaning (#1529). - old_id_abs = _make_id(str(path.resolve())) + old_id_abs = _make_id(str(_resolve_path(path))) if old_id_abs != new_id: id_remap[old_id_abs] = new_id old_prefs: list[tuple[str, str]] = [] old_pref = _file_node_id(path) if old_pref != new_id: old_prefs.append((old_pref, new_id)) - old_pref_abs = _file_node_id(path.resolve()) + old_pref_abs = _file_node_id(_resolve_path(path)) if old_pref_abs != new_id and old_pref_abs != old_pref: old_prefs.append((old_pref_abs, new_id)) # Bash entrypoint node ids append "__entry" to the file-level id @@ -4878,10 +4887,10 @@ def _portable_out_of_root_sf(p: Path) -> str: if _entry_old != _entry_new: id_remap.setdefault(_entry_old, _entry_new) if old_prefs: - prefix_remap[path.resolve()] = old_prefs + prefix_remap[_resolve_path(path)] = old_prefs # Absolute form first: it is the longest, so prefix decomposition can # try forms in order without a shorter form shadowing it. - stem_forms[path.resolve()] = ( + stem_forms[_resolve_path(path)] = ( new_id, [old_pref_abs, old_pref, new_id] ) if id_remap: @@ -4919,7 +4928,7 @@ def _portable_out_of_root_sf(p: Path) -> str: if n.get("type") == "package": continue try: - entry = prefix_remap.get(Path(sf).resolve()) + entry = prefix_remap.get(_resolve_path(sf)) except Exception: continue if entry is None: @@ -5018,7 +5027,7 @@ def _edge_key(edge: dict) -> str: def _decompose(target: str, tf: str) -> "tuple[str, str] | None": try: - forms = stem_forms.get(Path(tf).resolve()) + forms = stem_forms.get(_resolve_path(tf)) except (OSError, RuntimeError): return None if not forms: @@ -5520,7 +5529,7 @@ def _sf_entry(sf: str, sf_path: Path) -> tuple[str, str, tuple[str, ...]]: canonical_id = _file_node_id(rel) new_sf = rel.as_posix() try: - sf_resolved = sf_path.resolve() + sf_resolved = _resolve_path(sf_path) except (OSError, RuntimeError): sf_resolved = sf_path # Learn the STEM (extension-dropped) forms too: symbol producers mint @@ -5641,7 +5650,7 @@ def _canon(nid: str) -> str: def collect_files(target: Path, *, follow_symlinks: bool = False, root: Path | None = None) -> list[Path]: containment_root = root if root is not None else target from graphify.detect import _resolves_under_root - if target.is_file(): + if _path_is_file(target): return [target] if _resolves_under_root(target, containment_root) else [] _EXTENSIONS = set(_DISPATCH.keys()) from graphify.detect import _is_ignored, _is_noise_dir, _load_graphifyignore @@ -5664,7 +5673,7 @@ def _ignored(p: Path) -> bool: # conservatism as detect's scan walk). has_negation = any(pat.startswith("!") for _, pat in patterns) results: list[Path] = [] - for dirpath, dirnames, filenames in os.walk(target): + for dirpath, dirnames, filenames in _walk_path(target): dp = Path(dirpath) dirnames[:] = [ d for d in dirnames @@ -5679,10 +5688,10 @@ def _ignored(p: Path) -> bool: return sorted(results) # Walk with symlink following + cycle detection results = [] - for dirpath, dirnames, filenames in os.walk(target, followlinks=True): - if os.path.islink(dirpath): - real = os.path.realpath(dirpath) - parent_real = os.path.realpath(os.path.dirname(dirpath)) + for dirpath, dirnames, filenames in _walk_path(target, followlinks=True): + if _path_is_symlink(dirpath): + real = str(_resolve_path(dirpath)) + parent_real = str(_resolve_path(Path(dirpath).parent)) if parent_real == real or parent_real.startswith(real + os.sep): dirnames.clear() continue @@ -5690,7 +5699,7 @@ def _ignored(p: Path) -> bool: dirnames[:] = [ d for d in dirnames if not _is_noise_dir(d, dp) # pass parent so "env"/"*_env" is marker-gated (#2058) - and (not (dp / d).is_symlink() or _resolves_under_root(dp / d, containment_root)) + and (not _path_is_symlink(dp / d) or _resolves_under_root(dp / d, containment_root)) ] for fname in filenames: p = dp / fname diff --git a/graphify/extractors/apex.py b/graphify/extractors/apex.py index 928923a64..578ea4c49 100644 --- a/graphify/extractors/apex.py +++ b/graphify/extractors/apex.py @@ -3,6 +3,7 @@ from pathlib import Path +from graphify.paths import read_text as _read_file_text from graphify.extractors.base import _file_stem, _make_id @@ -11,7 +12,7 @@ def extract_apex(path: Path) -> dict: Apex .cls and .trigger files using regex (no tree-sitter grammar on PyPI).""" import re as _re try: - source = path.read_text(encoding="utf-8", errors="replace") + source = _read_file_text(path, encoding="utf-8", errors="replace") except OSError: return {"nodes": [], "edges": []} diff --git a/graphify/extractors/bash.py b/graphify/extractors/bash.py index 984676b2a..0923329d0 100644 --- a/graphify/extractors/bash.py +++ b/graphify/extractors/bash.py @@ -6,6 +6,12 @@ from typing import Any from graphify.extractors.base import _file_stem, _make_id, _read_text +from graphify.paths import ( + path_exists as _path_exists, + path_is_file as _path_is_file, + read_bytes as _read_file_bytes, + resolve_path as _resolve_path, +) # Leading `${VAR}` / `$VAR` expansion segment(s) of a `source` path argument. The @@ -79,7 +85,7 @@ def extract_bash(path: Path) -> dict: try: language = Language(tsbash.language()) parser = Parser(language) - source = path.read_bytes() + source = _read_file_bytes(path) tree = parser.parse(source) root = tree.root_node except Exception as e: @@ -266,12 +272,12 @@ def walk(node, parent_nid: str) -> None: raw = _read_text(args[0], source).strip().strip("'\"") line = node.start_point[0] + 1 if raw.startswith((".", "/")): - resolved = (path.parent / raw).resolve() + resolved = _resolve_path(path.parent / raw) # Only emit the edge if the target actually exists on # disk — prevents graph pollution from crafted paths # like `source ../../etc/passwd` that traverse outside # the project tree (B-1). - if resolved.exists(): + if _path_exists(resolved): tgt_nid = _make_id(str(resolved)) add_edge(file_nid, tgt_nid, "imports_from", line, context="import", @@ -306,8 +312,8 @@ def walk(node, parent_nid: str) -> None: var_name = var_match.group(1) or var_match.group(2) if var_name in var_bases: base = var_bases[var_name] - resolved = (base / suffix).resolve() - if resolved.is_file(): + resolved = _resolve_path(base / suffix) + if _path_is_file(resolved): add_edge(file_nid, _make_id(str(resolved)), "imports_from", line, confidence="INFERRED", context="import", @@ -335,8 +341,8 @@ def walk(node, parent_nid: str) -> None: if raw: try: candidate = path.parent / raw - if candidate.is_file(): - sibling = candidate.resolve() + if _path_is_file(candidate): + sibling = _resolve_path(candidate) except OSError: sibling = None if sibling is not None: @@ -362,12 +368,12 @@ def walk(node, parent_nid: str) -> None: if cmd in _BASH_SCRIPT_RUNNERS and args: raw = literal(args[0]) if raw and raw.endswith(".sh"): - resolved = (path.parent / raw).resolve() - if resolved.is_file(): + resolved = _resolve_path(path.parent / raw) + if _path_is_file(resolved): target_path = resolved if not path.is_absolute(): try: - target_path = resolved.relative_to(Path.cwd().resolve()) + target_path = resolved.relative_to(_resolve_path(Path.cwd())) except ValueError: pass caller_nid = entry_nid if parent_nid == file_nid else parent_nid diff --git a/graphify/extractors/blade.py b/graphify/extractors/blade.py index 63e4fac9a..92b466ae9 100644 --- a/graphify/extractors/blade.py +++ b/graphify/extractors/blade.py @@ -2,6 +2,7 @@ from __future__ import annotations from pathlib import Path +from graphify.paths import read_text as _read_file_text from graphify.extractors.base import _make_id @@ -10,7 +11,7 @@ def extract_blade(path: Path) -> dict: """Extract @include, components, and wire:click bindings from Blade templates.""" import re try: - src = path.read_text(encoding="utf-8", errors="replace") + src = _read_file_text(path, encoding="utf-8", errors="replace") except OSError: return {"error": f"cannot read {path}"} diff --git a/graphify/extractors/dart.py b/graphify/extractors/dart.py index acbe19583..324d6c250 100644 --- a/graphify/extractors/dart.py +++ b/graphify/extractors/dart.py @@ -4,13 +4,19 @@ import re from pathlib import Path + from graphify.extractors.base import _file_stem, _make_id +from graphify.paths import ( + path_exists as _path_exists, + read_text as _read_file_text, + resolve_path as _resolve_path, +) def extract_dart(path: Path) -> dict: """Extract classes, mixins, functions, imports, generic calls, and annotations from a .dart file using regex.""" try: - src = path.read_text(encoding="utf-8", errors="replace") + src = _read_file_text(path, encoding="utf-8", errors="replace") except OSError: return {"error": f"cannot read {path}"} @@ -40,8 +46,8 @@ def _comment_replace(match: re.Match) -> str: parent_ref = part_of_match.group(1) if parent_ref.endswith(".dart"): try: - parent_path = (path.parent / parent_ref).resolve() - if parent_path.exists(): + parent_path = _resolve_path(path.parent / parent_ref) + if _path_exists(parent_path): stem = _file_stem(parent_path) file_nid = _make_id(str(parent_path)) is_part = True diff --git a/graphify/extractors/dm.py b/graphify/extractors/dm.py index 7961022cb..24571517c 100644 --- a/graphify/extractors/dm.py +++ b/graphify/extractors/dm.py @@ -5,7 +5,15 @@ from pathlib import Path from typing import Any + from graphify.extractors.base import _file_stem, _make_id, _read_text +from graphify.paths import ( + path_exists as _path_exists, + path_stat as _path_stat, + read_bytes as _read_file_bytes, + read_text as _read_file_text, + resolve_path as _resolve_path, +) def extract_dm(path: Path) -> dict: @@ -18,7 +26,7 @@ def extract_dm(path: Path) -> dict: try: language = Language(tsdm.language()) parser = Parser(language) - source = path.read_bytes() + source = _read_file_bytes(path) tree = parser.parse(source) root = tree.root_node except Exception as e: @@ -87,18 +95,18 @@ def walk(node, parent_type_path: "str | None" = None, raw = _read_include_path(file_node) if raw: norm = re.sub(r"^\./", "", raw.replace("\\", "/")) - resolved = (path.parent / norm).resolve() + resolved = _resolve_path(path.parent / norm) edge: dict = { "source": file_nid, - "target": _make_id(str(resolved)) if resolved.exists() else _make_id(norm), - "relation": "imports_from" if resolved.exists() else "imports", + "target": _make_id(str(resolved)) if _path_exists(resolved) else _make_id(norm), + "relation": "imports_from" if _path_exists(resolved) else "imports", "context": "import", "confidence": "EXTRACTED", "source_file": str_path, "source_location": f"L{line}", "weight": 1.0, } - if not resolved.exists(): + if not _path_exists(resolved): edge["external"] = True edges.append(edge) return @@ -275,7 +283,7 @@ def _read_dmi_description(data: bytes) -> str: def extract_dmi(path: Path) -> dict: """Extract icon state names from a .dmi (BYOND PNG icon sheet).""" try: - data = path.read_bytes() + data = _read_file_bytes(path) except Exception as e: return {"nodes": [], "edges": [], "error": str(e)} @@ -364,9 +372,9 @@ def _dmm_type_path(entry: str) -> str: def extract_dmm(path: Path) -> dict: """Extract type-path references from a .dmm map file's tile dictionary.""" try: - if path.stat().st_size > 50 * 1024 * 1024: + if _path_stat(path).st_size > 50 * 1024 * 1024: return {"nodes": [], "edges": [], "error": "file too large (>50 MB)"} - text = path.read_text(encoding="utf-8", errors="replace") + text = _read_file_text(path, encoding="utf-8", errors="replace") except Exception as e: return {"nodes": [], "edges": [], "error": str(e)} @@ -436,7 +444,7 @@ def extract_dmm(path: Path) -> dict: def extract_dmf(path: Path) -> dict: """Extract windows and controls from a .dmf interface file.""" try: - text = path.read_text(encoding="utf-8", errors="replace") + text = _read_file_text(path, encoding="utf-8", errors="replace") except Exception as e: return {"nodes": [], "edges": [], "error": str(e)} diff --git a/graphify/extractors/elixir.py b/graphify/extractors/elixir.py index 0f1588ddb..9bd2d7ee0 100644 --- a/graphify/extractors/elixir.py +++ b/graphify/extractors/elixir.py @@ -2,6 +2,7 @@ from __future__ import annotations from pathlib import Path +from graphify.paths import read_bytes as _read_file_bytes from typing import Any from graphify.extractors.base import _LANGUAGE_BUILTIN_GLOBALS, _file_stem, _make_id @@ -18,7 +19,7 @@ def extract_elixir(path: Path) -> dict: try: language = Language(tselixir.language()) parser = Parser(language) - source = path.read_bytes() + source = _read_file_bytes(path) tree = parser.parse(source) root = tree.root_node except Exception as e: diff --git a/graphify/extractors/engine.py b/graphify/extractors/engine.py index ab6c1657c..40d9f3b98 100644 --- a/graphify/extractors/engine.py +++ b/graphify/extractors/engine.py @@ -9,6 +9,7 @@ from graphify.extractors.resolution import _resolve_js_import_target from graphify.security import sanitize_metadata from pathlib import Path +from graphify.paths import read_bytes as _read_file_bytes def _csharp_namespace_id(dotted_name: str) -> str: @@ -2212,7 +2213,7 @@ def _extract_generic( try: parser = Parser(language) - source = path.read_bytes() if source_override is None else source_override + source = _read_file_bytes(path) if source_override is None else source_override tree = parser.parse(source) root = tree.root_node except Exception as e: diff --git a/graphify/extractors/fortran.py b/graphify/extractors/fortran.py index 58ffedfc8..6d03e1d7f 100644 --- a/graphify/extractors/fortran.py +++ b/graphify/extractors/fortran.py @@ -1,9 +1,13 @@ """Fortran extractor. Moved verbatim from graphify/extract.py.""" from __future__ import annotations - from pathlib import Path + from graphify.extractors.base import _file_stem, _make_id, _read_text +from graphify.paths import ( + read_bytes as _read_file_bytes, + resolve_path as _resolve_path, +) _FORTRAN_CPP_EXTS = {".F", ".F90", ".F95", ".F03", ".F08"} @@ -25,13 +29,21 @@ def _cpp_preprocess(path: Path) -> bytes: import shutil import subprocess if not shutil.which("cpp"): - return path.read_bytes() + return _read_file_bytes(path) try: # Pass an absolute path so a corpus file named like "-I/etc/x.F90" cannot # be parsed by cpp as an option (cpp does not accept a "--" end-of-options # terminator). An absolute path always begins with "/". result = subprocess.run( - ["cpp", "-w", "-P", "-nostdinc", "-I", "/dev/null", str(path.resolve())], + [ + "cpp", + "-w", + "-P", + "-nostdinc", + "-I", + "/dev/null", + str(_resolve_path(path)), + ], capture_output=True, timeout=30, ) @@ -39,7 +51,7 @@ def _cpp_preprocess(path: Path) -> bytes: return result.stdout except Exception: pass - return path.read_bytes() + return _read_file_bytes(path) def extract_fortran(path: Path) -> dict: """Extract programs, modules, subroutines, functions, use statements, and calls from Fortran files. @@ -56,7 +68,11 @@ def extract_fortran(path: Path) -> dict: try: language = Language(tsfortran.language()) parser = Parser(language) - source = _cpp_preprocess(path) if path.suffix in _FORTRAN_CPP_EXTS else path.read_bytes() + source = ( + _cpp_preprocess(path) + if path.suffix in _FORTRAN_CPP_EXTS + else _read_file_bytes(path) + ) tree = parser.parse(source) root = tree.root_node except Exception as e: diff --git a/graphify/extractors/go.py b/graphify/extractors/go.py index a0db9a693..205716625 100644 --- a/graphify/extractors/go.py +++ b/graphify/extractors/go.py @@ -3,6 +3,7 @@ from pathlib import Path +from graphify.paths import read_bytes as _read_file_bytes from graphify.extractors.base import _LANGUAGE_BUILTIN_GLOBALS, _file_stem, _make_id, _read_text @@ -61,7 +62,7 @@ def extract_go(path: Path) -> dict: try: language = Language(tsgo.language()) parser = Parser(language) - source = path.read_bytes() + source = _read_file_bytes(path) tree = parser.parse(source) root = tree.root_node except Exception as e: diff --git a/graphify/extractors/json_config.py b/graphify/extractors/json_config.py index 6a9b641a9..4c8e7d956 100644 --- a/graphify/extractors/json_config.py +++ b/graphify/extractors/json_config.py @@ -3,6 +3,7 @@ from pathlib import Path +from graphify.paths import read_bytes as _read_file_bytes from graphify.extractors.base import _file_stem, _make_id, _read_text from graphify.ids import normalize_id @@ -68,8 +69,7 @@ def extract_json(path: Path) -> dict: # Bounded read instead of stat()+read() to eliminate TOCTOU (J-1): # read one byte beyond the limit so we can detect oversized files even # if the file grows between stat and read. - with path.open("rb") as _f: - source = _f.read(_JSON_MAX_BYTES + 1) + source = _read_file_bytes(path, limit=_JSON_MAX_BYTES + 1) if len(source) > _JSON_MAX_BYTES: return {"nodes": [], "edges": [], "error": "json file too large to index"} language = Language(tsjson.language()) diff --git a/graphify/extractors/julia.py b/graphify/extractors/julia.py index 95846c7d9..a9ae03794 100644 --- a/graphify/extractors/julia.py +++ b/graphify/extractors/julia.py @@ -4,6 +4,7 @@ from graphify.extractors.base import _file_stem, _make_id, _read_text from graphify.extractors.engine import _semantic_reference_edge from pathlib import Path +from graphify.paths import read_bytes as _read_file_bytes def extract_julia(path: Path) -> dict: @@ -17,7 +18,7 @@ def extract_julia(path: Path) -> dict: try: language = Language(tsjulia.language()) parser = Parser(language) - source = path.read_bytes() + source = _read_file_bytes(path) tree = parser.parse(source) root = tree.root_node except Exception as e: diff --git a/graphify/extractors/markdown.py b/graphify/extractors/markdown.py index e1b24409a..d28e92658 100644 --- a/graphify/extractors/markdown.py +++ b/graphify/extractors/markdown.py @@ -5,6 +5,7 @@ import os from pathlib import Path +from graphify.paths import read_text as _read_file_text, path_is_file as _path_is_file from graphify.extractors.base import _file_stem, _make_id @@ -77,7 +78,7 @@ def extract_markdown(path: Path) -> dict: No tree-sitter dependency — pure line-by-line parsing. """ try: - source = path.read_text(encoding="utf-8", errors="replace") + source = _read_file_text(path, encoding="utf-8", errors="replace") except Exception as e: return {"nodes": [], "edges": [], "error": str(e)} @@ -134,7 +135,7 @@ def add_link(raw: str, line: int) -> None: # and popped before graph.json ships. target_file = None try: - if resolved.is_file(): + if _path_is_file(resolved): target_file = str(resolved) except OSError: pass diff --git a/graphify/extractors/objc.py b/graphify/extractors/objc.py index 9f978a50f..3884e3dfd 100644 --- a/graphify/extractors/objc.py +++ b/graphify/extractors/objc.py @@ -5,6 +5,7 @@ from graphify.extractors.engine import _cpp_declarator_name, _semantic_reference_edge from graphify.extractors.resolution import _resolve_c_include_path from pathlib import Path +from graphify.paths import read_bytes as _read_file_bytes from typing import Any @@ -51,7 +52,7 @@ def extract_objc(path: Path) -> dict: try: language = Language(tsobjc.language()) parser = Parser(language) - source = path.read_bytes() + source = _read_file_bytes(path) # tree-sitter-objc cannot expand these argument-less annotation macros (no # trailing ';'), and their presence before @interface makes the parser fail to # emit a class_interface node (#1475). Blank them to equal-length spaces so byte diff --git a/graphify/extractors/pascal.py b/graphify/extractors/pascal.py index 398edb22e..c8e8e78ab 100644 --- a/graphify/extractors/pascal.py +++ b/graphify/extractors/pascal.py @@ -5,6 +5,7 @@ from graphify.extractors.base import _file_stem, _make_id from graphify.extractors.resolution import _pascal_resolve_class, _pascal_resolve_unit from pathlib import Path +from graphify.paths import read_bytes as _read_file_bytes, read_text as _read_file_text from typing import Any, Callable @@ -233,7 +234,7 @@ def _extract_pascal_regex(path: Path) -> dict: is unavailable. Produces the same node/edge schema as the tree-sitter pass. """ try: - raw = path.read_text(encoding="utf-8", errors="replace") + raw = _read_file_text(path, encoding="utf-8", errors="replace") except Exception as exc: return {"nodes": [], "edges": [], "error": str(exc)} @@ -457,7 +458,7 @@ def extract_pascal(path: Path) -> dict: try: language = Language(tspascal.language()) parser = Parser(language) - source = path.read_bytes() + source = _read_file_bytes(path) tree = parser.parse(source) root = tree.root_node except Exception: diff --git a/graphify/extractors/pascal_forms.py b/graphify/extractors/pascal_forms.py index 1f81ad875..9cd09f4ff 100644 --- a/graphify/extractors/pascal_forms.py +++ b/graphify/extractors/pascal_forms.py @@ -3,6 +3,7 @@ from pathlib import Path +from graphify.paths import read_bytes as _read_file_bytes, read_text as _read_file_text from typing import Any from graphify.extractors.base import _file_stem, _make_id @@ -30,7 +31,7 @@ def extract_lazarus_form(path: Path) -> dict: - component --references--> event handler (context: "event") """ try: - text = path.read_text(encoding="utf-8", errors="replace") + text = _read_file_text(path, encoding="utf-8", errors="replace") except Exception as e: return {"nodes": [], "edges": [], "error": str(e)} @@ -116,7 +117,7 @@ def extract_delphi_form(path: Path) -> dict: (`contains`) and event handler references (`references`, context "event"). """ try: - raw = path.read_bytes() + raw = _read_file_bytes(path) except Exception as e: return {"nodes": [], "edges": [], "error": str(e)} diff --git a/graphify/extractors/powershell.py b/graphify/extractors/powershell.py index 9f8526c7b..2345a9d9a 100644 --- a/graphify/extractors/powershell.py +++ b/graphify/extractors/powershell.py @@ -4,6 +4,7 @@ import re from pathlib import Path +from graphify.paths import read_bytes as _read_file_bytes from typing import Any from graphify.extractors.base import _file_stem, _make_id, _read_text @@ -19,7 +20,7 @@ def extract_powershell(path: Path) -> dict: try: language = Language(tsps.language()) parser = Parser(language) - source = path.read_bytes() + source = _read_file_bytes(path) tree = parser.parse(source) root = tree.root_node except Exception as e: @@ -370,7 +371,7 @@ def extract_powershell_manifest(path: Path) -> dict: try: language = Language(tsps.language()) parser = Parser(language) - source = path.read_bytes() + source = _read_file_bytes(path) tree = parser.parse(source) root = tree.root_node except Exception as e: diff --git a/graphify/extractors/razor.py b/graphify/extractors/razor.py index 09dc54a6b..362031e44 100644 --- a/graphify/extractors/razor.py +++ b/graphify/extractors/razor.py @@ -3,6 +3,7 @@ import re from pathlib import Path +from graphify.paths import read_text as _read_file_text from graphify.extractors.base import _file_stem, _make_id @@ -10,7 +11,7 @@ def extract_razor(path: Path) -> dict: """Extract directives, component refs, and @code methods from .razor/.cshtml.""" try: - src = path.read_text(encoding="utf-8", errors="replace") + src = _read_file_text(path, encoding="utf-8", errors="replace") except OSError: return {"nodes": [], "edges": [], "error": f"cannot read {path}"} diff --git a/graphify/extractors/resolution.py b/graphify/extractors/resolution.py index 097c32b6a..f3569c2aa 100644 --- a/graphify/extractors/resolution.py +++ b/graphify/extractors/resolution.py @@ -1,8 +1,19 @@ """resolution — moved verbatim from graphify/extract.py.""" from __future__ import annotations -from typing import Any, Callable from pathlib import Path +from typing import Any, Callable + +from graphify.paths import ( + glob_paths as _glob_paths, + path_exists as _path_exists, + path_is_dir as _path_is_dir, + path_is_file as _path_is_file, + path_stat as _path_stat, + read_bytes as _read_file_bytes, + read_text as _read_file_text, + resolve_path as _resolve_path, +) from graphify.extractors.models import LanguageConfig, _JS_CACHE_BYPASS_SUFFIXES, _NamespaceExportFact, _StarExportFact, _SymbolAliasFact, _SymbolDeclarationFact, _SymbolExportFact, _SymbolImportFact, _SymbolResolutionFacts, _SymbolUseFact, _WORKSPACE_PACKAGE_CACHE # noqa: E402,F401 from graphify.extractors.base import ( # noqa: F401 _LANGUAGE_BUILTIN_GLOBALS, @@ -31,31 +42,31 @@ def _resolve_js_import_path(candidate: Path) -> Path: """Resolve a JS/TS/Svelte import target to a local file when it exists.""" candidate = Path(os.path.normpath(candidate)) - if candidate.is_file(): + if _path_is_file(candidate): return candidate # TS ESM convention: imports often spell .js/.jsx while source is .ts/.tsx. if candidate.suffix == ".js": ts_candidate = candidate.with_suffix(".ts") - if ts_candidate.is_file(): + if _path_is_file(ts_candidate): return ts_candidate elif candidate.suffix == ".jsx": tsx_candidate = candidate.with_suffix(".tsx") - if tsx_candidate.is_file(): + if _path_is_file(tsx_candidate): return tsx_candidate # Append extensions to the full filename, which covers extensionless imports, # multi-dot helpers, and Svelte 5 rune files like Foo.svelte.ts. for ext in _JS_RESOLVE_EXTS: with_ext = candidate.parent / f"{candidate.name}{ext}" - if with_ext.is_file(): + if _path_is_file(with_ext): return with_ext # Only fall back to directory indexes after file candidates lose. - if candidate.is_dir(): + if _path_is_dir(candidate): for index_name in _JS_INDEX_FILES: index_candidate = candidate / index_name - if index_candidate.is_file(): + if _path_is_file(index_candidate): return index_candidate return candidate @@ -98,7 +109,7 @@ def _read_tsconfig_aliases(tsconfig: Path, base_dir: Path, seen: set) -> dict[st return {} seen.add(str(tsconfig)) try: - raw = tsconfig.read_text(encoding="utf-8") + raw = _read_file_text(tsconfig, encoding="utf-8") except Exception as e: print(f" warning: could not read {tsconfig} ({type(e).__name__}: {e})", file=sys.stderr, flush=True) return {} @@ -132,10 +143,10 @@ def _read_tsconfig_aliases(tsconfig: Path, base_dir: Path, seen: set) -> dict[st # Skip scoped npm package configs (e.g. @tsconfig/svelte) — not on disk. if not ext or ext.startswith("@"): continue - extended_path = (base_dir / ext).resolve() + extended_path = _resolve_path(base_dir / ext) if not extended_path.suffix: extended_path = extended_path.with_suffix(".json") - if extended_path.exists(): + if _path_exists(extended_path): aliases.update(_read_tsconfig_aliases(extended_path, extended_path.parent, seen)) # tsconfig `paths` are resolved relative to `baseUrl` (itself relative to @@ -175,7 +186,7 @@ def _read_json_config(path: Path) -> "dict | None": baseUrl" instead of raising. """ try: - raw = path.read_text(encoding="utf-8", errors="replace") + raw = _read_file_text(path, encoding="utf-8", errors="replace") except OSError: return None for candidate in (raw, _strip_jsonc(raw)): @@ -195,11 +206,11 @@ def _find_js_config(start_dir: Path) -> "tuple[Path, Path] | None": tsconfig.json wins when both sit in one directory, matching tsc and editors, which consult jsconfig.json only when there is no tsconfig.json. """ - current = start_dir.resolve() + current = _resolve_path(start_dir) for candidate in [current, *current.parents]: for name in ("tsconfig.json", "jsconfig.json"): config = candidate / name - if config.exists(): + if _path_exists(config): return config, candidate return None @@ -299,7 +310,7 @@ def _resolve_tsconfig_alias(raw: str, aliases: dict[str, list[str]], if base_url is not None: candidate = Path(os.path.normpath(base_url / raw)) resolved = _resolve_js_import_path(candidate) - if resolved.is_file(): + if _path_is_file(resolved): return resolved return None @@ -315,21 +326,21 @@ def _resolve_tsconfig_alias(raw: str, aliases: dict[str, list[str]], if captured: cand = Path(os.path.normpath(cand / captured)) resolved = _resolve_js_import_path(cand) - if resolved.is_file(): + if _path_is_file(resolved): return resolved if first is None: first = cand return first def _find_workspace_root(start_dir: Path) -> Path | None: - current = start_dir.resolve() + current = _resolve_path(start_dir) for candidate in [current, *current.parents]: - if (candidate / "pnpm-workspace.yaml").exists(): + if _path_exists(candidate / "pnpm-workspace.yaml"): return candidate package_json = candidate / "package.json" - if package_json.is_file(): + if _path_is_file(package_json): try: - data = json.loads(package_json.read_text(encoding="utf-8")) + data = json.loads(_read_file_text(package_json, encoding="utf-8")) except Exception: continue if "workspaces" in data: @@ -339,7 +350,8 @@ def _find_workspace_root(start_dir: Path) -> Path | None: def _pnpm_workspace_globs(workspace_file: Path) -> list[str]: globs: list[str] = [] in_packages = False - for raw_line in workspace_file.read_text(encoding="utf-8", errors="replace").splitlines(): + text = _read_file_text(workspace_file, encoding="utf-8", errors="replace") + for raw_line in text.splitlines(): line = raw_line.strip() if not line or line.startswith("#"): continue @@ -357,12 +369,12 @@ def _pnpm_workspace_globs(workspace_file: Path) -> list[str]: def _workspace_globs(root: Path) -> list[str]: pnpm_workspace = root / "pnpm-workspace.yaml" - if pnpm_workspace.exists(): + if _path_exists(pnpm_workspace): return _pnpm_workspace_globs(pnpm_workspace) package_json = root / "package.json" try: - data = json.loads(package_json.read_text(encoding="utf-8")) + data = json.loads(_read_file_text(package_json, encoding="utf-8")) except Exception: return [] @@ -380,9 +392,9 @@ def _load_workspace_packages(start_dir: Path) -> dict[str, Path]: if root is None: return {} manifest_mtimes = tuple( - (name, (root / name).stat().st_mtime_ns) + (name, _path_stat(root / name).st_mtime_ns) for name in _WORKSPACE_MANIFEST_NAMES - if (root / name).is_file() + if _path_is_file(root / name) ) key = str((root, manifest_mtimes)) if key in _WORKSPACE_PACKAGE_CACHE: @@ -390,13 +402,15 @@ def _load_workspace_packages(start_dir: Path) -> dict[str, Path]: packages: dict[str, Path] = {} for pattern in _workspace_globs(root): - package_dirs: list[Path] = [root] if pattern in (".", "./") else list(root.glob(pattern)) + package_dirs: list[Path] = ( + [root] if pattern in (".", "./") else list(_glob_paths(root, pattern)) + ) for package_dir in package_dirs: manifest = package_dir / "package.json" - if not manifest.is_file(): + if not _path_is_file(manifest): continue try: - data = json.loads(manifest.read_text(encoding="utf-8")) + data = json.loads(_read_file_text(manifest, encoding="utf-8")) except Exception: continue name = data.get("name") @@ -431,7 +445,7 @@ def _contained_in_package(resolved: Path, package_dir: Path) -> bool: (e.g. "./evil": "../../../etc/passwd"). Only accept paths that stay within package_dir after resolution.""" try: - return resolved.resolve().is_relative_to(package_dir.resolve()) + return _resolve_path(resolved).is_relative_to(_resolve_path(package_dir)) except ValueError: return False @@ -439,7 +453,7 @@ def _package_entry_candidates(package_dir: Path, subpath: str) -> list[Path]: manifest = package_dir / "package.json" manifest_data: dict[str, Any] = {} try: - manifest_data = json.loads(manifest.read_text(encoding="utf-8")) + manifest_data = json.loads(_read_file_text(manifest, encoding="utf-8")) except Exception: pass @@ -498,7 +512,7 @@ def _resolve_workspace_import(raw: str, start_dir: Path) -> Path | None: continue for candidate in _package_entry_candidates(package_dir, subpath): resolved = _resolve_js_import_path(candidate) - if resolved.is_file(): + if _path_is_file(resolved): return resolved return None @@ -560,8 +574,8 @@ def _resolve_c_include_path(raw: str, str_path: str) -> "Path | None": """ if not raw: return None - candidate = (Path(str_path).parent / raw).resolve() - if candidate.is_file(): + candidate = _resolve_path(Path(str_path).parent / raw) + if _path_is_file(candidate): return candidate return None @@ -591,11 +605,11 @@ def _resolve_lua_import_target(raw_module: str, str_path: str) -> str: for _ in range(6): for suffix in (".lua", ".luau"): cand = probe / f"{rel}{suffix}" - if cand.is_file(): + if _path_is_file(cand): return _make_id(str(cand)) for suffix in (".lua", ".luau"): cand = probe / rel / f"init{suffix}" - if cand.is_file(): + if _path_is_file(cand): return _make_id(str(cand)) if probe.parent == probe: break @@ -642,7 +656,7 @@ def _source_key(source_file: str, root: Path) -> str: return "" source_path = Path(source_file) try: - return str(source_path.resolve().relative_to(root)) + return str(_resolve_path(source_path).relative_to(_resolve_path(root))) except Exception: return str(source_path) @@ -812,7 +826,7 @@ def _js_source_path(source_file: str, root: Path) -> Path | None: if not path.is_absolute(): path = root / path try: - return path.resolve() + return _resolve_path(path) except Exception: return path @@ -836,8 +850,8 @@ def _apply_symbol_resolution_facts( ): return - path_by_resolved = {path.resolve(): path for path in paths} - source_file_id = {path.resolve(): _make_id(str(path)) for path in paths} + path_by_resolved = {_resolve_path(path): path for path in paths} + source_file_id = {_resolve_path(path): _make_id(str(path)) for path in paths} symbol_nodes: dict[tuple[Path, str], str] = {} for node in nodes: source_path = _js_source_path(str(node.get("source_file", "")), root) @@ -848,7 +862,7 @@ def _apply_symbol_resolution_facts( symbol_nodes[(source_path, label)] = str(node["id"]) def ensure_symbol_node(path: Path, name: str, line: int) -> str: - resolved_path = path.resolve() + resolved_path = _resolve_path(path) existing = symbol_nodes.get((resolved_path, name)) if existing is not None: return existing @@ -905,15 +919,16 @@ def add_edge(source: str, target: str, relation: str, context: str, line: int, s local_aliases_by_file: dict[Path, dict[str, tuple[Path, str]]] = {} for import_fact in facts.imports: - file_path = import_fact.file_path.resolve() + file_path = _resolve_path(import_fact.file_path) local_aliases_by_file.setdefault(file_path, {})[import_fact.local_name] = ( - import_fact.target_path.resolve(), + _resolve_path(import_fact.target_path), import_fact.imported_name, ) pending_aliases_by_file: dict[Path, list[_SymbolAliasFact]] = {} for alias_fact in facts.aliases: - pending_aliases_by_file.setdefault(alias_fact.file_path.resolve(), []).append(alias_fact) + resolved_file = _resolve_path(alias_fact.file_path) + pending_aliases_by_file.setdefault(resolved_file, []).append(alias_fact) for file_path, aliases in pending_aliases_by_file.items(): local_aliases = local_aliases_by_file.setdefault(file_path, {}) @@ -932,8 +947,8 @@ def add_edge(source: str, target: str, relation: str, context: str, line: int, s star_exports_by_file: dict[Path, list[Path]] = {} for star_fact in facts.star_exports: - source_path = star_fact.file_path.resolve() - target_path = star_fact.target_path.resolve() + source_path = _resolve_path(star_fact.file_path) + target_path = _resolve_path(star_fact.target_path) star_exports_by_file.setdefault(source_path, []).append(target_path) source_id = source_file_id.get(source_path) if source_id is not None: @@ -948,8 +963,8 @@ def add_edge(source: str, target: str, relation: str, context: str, line: int, s ) for namespace_fact in facts.namespace_exports: - source_path = namespace_fact.file_path.resolve() - target_path = namespace_fact.target_path.resolve() + source_path = _resolve_path(namespace_fact.file_path) + target_path = _resolve_path(namespace_fact.target_path) namespace_id = ensure_symbol_node( namespace_fact.file_path, namespace_fact.exported_name, @@ -979,10 +994,10 @@ def add_edge(source: str, target: str, relation: str, context: str, line: int, s ) for export_fact in facts.exports: - file_path = export_fact.file_path.resolve() + file_path = _resolve_path(export_fact.file_path) origin: tuple[Path, str] | None = None if export_fact.target_path is not None and export_fact.target_name is not None: - origin = (export_fact.target_path.resolve(), export_fact.target_name) + origin = (_resolve_path(export_fact.target_path), export_fact.target_name) elif export_fact.local_name is not None: origin = local_aliases_by_file.get(file_path, {}).get(export_fact.local_name) if origin is None and (file_path, export_fact.local_name) in symbol_nodes: @@ -1004,7 +1019,7 @@ def add_edge(source: str, target: str, relation: str, context: str, line: int, s ) def resolve_exported_origin(target_path: Path, imported_name: str, seen: set[tuple[Path, str]] | None = None) -> tuple[Path, str]: - target_path = target_path.resolve() + target_path = _resolve_path(target_path) key = (target_path, imported_name) if seen is None: seen = set() @@ -1024,7 +1039,7 @@ def resolve_exported_origin(target_path: Path, imported_name: str, seen: set[tup return key for import_fact in facts.imports: - source_id = source_file_id.get(import_fact.file_path.resolve()) + source_id = source_file_id.get(_resolve_path(import_fact.file_path)) if source_id is None: continue origin_path, origin_symbol = resolve_exported_origin( @@ -1068,7 +1083,7 @@ def resolve_exported_origin(target_path: Path, imported_name: str, seen: set[tup # canonicalizes — or drop it when no file node id is available. owned = {str(n.get("id")) for n in nodes} for use_fact in facts.uses: - file_path = use_fact.file_path.resolve() + file_path = _resolve_path(use_fact.file_path) target_id = None unresolved_origin = local_aliases_by_file.get(file_path, {}).get(use_fact.local_name) if unresolved_origin is not None: @@ -1106,11 +1121,11 @@ def _parse_js_tree(path: Path): vue_lang: str | None = None if path.suffix == ".vue": masked, vue_lang = _vue_mask_non_script( - path.read_text(encoding="utf-8", errors="replace") + _read_file_text(path, encoding="utf-8", errors="replace") ) source = masked.encode("utf-8") else: - source = path.read_bytes() + source = _read_file_bytes(path) use_ts = path.suffix in (".ts", ".mts", ".cts") or ( path.suffix == ".vue" and vue_lang not in ("js", "jsx") ) @@ -1475,7 +1490,7 @@ def _collect_js_symbol_resolution_facts(paths: list[Path], facts: _SymbolResolut trees: dict[Path, tuple[bytes, object]] = {} for path in js_paths: - resolved_path = path.resolve() + resolved_path = _resolve_path(path) parsed = _parse_js_tree(path) if parsed is None: continue @@ -1497,7 +1512,7 @@ def _collect_js_symbol_resolution_facts(paths: list[Path], facts: _SymbolResolut target_path = _resolve_js_module_path(raw_module, path.parent) if target_path is None: continue - target_path = target_path.resolve() + target_path = _resolve_path(target_path) for imported_name, local_name in _js_named_specifiers(node, source, "import_specifier"): facts.imports.append( _SymbolImportFact( @@ -1527,7 +1542,7 @@ def _collect_js_symbol_resolution_facts(paths: list[Path], facts: _SymbolResolut ) for path in js_paths: - resolved_path = path.resolve() + resolved_path = _resolve_path(path) parsed = trees.get(resolved_path) if parsed is None: continue @@ -1543,7 +1558,7 @@ def _collect_js_symbol_resolution_facts(paths: list[Path], facts: _SymbolResolut target_path = _resolve_js_module_path(raw_module, path.parent) if target_path is None: continue - target_path = target_path.resolve() + target_path = _resolve_path(target_path) namespace_name = _js_namespace_export_name(node, source) if namespace_name is not None: facts.namespace_exports.append( @@ -1613,7 +1628,7 @@ def _collect_js_symbol_resolution_facts(paths: list[Path], facts: _SymbolResolut ) for path in js_paths: - resolved_path = path.resolve() + resolved_path = _resolve_path(path) parsed = trees.get(resolved_path) if parsed is None: continue @@ -1635,7 +1650,7 @@ def _collect_js_symbol_resolution_facts(paths: list[Path], facts: _SymbolResolut ) for path in js_paths: - resolved_path = path.resolve() + resolved_path = _resolve_path(path) parsed = trees.get(resolved_path) if parsed is None: continue @@ -1661,7 +1676,7 @@ def _parse_python_tree(path: Path): try: import tree_sitter_python as tspython from tree_sitter import Language, Parser - source = path.read_bytes() + source = _read_file_bytes(path) parser = Parser(Language(tspython.language())) return source, parser.parse(source).root_node except Exception: @@ -1718,14 +1733,14 @@ def _python_imported_names(node, source: bytes) -> list[tuple[str, str]]: def _probe_python_module_candidate(candidate: Path) -> Path | None: """Resolve one module-path candidate to a .py file (dir+__init__, exact, or with a .py suffix), or None.""" - if candidate.is_dir(): + if _path_is_dir(candidate): init_path = candidate / "__init__.py" - if init_path.is_file(): + if _path_is_file(init_path): return init_path - if candidate.is_file(): + if _path_is_file(candidate): return candidate py_candidate = candidate.with_suffix(".py") - if py_candidate.is_file(): + if _path_is_file(py_candidate): return py_candidate return None @@ -1761,7 +1776,7 @@ def _resolve_python_module_path(module_name: str, current_path: Path, root: Path # implicit-relative semantics), fabricating edges to what may be an # external dependency (#2072 review). A src-layout root (src/, no # __init__.py) is still probed. - if (anc / "__init__.py").is_file(): + if _path_is_file(anc / "__init__.py"): continue cand = _probe_python_module_candidate(anc / rel) if cand is not None: @@ -1803,7 +1818,7 @@ def _collect_python_symbol_resolution_facts( if parsed is None: continue source, root_node = parsed - trees[path.resolve()] = parsed + trees[_resolve_path(path)] = parsed for node in _walk_python_tree(root_node): if node.type != "import_from_statement": @@ -1825,7 +1840,11 @@ def _collect_python_symbol_resolution_facts( if pkg_dir is not None: sub_py = pkg_dir / f"{imported_name}.py" sub_pkg = pkg_dir / imported_name / "__init__.py" - submodule = sub_py if sub_py.is_file() else (sub_pkg if sub_pkg.is_file() else None) + submodule = ( + sub_py + if _path_is_file(sub_py) + else (sub_pkg if _path_is_file(sub_pkg) else None) + ) if submodule is not None: facts.module_imports.append((path, submodule, line, local_name)) continue @@ -1844,7 +1863,7 @@ def _collect_python_symbol_resolution_facts( ) for path in py_paths: - parsed = trees.get(path.resolve()) + parsed = trees.get(_resolve_path(path)) if parsed is None: continue source, root_node = parsed @@ -1956,7 +1975,7 @@ def _resolve_cross_file_imports( # Parse imports from this file try: - source = path.read_bytes() + source = _read_file_bytes(path) tree = parser.parse(source) except Exception: continue @@ -2208,7 +2227,7 @@ def _resolve_cross_file_java_imports( for path in paths: file_nid = _make_id(str(path)) try: - source = path.read_bytes() + source = _read_file_bytes(path) tree = parser.parse(source) except Exception: continue @@ -2291,7 +2310,7 @@ def _resolve_java_type_references( if not srcs: continue try: - source = path.read_bytes() + source = _read_file_bytes(path) tree = parser.parse(source) except Exception: continue @@ -2454,7 +2473,7 @@ def _resolve_php_type_references( if not srcs: continue try: - source = path.read_bytes() + source = _read_file_bytes(path) tree = parser.parse(source) except Exception: continue @@ -2670,8 +2689,8 @@ def _pascal_project_root(from_path: Path) -> Path: for _ in range(12): if len(current.parts) <= 1: break # never use a filesystem root (D:/, C:/, /) - pas_count = sum(1 for _ in current.glob("*.pas")) - dpr_count = sum(1 for _ in current.glob("*.dpr")) + pas_count = sum(1 for _ in _glob_paths(current, "*.pas")) + dpr_count = sum(1 for _ in _glob_paths(current, "*.dpr")) if pas_count >= 2 or dpr_count >= 1: best = current parent = current.parent @@ -2694,7 +2713,7 @@ def _pascal_resolve_unit(from_path: Path, unit_name: str) -> str: if root_key not in _pascal_unit_cache: unit_map: dict[str, str] = {} for ext in (".pas", ".pp", ".dpr", ".dpk", ".inc"): - for f in root.rglob("*" + ext): + for f in _glob_paths(root, "**/*" + ext): unit_map[f.stem.lower()] = _make_id(str(f)) _pascal_unit_cache[root_key] = unit_map return _pascal_unit_cache[root_key].get(unit_name.lower(), _make_id(unit_name)) @@ -2717,7 +2736,7 @@ def _pascal_resolve_class(from_path: Path, class_name: str) -> str | None: if root_key not in _pascal_class_stem_cache: stem_map: dict[str, str] = {} for ext in (".pas", ".pp", ".dpr", ".dpk"): - for f in root.rglob("*" + ext): + for f in _glob_paths(root, "**/*" + ext): stem_map[f.stem.lower()] = _file_stem(f) _pascal_class_stem_cache[root_key] = stem_map diff --git a/graphify/extractors/rust.py b/graphify/extractors/rust.py index b663bd625..19e3927d2 100644 --- a/graphify/extractors/rust.py +++ b/graphify/extractors/rust.py @@ -3,6 +3,7 @@ from pathlib import Path +from graphify.paths import read_bytes as _read_file_bytes from graphify.extractors.base import _LANGUAGE_BUILTIN_GLOBALS, _file_stem, _make_id, _read_text @@ -69,7 +70,7 @@ def extract_rust(path: Path) -> dict: try: language = Language(tsrust.language()) parser = Parser(language) - source = path.read_bytes() + source = _read_file_bytes(path) tree = parser.parse(source) root = tree.root_node except Exception as e: diff --git a/graphify/extractors/sln.py b/graphify/extractors/sln.py index 936ff9fff..8d99c4a33 100644 --- a/graphify/extractors/sln.py +++ b/graphify/extractors/sln.py @@ -4,13 +4,14 @@ import re from pathlib import Path +from graphify.paths import read_text as _read_file_text, resolve_path as _resolve_path from graphify.extractors.base import _make_id def extract_sln(path: Path) -> dict: """Extract projects and inter-project dependencies from a .sln file.""" try: - src = path.read_text(encoding="utf-8", errors="replace") + src = _read_file_text(path, encoding="utf-8", errors="replace") except OSError: return {"nodes": [], "edges": [], "error": f"cannot read {path}"} @@ -46,7 +47,7 @@ def extract_sln(path: Path) -> dict: abs_proj = proj_name else: try: - abs_proj = str((path.parent / proj_path).resolve()) + abs_proj = str(_resolve_path(path.parent / proj_path)) except Exception: abs_proj = proj_path proj_nid = _make_id(abs_proj) diff --git a/graphify/extractors/sql.py b/graphify/extractors/sql.py index c2033ec11..604bef267 100644 --- a/graphify/extractors/sql.py +++ b/graphify/extractors/sql.py @@ -4,6 +4,7 @@ import re from pathlib import Path +from graphify.paths import read_bytes as _read_file_bytes from graphify.extractors.base import _file_stem, _make_id @@ -21,7 +22,7 @@ def extract_sql(path: Path, content: str | bytes | None = None) -> dict: source = ( content.encode("utf-8") if isinstance(content, str) else content if content is not None - else path.read_bytes() + else _read_file_bytes(path) ) tree = parser.parse(source) root = tree.root_node diff --git a/graphify/extractors/terraform.py b/graphify/extractors/terraform.py index b5fb78f99..2065d584a 100644 --- a/graphify/extractors/terraform.py +++ b/graphify/extractors/terraform.py @@ -3,6 +3,7 @@ from pathlib import Path +from graphify.paths import read_bytes as _read_file_bytes from graphify.extractors.base import _make_id @@ -31,7 +32,7 @@ def extract_terraform(path: Path) -> dict: try: language = Language(tshcl.language()) parser = Parser(language) - source = path.read_bytes() + source = _read_file_bytes(path) tree = parser.parse(source) root = tree.root_node except Exception as e: diff --git a/graphify/extractors/verilog.py b/graphify/extractors/verilog.py index 2f5fea49b..2de5df442 100644 --- a/graphify/extractors/verilog.py +++ b/graphify/extractors/verilog.py @@ -4,6 +4,7 @@ import re from pathlib import Path +from graphify.paths import read_bytes as _read_file_bytes from graphify.extractors.base import _file_stem, _make_id, _read_text @@ -215,7 +216,7 @@ def extract_verilog(path: Path) -> dict: try: language = Language(tsverilog.language()) parser = Parser(language) - source = path.read_bytes() + source = _read_file_bytes(path) tree = parser.parse(source) root = tree.root_node except Exception as e: diff --git a/graphify/extractors/zig.py b/graphify/extractors/zig.py index 9744c514b..76bbc2d78 100644 --- a/graphify/extractors/zig.py +++ b/graphify/extractors/zig.py @@ -2,6 +2,7 @@ from __future__ import annotations from pathlib import Path +from graphify.paths import read_bytes as _read_file_bytes from typing import Any from graphify.extractors.base import _file_stem, _make_id, _read_text @@ -18,7 +19,7 @@ def extract_zig(path: Path) -> dict: try: language = Language(tszig.language()) parser = Parser(language) - source = path.read_bytes() + source = _read_file_bytes(path) tree = parser.parse(source) root = tree.root_node except Exception as e: diff --git a/graphify/file_slice.py b/graphify/file_slice.py index 30dc49cfb..0bb98986d 100644 --- a/graphify/file_slice.py +++ b/graphify/file_slice.py @@ -22,6 +22,8 @@ from dataclasses import dataclass from pathlib import Path +from graphify.paths import read_text as _read_file_text + # Plain-text document types where boundary-based slicing is meaningful and where # `_file_to_text` is a straight ``read_text`` (so a char range matches the bytes # the model is shown). Deliberately excludes code (.py, .ts, ...) and binary @@ -119,7 +121,7 @@ def expand_oversized_files( out.append(f) continue try: - text = f.read_text(encoding="utf-8", errors="replace") + text = _read_file_text(f, encoding="utf-8", errors="replace") except OSError: out.append(f) continue @@ -135,7 +137,7 @@ def expand_oversized_files( def read_slice_text(fs: FileSlice) -> str: """Read just this slice's characters from its parent file.""" - text = fs.path.read_text(encoding="utf-8", errors="replace") + text = _read_file_text(fs.path, encoding="utf-8", errors="replace") return text[fs.start:fs.end] @@ -149,7 +151,7 @@ def bisect_slice(fs: FileSlice) -> tuple[FileSlice, FileSlice] | None: if fs.end - fs.start <= 1: return None try: - text = fs.path.read_text(encoding="utf-8", errors="replace") + text = _read_file_text(fs.path, encoding="utf-8", errors="replace") except OSError: return None mid = (fs.start + fs.end) // 2 diff --git a/graphify/google_workspace.py b/graphify/google_workspace.py index 1feeb8acd..c74d5e26e 100644 --- a/graphify/google_workspace.py +++ b/graphify/google_workspace.py @@ -18,6 +18,15 @@ from pathlib import Path from typing import Callable, Any +from graphify.paths import ( + io_path as _io_path, + make_dirs as _make_dirs, + read_text as _read_file_text, + resolve_path as _resolve_path, + unlink_path as _unlink_path, + write_text as _write_file_text, +) + GOOGLE_WORKSPACE_EXTENSIONS = {".gdoc", ".gsheet", ".gslides"} @@ -63,7 +72,7 @@ def _extract_resource_key(url: str, data: dict[str, Any]) -> str | None: def read_google_shortcut(path: Path) -> dict[str, str | None]: """Read a .gdoc/.gsheet/.gslides shortcut and return export metadata.""" try: - data = json.loads(path.read_text(encoding="utf-8")) + data = json.loads(_read_file_text(path, encoding="utf-8")) except Exception as exc: raise RuntimeError(f"could not read Google Workspace shortcut {path}: {exc}") from exc @@ -104,12 +113,14 @@ def _run_gws_export(file_id: str, mime_type: str, output: Path, resource_key: st # gws export command has no custom-header flag, so do not pass resourceKey # as an unsupported query parameter. _ = resource_key - output = output.resolve() - output.parent.mkdir(parents=True, exist_ok=True) + output = _resolve_path(output) + _make_dirs(output.parent, exist_ok=True) timeout = int(os.environ.get("GRAPHIFY_GOOGLE_WORKSPACE_TIMEOUT", "120")) result = subprocess.run( [exe, "drive", "files", "export", "--params", json.dumps(params), "-o", output.name], capture_output=True, + # Keep transport-only ``\\?\`` spelling inside Python filesystem + # calls; third-party executables should receive the ordinary path. cwd=output.parent, text=True, timeout=timeout, @@ -132,9 +143,9 @@ def _sidecar_path(path: Path, out_dir: Path, root: "Path | None" = None) -> Path if root is None: root = out_dir.parent.parent try: - key = path.resolve().relative_to(Path(root).resolve()).as_posix() + key = _resolve_path(path).relative_to(_resolve_path(root)).as_posix() except (ValueError, OSError): - key = str(path.resolve()) + key = str(_resolve_path(path)) name_hash = hashlib.sha256(unicodedata.normalize("NFC", key).encode()).hexdigest()[:8] return out_dir / f"{path.stem}_{name_hash}.md" @@ -177,39 +188,63 @@ def convert_google_workspace_file( return None shortcut = read_google_shortcut(path) - out_dir.mkdir(parents=True, exist_ok=True) + _make_dirs(out_dir, exist_ok=True) out_path = _sidecar_path(path, out_dir, root=root) if ext == ".gdoc": - with tempfile.NamedTemporaryFile("w+b", suffix=".md", delete=False, dir=out_dir) as tmp: + with tempfile.NamedTemporaryFile( + "w+b", suffix=".md", delete=False, dir=_io_path(out_dir) + ) as tmp: tmp_path = Path(tmp.name) try: - _run_gws_export(shortcut["file_id"] or "", "text/markdown", tmp_path, shortcut.get("resource_key")) - body = tmp_path.read_text(encoding="utf-8", errors="replace") + _run_gws_export( + shortcut["file_id"] or "", + "text/markdown", + tmp_path, + shortcut.get("resource_key"), + ) + body = _read_file_text(tmp_path, encoding="utf-8", errors="replace") finally: - tmp_path.unlink(missing_ok=True) + _unlink_path(tmp_path, missing_ok=True) if not body.strip(): return None - out_path.write_text(_with_frontmatter(path, shortcut, body, "text/markdown"), encoding="utf-8") + _write_file_text( + out_path, + _with_frontmatter(path, shortcut, body, "text/markdown"), + encoding="utf-8", + ) return out_path if ext == ".gslides": - with tempfile.NamedTemporaryFile("w+b", suffix=".txt", delete=False, dir=out_dir) as tmp: + with tempfile.NamedTemporaryFile( + "w+b", suffix=".txt", delete=False, dir=_io_path(out_dir) + ) as tmp: tmp_path = Path(tmp.name) try: - _run_gws_export(shortcut["file_id"] or "", "text/plain", tmp_path, shortcut.get("resource_key")) - body = tmp_path.read_text(encoding="utf-8", errors="replace") + _run_gws_export( + shortcut["file_id"] or "", + "text/plain", + tmp_path, + shortcut.get("resource_key"), + ) + body = _read_file_text(tmp_path, encoding="utf-8", errors="replace") finally: - tmp_path.unlink(missing_ok=True) + _unlink_path(tmp_path, missing_ok=True) if not body.strip(): return None - out_path.write_text(_with_frontmatter(path, shortcut, body, "text/plain"), encoding="utf-8") + _write_file_text( + out_path, + _with_frontmatter(path, shortcut, body, "text/plain"), + encoding="utf-8", + ) return out_path if ext == ".gsheet": if xlsx_to_markdown is None: raise RuntimeError("Google Sheets export requires the office extra: pip install graphifyy[office,google]") - with tempfile.NamedTemporaryFile("w+b", suffix=".xlsx", delete=False, dir=out_dir) as tmp: + with tempfile.NamedTemporaryFile( + "w+b", suffix=".xlsx", delete=False, dir=_io_path(out_dir) + ) as tmp: tmp_path = Path(tmp.name) try: _run_gws_export( @@ -220,10 +255,11 @@ def convert_google_workspace_file( ) body = xlsx_to_markdown(tmp_path) finally: - tmp_path.unlink(missing_ok=True) + _unlink_path(tmp_path, missing_ok=True) if not body.strip(): return None - out_path.write_text( + _write_file_text( + out_path, _with_frontmatter( path, shortcut, diff --git a/graphify/llm.py b/graphify/llm.py index 30d7a6d6f..24937684e 100644 --- a/graphify/llm.py +++ b/graphify/llm.py @@ -16,6 +16,14 @@ from dataclasses import dataclass, replace from pathlib import Path +from graphify.paths import ( + path_is_file as _path_is_file, + path_stat as _path_stat, + read_bytes as _read_file_bytes, + read_text as _read_file_text, + resolve_path as _resolve_path, +) + from graphify.file_slice import ( FileSlice, bisect_slice, @@ -269,7 +277,7 @@ def _load_custom_providers() -> dict[str, dict]: local_path = _custom_providers_path(global_=False) global_path = _custom_providers_path(global_=True) allow_local = os.environ.get("GRAPHIFY_ALLOW_LOCAL_PROVIDERS", "").strip().lower() in ("1", "true", "yes") - if local_path.is_file() and not allow_local: + if _path_is_file(local_path) and not allow_local: print( f"[graphify] WARNING: ignoring project-local {local_path} (custom providers control " "where your corpus and API key are sent). Set GRAPHIFY_ALLOW_LOCAL_PROVIDERS=1 to load it.", @@ -279,9 +287,9 @@ def _load_custom_providers() -> dict[str, dict]: providers: dict[str, dict] = {} paths = [local_path, global_path] if allow_local else [global_path] for path in paths: - if path.is_file(): + if _path_is_file(path): try: - data = json.loads(path.read_text(encoding="utf-8")) + data = json.loads(_read_file_text(path, encoding="utf-8")) if isinstance(data, dict): for name, cfg in data.items(): if not (isinstance(name, str) and isinstance(cfg, dict)): @@ -505,14 +513,14 @@ def _file_to_text(path: Path) -> str: if path.suffix.lower() == ".pdf": from graphify.detect import extract_pdf_text return extract_pdf_text(path) - return path.read_text(encoding="utf-8", errors="replace") + return _read_file_text(path, encoding="utf-8", errors="replace") def _resolve_under_root(path: Path, root: Path) -> Path | None: """Return the resolved path only when it stays inside ``root``.""" try: - resolved_root = root.resolve() - resolved_path = path.resolve() + resolved_root = _resolve_path(root) + resolved_path = _resolve_path(path) resolved_path.relative_to(resolved_root) except (OSError, RuntimeError, ValueError): return None @@ -698,7 +706,7 @@ def _bind_node_evidence(result: dict, text_units: "list[Path | FileSlice]", root if not p.is_absolute(): p = root / p try: - key = p.resolve() + key = _resolve_path(p) except (OSError, RuntimeError): continue src = source_by_path.get(key) @@ -819,7 +827,7 @@ def _build_image_refs(image_files: list[Path], root: Path, *, read_bytes: bool = raw: bytes | None = None if read_bytes: try: - raw = abs_path.read_bytes() + raw = _read_file_bytes(abs_path) except OSError as exc: print(f"[graphify] could not read image {rel}: {exc}", file=sys.stderr) raw = None @@ -1842,14 +1850,14 @@ def _estimate_file_tokens(unit: "Path | FileSlice") -> int: return _IMAGE_TOKEN_ESTIMATE if _TOKENIZER is None: try: - size = path.stat().st_size + size = _path_stat(path).st_size except OSError: return 0 chars = min(size, _FILE_CHAR_CAP) + _PER_FILE_OVERHEAD_CHARS return chars // _CHARS_PER_TOKEN try: - content = path.read_text(encoding="utf-8", errors="replace")[:_FILE_CHAR_CAP] + content = _read_file_text(path, encoding="utf-8", errors="replace")[:_FILE_CHAR_CAP] except OSError: return 0 return len(_TOKENIZER.encode(content, disallowed_special=())) + (_PER_FILE_OVERHEAD_CHARS // _CHARS_PER_TOKEN) @@ -2407,7 +2415,7 @@ def _resolve_against_root(value: "str | Path") -> Path: if not p.is_absolute(): p = root / p try: - return p.resolve() + return _resolve_path(p) except (OSError, RuntimeError): return p @@ -2418,7 +2426,7 @@ def _out_of_scope(item: dict) -> bool: if not sf: return False p = _resolve_against_root(sf) - return p.is_file() and p not in _dispatched_resolved + return _path_is_file(p) and p not in _dispatched_resolved dropped_ids: set = set() dropped_files: set[str] = set() @@ -2466,7 +2474,7 @@ def _out_of_scope(item: dict) -> bool: covered.add(p if p.is_absolute() else (root / p)) uncovered = sorted( p for p in dispatched - if p.resolve() not in {c.resolve() for c in covered} + if _resolve_path(p) not in {_resolve_path(c) for c in covered} ) merged["uncovered_files"] = [str(p) for p in uncovered] if uncovered: diff --git a/graphify/manifest_ingest.py b/graphify/manifest_ingest.py index ae3aa61fc..f6c712ad7 100644 --- a/graphify/manifest_ingest.py +++ b/graphify/manifest_ingest.py @@ -21,6 +21,7 @@ from typing import Any from graphify.ids import make_id +from graphify.paths import path_stat as _path_stat, read_text as _read_file_text __all__ = ["is_package_manifest_path", "extract_package_manifest", "PACKAGE_MANIFEST_NAMES"] @@ -51,9 +52,9 @@ def _pkg_id(name: str) -> str: def extract_package_manifest(path: Path) -> dict[str, Any]: """Parse a package manifest into a canonical package node + ``depends_on`` edges.""" try: - if path.stat().st_size > _MAX_MANIFEST_BYTES: + if _path_stat(path).st_size > _MAX_MANIFEST_BYTES: return {"nodes": [], "edges": [], "error": "manifest too large to index"} - text = path.read_text(encoding="utf-8", errors="replace") + text = _read_file_text(path, encoding="utf-8", errors="replace") except OSError as exc: return {"nodes": [], "edges": [], "error": f"manifest read error: {exc}"} diff --git a/graphify/mcp_ingest.py b/graphify/mcp_ingest.py index 152e4093f..6bc2d9ca5 100644 --- a/graphify/mcp_ingest.py +++ b/graphify/mcp_ingest.py @@ -64,6 +64,7 @@ from typing import Any from graphify.ids import make_id as _shared_make_id +from graphify.paths import read_bytes as _read_file_bytes from graphify.security import sanitize_label @@ -92,8 +93,7 @@ def extract_mcp_config(path: Path) -> dict[str, Any]: failure, oversize file, or missing ``mcpServers`` map """ try: - with path.open("rb") as fh: - raw = fh.read(_MAX_BYTES + 1) + raw = _read_file_bytes(path, limit=_MAX_BYTES + 1) except OSError as exc: return {"nodes": [], "edges": [], "error": f"mcp_ingest read error: {exc}"} diff --git a/graphify/paths.py b/graphify/paths.py index a1adaf9f2..837686906 100644 --- a/graphify/paths.py +++ b/graphify/paths.py @@ -1,31 +1,290 @@ -"""Single source of truth for the graphify output-directory name. +r"""Cross-platform filesystem boundaries and Graphify output paths. + +Graphify stores ordinary, user-facing paths in graph IDs, manifests, cache keys, +and diagnostics. On Windows, direct filesystem calls additionally need the +extended-length namespace (``\\?\`` for local paths and ``\\?\UNC\`` for +UNC paths) to reach deeply nested corpus files without depending on machine-wide +policy. The helpers in this module keep that transport-only spelling at the I/O +boundary so Linux and macOS remain no-ops and Windows paths retain one stable +logical identity. The output directory is ``graphify-out`` by default and overridable with the -``GRAPHIFY_OUT`` env var (worktrees or shared-output setups, #686). It accepts a -relative name (``"graphify-out-feature"``) or an absolute path +``GRAPHIFY_OUT`` environment variable (worktrees or shared-output setups, #686). +It accepts a relative name (``"graphify-out-feature"``) or an absolute path (``"/shared/graphify-out"``). This used to be duplicated as an identical ``_GRAPHIFY_OUT`` constant in ``__main__``, ``cache``, and ``watch``, while ``security`` and ``callflow_html`` hardcoded the literal ``"graphify-out"`` and silently ignored the override (#1423). Centralising it here keeps the name in one place. The value is read -once at import time, matching the previous per-module constants — set -``GRAPHIFY_OUT`` before the process starts (the normal worktree/shared-output -flow) and every reader honours it. +once at import time, matching the previous per-module constants; set +``GRAPHIFY_OUT`` before the process starts and every reader honours it. """ from __future__ import annotations import json +import ntpath import os import re import stat +import sys import tempfile +from collections.abc import Callable, Iterator from pathlib import Path, PurePosixPath +from typing import Any GRAPHIFY_OUT = os.environ.get("GRAPHIFY_OUT", "graphify-out") +PathLike = str | os.PathLike[str] + + +def io_path(path: PathLike) -> str: + r"""Return a path string suitable for Windows file-system I/O. + + Windows' legacy Win32 namespace rejects paths near ``MAX_PATH`` unless the + machine-wide long-path policy is enabled. The extended-length path syntax + does not depend on that policy: local paths use ``\\?\C:\...`` + and UNC paths use ``\\?\UNC\server\share\...``. + + Keep this conversion at the I/O boundary only. Graph IDs, manifests, + diagnostics, ignore matching, and user-visible paths should retain their + ordinary spelling, because the extended prefix is an API transport detail. + """ + value = os.fspath(path) + if sys.platform != "win32": + return value + if value.startswith(("\\\\?\\", "\\\\.\\")): + return value + + # Extended-length paths must be absolute and use backslashes. ntpath is + # used explicitly so this helper is unit-testable from non-Windows CI. + value = ntpath.abspath(value) + if value.startswith("\\\\"): + return "\\\\?\\UNC\\" + value[2:] + return "\\\\?\\" + value + + +def logical_path(path: PathLike) -> str: + r"""Remove a Windows extended-length prefix from a path string. + + ``os.walk(io_path(root))`` yields prefixed directory names. Strip the + transport prefix before paths enter graphify's matching/storage logic so a + long path does not acquire a second identity merely because it was opened + through the Windows extended namespace. + """ + value = os.fspath(path) + if value.upper().startswith("\\\\?\\UNC\\"): + return "\\\\" + value[8:] + if value.startswith("\\\\?\\"): + return value[4:] + return value + + +def resolve_path(path: PathLike) -> Path: + """Resolve ``path`` through the I/O namespace and return ordinary spelling. + + This mirrors ``Path.resolve(strict=False)`` while ensuring the operation can + reach a long Windows path even when the host's global long-path policy is + disabled. The returned ``Path`` intentionally has no extended prefix. + """ + return Path(logical_path(os.path.realpath(io_path(path)))) + + +def path_exists(path: PathLike) -> bool: + """Return whether ``path`` exists, using Windows-safe path spelling.""" + return os.path.exists(io_path(path)) + + +def path_is_file(path: PathLike) -> bool: + """Return whether ``path`` is a regular file, using Windows-safe spelling.""" + return os.path.isfile(io_path(path)) + + +def path_is_dir(path: PathLike) -> bool: + """Return whether ``path`` is a directory, using Windows-safe spelling.""" + return os.path.isdir(io_path(path)) + + +def path_is_symlink(path: PathLike) -> bool: + """Return whether ``path`` is a symbolic link, using Windows-safe spelling.""" + return os.path.islink(io_path(path)) + + +def path_stat(path: PathLike, *, follow_symlinks: bool = True) -> os.stat_result: + """Stat ``path`` through the Windows-safe I/O namespace.""" + return os.stat(io_path(path), follow_symlinks=follow_symlinks) + + +def make_dirs( + path: PathLike, + mode: int = 0o777, + *, + exist_ok: bool = False, +) -> None: + """Create a directory tree through the Windows-safe I/O namespace.""" + os.makedirs(io_path(path), mode=mode, exist_ok=exist_ok) + + +def scandir_path(path: PathLike) -> os.ScandirIterator[str]: + """Return ``os.scandir`` for ``path`` using Windows-safe spelling.""" + return os.scandir(io_path(path)) + + +def iterdir_path(path: PathLike) -> Iterator[Path]: + r"""Yield direct children while preserving the caller's path spelling. + + ``DirEntry.path`` inherits the extended absolute root supplied to + :func:`os.scandir`. Rebuild each child from the logical input root and the + entry name so a relative input remains relative and ``\\?\`` never escapes. + """ + logical_root = Path(logical_path(path)) + with scandir_path(path) as entries: + for entry in entries: + yield logical_root / entry.name + + +def glob_paths(path: PathLike, pattern: str) -> Iterator[Path]: + """Yield glob matches below ``path`` while retaining logical spelling. + + Prefix the root before delegating to :meth:`Path.glob`; this preserves + pathlib's matching behavior (including dotfiles) while every recursive + ``scandir`` stays in the extended Windows namespace. Matches are then + reconstructed relative to the caller's root, avoiding an accidental + relative-to-absolute API change on Windows. + """ + logical_root = Path(logical_path(path)) + filesystem_root = Path(io_path(path)) + for match in filesystem_root.glob(pattern): + try: + relative = match.relative_to(filesystem_root) + except ValueError: + # Defensive fallback for an unusual pathlib implementation; glob + # matches should ordinarily remain below their root. + yield Path(logical_path(match)) + else: + yield logical_root / relative + + +def unlink_path(path: PathLike, *, missing_ok: bool = False) -> None: + """Remove a file or symlink through the Windows-safe namespace.""" + try: + os.unlink(io_path(path)) + except FileNotFoundError: + if not missing_ok: + raise + + +def replace_path(source: PathLike, destination: PathLike) -> None: + """Atomically replace ``destination`` using Windows-safe path spellings.""" + os.replace(io_path(source), io_path(destination)) + + +def walk_path( + path: PathLike, + *, + topdown: bool = True, + onerror: Callable[[OSError], Any] | None = None, + followlinks: bool = False, +) -> Iterator[tuple[str, list[str], list[str]]]: + """Walk ``path`` safely and yield ordinary, user-facing path spellings. + + ``os.walk`` must receive the extended Windows form at the traversal root; + otherwise a later ``scandir`` fails as soon as a descendant crosses the + legacy ``MAX_PATH`` boundary. The extended prefix is stripped from every + yielded directory and from errors before control returns to callers. A + relative input remains relative even though the Windows I/O root must be + absolute before it can use the extended namespace. + """ + logical_root = logical_path(path) + filesystem_root = io_path(path) + ordinary_filesystem_root = logical_path(filesystem_root) + + def _from_filesystem_path(value: PathLike) -> str: + ordinary = logical_path(value) + if sys.platform != "win32": + return ordinary + try: + relative = ntpath.relpath(ordinary, ordinary_filesystem_root) + except ValueError: + # Different UNC shares/drives cannot be relativized; stripping the + # transport prefix is still the safest public representation. + return ordinary + if relative == ".": + return logical_root + return ntpath.join(logical_root, relative) + + def _onerror(error: OSError) -> None: + filename = getattr(error, "filename", None) + if filename is not None: + try: + error.filename = _from_filesystem_path(filename) + except (AttributeError, TypeError): + pass + if onerror is not None: + onerror(error) + + handler = _onerror if onerror is not None else None + for dirpath, dirnames, filenames in os.walk( + filesystem_root, + topdown=topdown, + onerror=handler, + followlinks=followlinks, + ): + yield _from_filesystem_path(dirpath), dirnames, filenames + + +def read_bytes(path: PathLike, *, limit: int | None = None) -> bytes: + """Read bytes through the Windows-safe I/O spelling of ``path``. + + ``limit`` mirrors ``file.read(limit)`` and supports bounded probes without a + separate, long-path-unsafe ``Path.open`` call. + """ + with open(io_path(path), "rb") as fh: + return fh.read() if limit is None else fh.read(limit) + + +def read_text( + path: PathLike, + *, + encoding: str | None = None, + errors: str | None = None, +) -> str: + """Read text through the Windows-safe I/O spelling of ``path``. + + The default encoding intentionally mirrors :meth:`Path.read_text` and the + built-in :func:`open`; corpus readers that require UTF-8 pass it explicitly. + """ + with open(io_path(path), "r", encoding=encoding, errors=errors) as fh: + return fh.read() + + +def write_text( + path: PathLike, + text: str, + *, + encoding: str | None = None, + errors: str | None = None, + newline: str | None = None, +) -> int: + """Write text through the Windows-safe I/O spelling of ``path``.""" + with open( + io_path(path), + "w", + encoding=encoding, + errors=errors, + newline=newline, + ) as fh: + return fh.write(text) + + +def write_bytes(path: PathLike, data: bytes) -> int: + """Write bytes through the Windows-safe I/O spelling of ``path``.""" + with open(io_path(path), "wb") as fh: + return fh.write(data) + + def _atomic_replace(path: "str | Path", write_fn) -> None: """Atomically replace ``path`` with content written by ``write_fn(f)``. @@ -43,9 +302,13 @@ def _atomic_replace(path: "str | Path", write_fn) -> None: """ # Resolve symlinks so the temp lands on the target's filesystem (same-fs # atomic rename) and the replace writes through the link, not over it. - real = Path(os.path.realpath(str(path))) - real.parent.mkdir(parents=True, exist_ok=True) - fd, tmp = tempfile.mkstemp(dir=str(real.parent), prefix=f".{real.name}.", suffix=".tmp") + real = resolve_path(path) + make_dirs(real.parent, exist_ok=True) + fd, tmp = tempfile.mkstemp( + dir=io_path(real.parent), + prefix=f".{real.name}.", + suffix=".tmp", + ) try: with os.fdopen(fd, "w", encoding="utf-8") as f: write_fn(f) @@ -54,7 +317,7 @@ def _atomic_replace(path: "str | Path", write_fn) -> None: # silently tightens a previously group/world-readable output to # owner-only. Best-effort — a chmod failure must not fail the write. try: - mode = stat.S_IMODE(os.stat(real).st_mode) + mode = stat.S_IMODE(path_stat(real).st_mode) except OSError: umask = os.umask(0) os.umask(umask) @@ -64,13 +327,13 @@ def _atomic_replace(path: "str | Path", write_fn) -> None: except OSError: pass try: - os.replace(tmp, str(real)) + os.replace(tmp, io_path(real)) except PermissionError: # Windows: os.replace fails (WinError 5/32) when the destination is # briefly locked by another handle (antivirus, an open reader). Fall # back to copy-then-delete, matching graphify.cache's atomic writer. import shutil - shutil.copy2(tmp, str(real)) + shutil.copy2(tmp, io_path(real)) os.unlink(tmp) except BaseException: try: @@ -336,7 +599,7 @@ def load_node_link_graph(path_or_data): p = Path(data) from graphify.security import check_graph_file_size_cap # lazy: security imports paths check_graph_file_size_cap(p) - data = json.loads(p.read_text(encoding="utf-8")) + data = json.loads(read_text(p, encoding="utf-8")) if isinstance(data, dict) and "links" not in data and "edges" in data: data = dict(data, links=data["edges"]) try: diff --git a/graphify/security.py b/graphify/security.py index 2dbe5bd77..d3dbebabf 100644 --- a/graphify/security.py +++ b/graphify/security.py @@ -15,7 +15,13 @@ import ipaddress import socket -from graphify.paths import GRAPHIFY_OUT, GRAPHIFY_OUT_NAME +from graphify.paths import ( + GRAPHIFY_OUT, + GRAPHIFY_OUT_NAME, + path_exists, + path_stat, + resolve_path, +) _ALLOWED_SCHEMES = {"http", "https"} _MAX_FETCH_BYTES = 52_428_800 # 50 MB hard cap for binary downloads @@ -324,22 +330,22 @@ def validate_graph_path(path: str | Path, base: Path | None = None) -> Path: FileNotFoundError - resolved path does not exist """ if base is None: - resolved_hint = Path(path).resolve() + resolved_hint = resolve_path(path) for candidate in [resolved_hint, *resolved_hint.parents]: if candidate.name == GRAPHIFY_OUT_NAME: base = candidate break if base is None: - base = Path(GRAPHIFY_OUT).resolve() + base = resolve_path(GRAPHIFY_OUT) - base = base.resolve() - if not base.exists(): + base = resolve_path(base) + if not path_exists(base): raise ValueError( f"Graph base directory does not exist: {base}. " "Run /graphify first to build the graph." ) - resolved = Path(path).resolve() + resolved = resolve_path(path) try: resolved.relative_to(base) except ValueError: @@ -348,7 +354,7 @@ def validate_graph_path(path: str | Path, base: Path | None = None) -> Path: "Only paths inside graphify-out/ are permitted." ) - if not resolved.exists(): + if not path_exists(resolved): raise FileNotFoundError(f"Graph file not found: {resolved}") return resolved @@ -372,7 +378,7 @@ def check_graph_file_size_cap(path: Path) -> None: """ cap = _max_graph_file_bytes() try: - size = path.stat().st_size + size = path_stat(path).st_size except OSError: return if size > cap: diff --git a/graphify/symbol_resolution.py b/graphify/symbol_resolution.py index 892f31065..d75c4717c 100644 --- a/graphify/symbol_resolution.py +++ b/graphify/symbol_resolution.py @@ -5,11 +5,16 @@ import ast import re import unicodedata +from collections.abc import Sequence from dataclasses import dataclass from pathlib import Path -from collections.abc import Sequence from typing import Any +from graphify.paths import ( + read_text as _read_file_text, + resolve_path as _resolve_path, +) + from graphify.ids import make_id as _shared_make_id from graphify.paths import disambiguate_ambiguous_candidates from graphify.security import sanitize_metadata @@ -135,7 +140,7 @@ def parse_python_import_aliases(path: Path) -> dict[str, ImportedSymbol]: """ try: - source = path.read_text(encoding="utf-8", errors="replace") + source = _read_file_text(path, encoding="utf-8", errors="replace") tree = ast.parse(source) except (OSError, SyntaxError): return {} @@ -395,7 +400,7 @@ def _file_node_id_for_path(path: Path, root: Path) -> str: # node instead of an orphan. _bash_make_id / _bash_file_stem are exact copies # of extract._make_id / extract._file_stem, so IDs match. try: - rel = path.resolve().relative_to(root.resolve()) + rel = _resolve_path(path).relative_to(_resolve_path(root)) except ValueError: return _bash_make_id(str(path)) # path outside root: hash absolute path as fallback return _bash_make_id(_bash_file_stem(rel)) @@ -427,7 +432,7 @@ def resolve_bash_source_edges( - Inputs of type ``str`` and ``pathlib.Path`` are processed. Anything else is silently skipped. """ - path_by_index = [Path(p).resolve() for p in paths] + path_by_index = [_resolve_path(p) for p in paths] file_nid_by_path = {p: _file_node_id_for_path(p, root) for p in path_by_index} # resolved paths only functions_by_file: dict[str, dict[str, str]] = {} @@ -478,7 +483,7 @@ def resolve_bash_source_edges( if not candidate.is_absolute(): candidate = path.parent / candidate try: - target_path = candidate.resolve() + target_path = _resolve_path(candidate) except (OSError, RuntimeError): continue target_file_nid = file_nid_by_path.get(target_path) diff --git a/graphify/transcribe.py b/graphify/transcribe.py index 8e45bcaa2..6f23022e5 100644 --- a/graphify/transcribe.py +++ b/graphify/transcribe.py @@ -5,7 +5,14 @@ import os from pathlib import Path -from graphify.paths import out_path as _out_path +from graphify.paths import ( + glob_paths as _glob_paths, + io_path as _io_path, + make_dirs as _make_dirs, + out_path as _out_path, + path_exists as _path_exists, + write_text as _write_file_text, +) VIDEO_EXTENSIONS = {'.mp4', '.mov', '.webm', '.mkv', '.avi', '.m4v', '.mp3', '.wav', '.m4a', '.ogg'} @@ -56,7 +63,7 @@ def download_audio(url: str, output_dir: Path) -> Path: from graphify.security import validate_url validate_url(url) # blocks private IPs, bad schemes before yt-dlp runs yt_dlp = _get_yt_dlp() - output_dir.mkdir(parents=True, exist_ok=True) + _make_dirs(output_dir, exist_ok=True) # yt-dlp uses %(title)s which can be long/weird — use a stable name based on URL hash import hashlib @@ -66,7 +73,7 @@ def download_audio(url: str, output_dir: Path) -> Path: # Check for already-downloaded file for ext in ('.m4a', '.opus', '.mp3', '.ogg', '.wav', '.webm'): candidate = output_dir / f"yt_{url_hash}{ext}" - if candidate.exists(): + if _path_exists(candidate): print(f" cached audio: {candidate.name}") return candidate @@ -84,9 +91,9 @@ def download_audio(url: str, output_dir: Path) -> Path: info = ydl.extract_info(url, download=True) ext = info.get('ext', 'm4a') downloaded = output_dir / f"yt_{url_hash}.{ext}" - if not downloaded.exists(): + if not _path_exists(downloaded): # yt-dlp may have picked a different extension - for p in output_dir.glob(f"yt_{url_hash}.*"): + for p in _glob_paths(output_dir, f"yt_{url_hash}.*"): downloaded = p break return downloaded @@ -131,7 +138,7 @@ def transcribe( force: re-transcribe even if transcript already exists. """ out_dir = Path(output_dir) if output_dir else Path(_TRANSCRIPTS_DIR) - out_dir.mkdir(parents=True, exist_ok=True) + _make_dirs(out_dir, exist_ok=True) if is_url(str(video_path)): audio_path = download_audio(str(video_path), out_dir / "downloads") @@ -139,7 +146,7 @@ def transcribe( audio_path = Path(video_path) transcript_path = out_dir / (audio_path.stem + ".txt") - if transcript_path.exists() and not force: + if _path_exists(transcript_path) and not force: return transcript_path WhisperModel = _get_whisper() @@ -149,7 +156,7 @@ def transcribe( print(f" transcribing {audio_path.name} (model={model_name}) ...", flush=True) model = WhisperModel(model_name, device="cpu", compute_type="int8") segments, info = model.transcribe( - str(audio_path), + _io_path(audio_path), beam_size=5, initial_prompt=prompt, ) @@ -157,7 +164,7 @@ def transcribe( lines = [segment.text.strip() for segment in segments if segment.text.strip()] transcript = "\n".join(lines) - transcript_path.write_text(transcript, encoding="utf-8") + _write_file_text(transcript_path, transcript, encoding="utf-8") lang = info.language if hasattr(info, "language") else "unknown" print(f" transcript saved -> {transcript_path} (lang={lang}, {len(lines)} segments)") return transcript_path diff --git a/graphify/watch.py b/graphify/watch.py index c87ec8e6f..fb3b55891 100644 --- a/graphify/watch.py +++ b/graphify/watch.py @@ -10,7 +10,20 @@ from pathlib import Path # Single source of truth in graphify.paths (#1423); re-exported as _GRAPHIFY_OUT. -from graphify.paths import GRAPHIFY_OUT as _GRAPHIFY_OUT +from graphify.paths import ( + GRAPHIFY_OUT as _GRAPHIFY_OUT, + glob_paths as _glob_paths, + io_path as _io_path, + make_dirs as _make_dirs, + path_exists as _path_exists, + path_is_dir as _path_is_dir, + path_is_file as _path_is_file, + read_text as _read_text, + replace_path as _replace_path, + resolve_path as _resolve_path, + unlink_path as _unlink_path, + write_text as _write_text, +) _PENDING_FILENAME = ".pending_changes" _PENDING_DRAIN_MAX_PASSES = 20 @@ -30,10 +43,10 @@ def _queue_pending(out_dir: Path, changed_paths: list[Path]) -> None: """ if not changed_paths: return - out_dir.mkdir(parents=True, exist_ok=True) + _make_dirs(out_dir, exist_ok=True) pending = out_dir / _PENDING_FILENAME payload = "".join(f"{os.fspath(p)}\n" for p in changed_paths) - with open(pending, "a", encoding="utf-8") as fh: + with open(_io_path(pending), "a", encoding="utf-8") as fh: fh.write(payload) @@ -45,10 +58,10 @@ def _drain_pending(out_dir: Path) -> list[Path]: fragment cannot poison the merge. """ pending = out_dir / _PENDING_FILENAME - if not pending.exists(): + if not _path_exists(pending): return [] try: - raw = pending.read_text(encoding="utf-8") + raw = _read_text(pending, encoding="utf-8") except OSError: return [] # Unlink BEFORE returning so a crash between read and process retains the @@ -57,7 +70,7 @@ def _drain_pending(out_dir: Path) -> list[Path]: # bug. Use missing_ok to tolerate a racing drain on platforms where # rename/unlink may interleave. with contextlib.suppress(FileNotFoundError): - pending.unlink() + _unlink_path(pending) seen: set[str] = set() out: list[Path] = [] for line in raw.splitlines(): @@ -89,10 +102,10 @@ def _write_build_config( if not excludes and gitignore is None: return try: - out_dir.mkdir(parents=True, exist_ok=True) + _make_dirs(out_dir, exist_ok=True) path = out_dir / _BUILD_CONFIG_FILENAME try: - config = json.loads(path.read_text(encoding="utf-8")) if path.is_file() else {} + config = json.loads(_read_text(path, encoding="utf-8")) if _path_is_file(path) else {} except (OSError, json.JSONDecodeError): config = {} if not isinstance(config, dict): @@ -101,7 +114,7 @@ def _write_build_config( config["excludes"] = list(excludes) if gitignore is not None: config["gitignore"] = gitignore - path.write_text(json.dumps(config), encoding="utf-8") + _write_text(path, json.dumps(config), encoding="utf-8") except OSError: pass @@ -110,8 +123,8 @@ def _read_build_excludes(out_dir: Path) -> list[str]: """Return the persisted ``--exclude`` patterns for this graph, or [].""" try: path = out_dir / _BUILD_CONFIG_FILENAME - if path.is_file(): - cfg = json.loads(path.read_text(encoding="utf-8")) + if _path_is_file(path): + cfg = json.loads(_read_text(path, encoding="utf-8")) ex = cfg.get("excludes") if isinstance(cfg, dict) else None if isinstance(ex, list): return [str(x) for x in ex if isinstance(x, str) and x] @@ -124,8 +137,8 @@ def _read_build_gitignore(out_dir: Path) -> bool: """Return whether rebuilds should honor VCS ignore files (default True).""" try: path = out_dir / _BUILD_CONFIG_FILENAME - if path.is_file(): - cfg = json.loads(path.read_text(encoding="utf-8")) + if _path_is_file(path): + cfg = json.loads(_read_text(path, encoding="utf-8")) if isinstance(cfg, dict) and isinstance(cfg.get("gitignore"), bool): return cfg["gitignore"] except (OSError, json.JSONDecodeError): @@ -175,12 +188,12 @@ def _rebuild_lock(out_dir: Path, *, blocking: bool = False): yield True return - out_dir.mkdir(parents=True, exist_ok=True) + _make_dirs(out_dir, exist_ok=True) lock_path = out_dir / ".rebuild.lock" # "a+" creates the file if missing without truncating an existing holder's # PID payload — important because another process may have already written # its PID before we attempt the flock. - fh = open(lock_path, "a+", encoding="utf-8") + fh = open(_io_path(lock_path), "a+", encoding="utf-8") acquired = False try: flags = fcntl.LOCK_EX if blocking else (fcntl.LOCK_EX | fcntl.LOCK_NB) @@ -211,7 +224,7 @@ def _rebuild_lock(out_dir: Path, *, blocking: bool = False): # unlinks; a non-acquiring caller leaves the existing lock in place. if acquired: with contextlib.suppress(OSError): - lock_path.unlink() + _unlink_path(lock_path) def _apply_resource_limits() -> None: @@ -288,14 +301,14 @@ def _changed_path_candidates(raw: Path, *, change_root: Path, watch_root: Path) """ if raw.is_absolute(): lexical = Path(os.path.abspath(raw)) - resolved = raw.resolve() + resolved = _resolve_path(raw) return [lexical] if lexical == resolved else [lexical, resolved] candidates: list[Path] = [] seen: set[str] = set() for base in (change_root, watch_root): lexical = Path(os.path.abspath(base / raw)) - for cand in (lexical, lexical.resolve()): + for cand in (lexical, _resolve_path(lexical)): key = os.fspath(cand) if key in seen: continue @@ -314,7 +327,7 @@ def _relativize_source_files(payload: dict, root: Path, *, scope: Path | None = if not source_path.is_absolute(): continue try: - resolved = source_path.resolve() + resolved = _resolve_path(source_path) if scope is not None and not _is_relative_to(resolved, scope): continue item["source_file"] = resolved.relative_to(root).as_posix() @@ -356,14 +369,14 @@ def __init__( relative_marker_prefix: str | None = None root_marker = out / ".graphify_root" - if root_marker.exists(): + if _path_exists(root_marker): try: - saved_root = Path(root_marker.read_text(encoding="utf-8").strip()) + saved_root = Path(_read_text(root_marker, encoding="utf-8").strip()) if saved_root.is_absolute(): - self.existing_source_root = saved_root.resolve() + self.existing_source_root = _resolve_path(saved_root) else: - invocation_root = Path.cwd().resolve() - if (invocation_root / saved_root).resolve() == watch_root: + invocation_root = _resolve_path(Path.cwd()) + if _resolve_path(invocation_root / saved_root) == watch_root: self.existing_source_root = invocation_root relative_marker_prefix = posixpath.normpath(saved_root.as_posix()) except (OSError, ValueError): @@ -460,7 +473,7 @@ def _reconcile_existing_graph( ) -> tuple[dict, dict]: """Merge fresh extraction with preserved graph entries and evict stale sources.""" existing_graph_data: dict = {} - if not existing_graph.exists(): + if not _path_exists(existing_graph): return result, existing_graph_data # Fail-closed load (#2251): reuse build._load_existing_graph, which raises @@ -478,7 +491,7 @@ def _reconcile_existing_graph( # topology compare) the (nodes, edges, hyperedges) tuple does not carry. # A failure here (e.g. a race rewriting the file) still propagates, # staying fail-closed. - existing = json.loads(existing_graph.read_text(encoding="utf-8")) + existing = json.loads(_read_text(existing_graph, encoding="utf-8")) existing_graph_data = existing try: @@ -539,7 +552,7 @@ def _reconcile_existing_graph( if identity: alive = _alive_cache.get(identity) if alive is None: - alive = Path(identity).exists() + alive = _path_exists(identity) _alive_cache[identity] = alive if not alive: normalized = source_paths.normalize(source_file) @@ -553,7 +566,7 @@ def _reconcile_existing_graph( if identity: alive = _alive_cache.get(identity) if alive is None: - alive = Path(identity).exists() + alive = _path_exists(identity) _alive_cache[identity] = alive if alive: excluded_alive_files.add(identity) @@ -808,7 +821,7 @@ def _accounted(n: dict) -> bool: if all(_accounted(n) for n in lost): return True if tmp is not None: - tmp.unlink(missing_ok=True) + _unlink_path(tmp, missing_ok=True) print( f"[graphify] WARNING: new graph has {len(new_nodes)} nodes but existing " f"graph.json has {len(existing_nodes)}. Refusing to overwrite — you may be " @@ -840,7 +853,7 @@ def _stabilize_rebuild_cwd(watch_path: Path) -> bool: return True repo_root = os.environ.get("GRAPHIFY_REPO_ROOT", "").strip() - if repo_root and Path(repo_root).is_dir(): + if repo_root and _path_is_dir(repo_root): try: os.chdir(repo_root) return True @@ -907,7 +920,7 @@ def _rebuild_code( with _rebuild_lock(out, blocking=block_on_lock) as got: if not got: print("[graphify watch] Rebuild already in progress for " - f"{watch_path.resolve()} - changes queued.") + f"{_resolve_path(watch_path)} - changes queued.") return False # Lock acquired. Drain anything queued by earlier contenders # (including, importantly, the paths we just queued ourselves) @@ -946,8 +959,8 @@ def _rebuild_code( ) and ok return ok - watch_root = watch_path.resolve() - project_root = Path.cwd().resolve() if not watch_path.is_absolute() else watch_root + watch_root = _resolve_path(watch_path) + project_root = _resolve_path(Path.cwd()) if not watch_path.is_absolute() else watch_root report_root = _report_root_label(watch_path) try: from graphify.extract import extract, _get_extractor @@ -979,7 +992,7 @@ def _rebuild_code( ast_doc_files.append(p) existing_graph = out / "graph.json" - if not code_files and not existing_graph.exists(): + if not code_files and not _path_exists(existing_graph): print("[graphify watch] No code files found - nothing to rebuild.") return False @@ -996,10 +1009,10 @@ def _rebuild_code( # graph must be allowed to self-heal on a full rebuild without the # shrink-guard refusing the smaller write. semantic_doc_files: set[Path] = set() - if ast_doc_files and existing_graph.exists(): + if ast_doc_files and _path_exists(existing_graph): try: check_graph_file_size_cap(existing_graph) - prior = json.loads(existing_graph.read_text(encoding="utf-8")) + prior = json.loads(_read_text(existing_graph, encoding="utf-8")) prior_paths = _StoredSourcePaths( prior, out=out, @@ -1060,14 +1073,17 @@ def _add_deleted_source(path: Path) -> None: # an incremental rebuild, or their semantic nodes would be wiped. semantic_doc_set = {Path(os.path.abspath(p)) for p in semantic_doc_files} wanted: list[Path] = [] - change_root = Path.cwd().resolve() + change_root = _resolve_path(Path.cwd()) for raw in changed_paths: candidates = _changed_path_candidates( raw, change_root=change_root, watch_root=watch_root, ) - tracked = next((cand for cand in candidates if cand.exists() and cand in code_set), None) + tracked = next( + (cand for cand in candidates if _path_exists(cand) and cand in code_set), + None, + ) if tracked is not None: if tracked not in wanted and tracked not in semantic_doc_set: wanted.append(tracked) @@ -1076,7 +1092,7 @@ def _add_deleted_source(path: Path) -> None: existing_in_root = next( ( cand for cand in candidates - if cand.exists() and _is_relative_to(cand, watch_root) + if _path_exists(cand) and _is_relative_to(cand, watch_root) ), None, ) @@ -1166,7 +1182,7 @@ def _add_deleted_source(path: Path) -> None: else: rebuilt_sources = {(_nsf(str(p), _rebuilt_root) or str(p)) for p in extract_targets} rebuilt_sources |= set(deleted_paths) - out.mkdir(exist_ok=True) + _make_dirs(out, exist_ok=True) if no_cluster: # Normalise to "links" key so schema is consistent with the full clustered path. @@ -1181,10 +1197,10 @@ def _add_deleted_source(path: Path) -> None: } candidate_graph_text = _json_text(candidate_graph_data) same_graph = False - if existing_graph.exists(): + if _path_exists(existing_graph): try: check_graph_file_size_cap(existing_graph) - existing_payload = json.loads(existing_graph.read_text(encoding="utf-8")) + existing_payload = json.loads(_read_text(existing_graph, encoding="utf-8")) except Exception as exc: # A load failure is NOT "graph changed" (#2251): refuse to # overwrite a graph we merely failed to read. Normally @@ -1216,12 +1232,12 @@ def _add_deleted_source(path: Path) -> None: # Atomic replace via tmp file, matching the clustered path: a # crash mid-write must not leave a truncated graph.json. graph_tmp = out / ".graph.tmp.json" - graph_tmp.write_text(candidate_graph_text, encoding="utf-8") - graph_tmp.replace(existing_graph) + _write_text(graph_tmp, candidate_graph_text, encoding="utf-8") + _replace_path(graph_tmp, existing_graph) # Write the user-supplied path only after the candidate graph is # accepted, so a refused shrink cannot mismatch graph and marker. - (out / ".graphify_root").write_text(str(watch_path), encoding="utf-8") + _write_text(out / ".graphify_root", str(watch_path), encoding="utf-8") try: from graphify.detect import save_manifest @@ -1238,8 +1254,8 @@ def _add_deleted_source(path: Path) -> None: # clear stale needs_update flag if present flag = out / "needs_update" - if flag.exists(): - flag.unlink() + if _path_exists(flag): + _unlink_path(flag) if same_graph: print("[graphify watch] No code-graph changes detected (--no-cluster); outputs left untouched.") @@ -1279,8 +1295,8 @@ def _add_deleted_source(path: Path) -> None: except Exception: pass flag = out / "needs_update" - if flag.exists(): - flag.unlink() + if _path_exists(flag): + _unlink_path(flag) print("[graphify watch] No code-graph topology changes detected; outputs left untouched.") return True @@ -1294,7 +1310,11 @@ def _add_deleted_source(path: Path) -> None: labels_file = out / ".graphify_labels.json" sig_file = out / (".graphify_labels.json" + ".sig") try: - raw = json.loads(labels_file.read_text(encoding="utf-8")) if labels_file.exists() else {} + raw = ( + json.loads(_read_text(labels_file, encoding="utf-8")) + if _path_exists(labels_file) + else {} + ) # Skip persisted "Community N" placeholders so the hub-fill below # replaces them instead of perpetuating them on every rebuild (#2073). labels = { @@ -1315,11 +1335,11 @@ def _add_deleted_source(path: Path) -> None: from graphify.cluster import community_member_sigs cur_sigs = community_member_sigs(communities) saved_sigs: dict[int, str] = {} - if sig_file.exists(): + if _path_exists(sig_file): try: saved_sigs = { int(k): v for k, v in - json.loads(sig_file.read_text(encoding="utf-8")).items() + json.loads(_read_text(sig_file, encoding="utf-8")).items() if isinstance(v, str) } except Exception: @@ -1359,19 +1379,19 @@ def _add_deleted_source(path: Path) -> None: json_written = to_json(G, communities, str(graph_tmp), force=True, built_at_commit=commit, community_labels=labels) if not json_written: return False - candidate_graph_data = json.loads(graph_tmp.read_text(encoding="utf-8")) + candidate_graph_data = json.loads(_read_text(graph_tmp, encoding="utf-8")) same_graph = False same_report = False - if existing_graph.exists(): + if _path_exists(existing_graph): try: check_graph_file_size_cap(existing_graph) - existing_payload = json.loads(existing_graph.read_text(encoding="utf-8")) + existing_payload = json.loads(_read_text(existing_graph, encoding="utf-8")) except Exception as exc: # A load failure is NOT "graph changed" (#2251): refuse to # overwrite a graph we merely failed to read. Normally # unreachable — the reconcile load above already failed # closed — but a race rewriting the file can land here. - graph_tmp.unlink(missing_ok=True) + _unlink_path(graph_tmp, missing_ok=True) print( f"error: Cannot read {existing_graph}: {exc}. " "Refusing to overwrite; delete the file and run a " @@ -1386,12 +1406,12 @@ def _add_deleted_source(path: Path) -> None: ) except Exception: same_graph = False - if report_path.exists(): - old_report = report_path.read_text(encoding="utf-8") + if _path_exists(report_path): + old_report = _read_text(report_path, encoding="utf-8") same_report = _report_for_compare(old_report) == _report_for_compare(report) no_change = same_graph and same_report if no_change: - graph_tmp.unlink(missing_ok=True) + _unlink_path(graph_tmp, missing_ok=True) print("[graphify watch] No code-graph changes detected; graph.json/GRAPH_REPORT.md left untouched.") else: if not _check_shrink( @@ -1403,17 +1423,20 @@ def _add_deleted_source(path: Path) -> None: return False from graphify.export import backup_if_protected as _backup _backup(out) - graph_tmp.replace(existing_graph) - report_path.write_text(report, encoding="utf-8") - labels_file.write_text(labels_json, encoding="utf-8") + _replace_path(graph_tmp, existing_graph) + _write_text(report_path, report, encoding="utf-8") + _write_text(labels_file, labels_json, encoding="utf-8") # Keep the membership signatures in step with the labels we just wrote. # Skipping this was the other half of the stale-label bug: labels.json # advanced every rebuild while the sidecar kept describing an older # clustering, so the guard above had nothing accurate to check against. - sig_file.write_text( - json.dumps({str(k): v for k, v in cur_sigs.items()}), encoding="utf-8") + _write_text( + sig_file, + json.dumps({str(k): v for k, v in cur_sigs.items()}), + encoding="utf-8", + ) - (out / ".graphify_root").write_text(str(watch_path), encoding="utf-8") + _write_text(out / ".graphify_root", str(watch_path), encoding="utf-8") try: from graphify.detect import save_manifest @@ -1442,8 +1465,8 @@ def _add_deleted_source(path: Path) -> None: # the community-aggregation view in exactly this case, so do # the same here: current AND present beats current OR present. from graphify.exporters.html import _viz_node_limit - if html_target.exists(): - html_target.unlink() + if _path_exists(html_target): + _unlink_path(html_target) limit = _viz_node_limit() if limit <= 0: # GRAPHIFY_VIZ_NODE_LIMIT=0 means "no HTML viz" (CI runners), @@ -1455,14 +1478,14 @@ def _add_deleted_source(path: Path) -> None: community_labels=labels or None, node_limit=limit) # The aggregator declines to write a single-community # graph, so trust the file rather than the call. - html_written = html_target.exists() + html_written = _path_exists(html_target) except Exception as fallback_err: print(f"[graphify watch] Skipped graph.html: {viz_err} " f"(aggregated view also failed: {fallback_err})") # Regenerate callflow HTML if the user previously generated one — # opt-in by existence so users who never ran callflow-html aren't affected. - callflow_files = list(out.glob("*-callflow.html")) + callflow_files = list(_glob_paths(out, "*-callflow.html")) if callflow_files and not no_change: try: from graphify.callflow_html import write_callflow_html @@ -1479,8 +1502,8 @@ def _add_deleted_source(path: Path) -> None: # clear stale needs_update flag if present flag = out / "needs_update" - if flag.exists(): - flag.unlink() + if _path_exists(flag): + _unlink_path(flag) if not no_change: print(f"[graphify watch] Rebuilt: {G.number_of_nodes()} nodes, " @@ -1505,7 +1528,7 @@ def check_update(watch_path: Path) -> bool: that the update is needed. """ flag = Path(watch_path) / _GRAPHIFY_OUT / "needs_update" - if flag.exists(): + if _path_exists(flag): print(f"[graphify check-update] Pending non-code changes in {watch_path}.") print("[graphify check-update] Run `/graphify --update` to apply semantic re-extraction.") return True @@ -1514,8 +1537,8 @@ def check_update(watch_path: Path) -> bool: def _notify_only(watch_path: Path) -> None: """Write a flag file and print a notification (fallback for non-code-only corpora).""" flag = watch_path / _GRAPHIFY_OUT / "needs_update" - flag.parent.mkdir(parents=True, exist_ok=True) - flag.write_text("1", encoding="utf-8") + _make_dirs(flag.parent, exist_ok=True) + _write_text(flag, "1", encoding="utf-8") print(f"\n[graphify watch] New or changed files detected in {watch_path}") print("[graphify watch] Non-code files changed - semantic re-extraction requires LLM.") print("[graphify watch] Run `/graphify --update` in Claude Code to update the graph.") @@ -1554,7 +1577,7 @@ def watch(watch_path: Path, debounce: float = 3.0) -> None: # (Time Machine writes, Docker/Colima VM I/O, Spotlight indexing, …) — # without this short-circuit a busy volume can saturate a CPU core # discarding events one extension at a time. (gh-928) - watch_root_for_ignore = watch_path.resolve() + watch_root_for_ignore = _resolve_path(watch_path) ignore_patterns = _load_graphifyignore( watch_root_for_ignore, gitignore=_read_build_gitignore(watch_path / _GRAPHIFY_OUT), @@ -1593,7 +1616,7 @@ def on_any_event(self, event): observer.schedule(handler, str(watch_path), recursive=True) observer.start() - print(f"[graphify watch] Watching {watch_path.resolve()} - press Ctrl+C to stop") + print(f"[graphify watch] Watching {_resolve_path(watch_path)} - press Ctrl+C to stop") print(f"[graphify watch] Code changes rebuild graph automatically. " f"Doc/image changes require /graphify --update.") print(f"[graphify watch] Debounce: {debounce}s") diff --git a/tests/test_atomic_writes.py b/tests/test_atomic_writes.py index 968cb0b96..ef5986bc6 100644 --- a/tests/test_atomic_writes.py +++ b/tests/test_atomic_writes.py @@ -6,6 +6,7 @@ """ import json import os +import stat import pytest @@ -41,18 +42,26 @@ def test_write_text_atomic_preserves_existing_mode(tmp_path): p = tmp_path / "graph.json" p.write_text("{}", encoding="utf-8") os.chmod(p, 0o644) + # Windows exposes a synthetic permission mask (typically 0666 for a + # writable file) rather than preserving POSIX owner/group distinctions. + # Compare with the mode the platform actually reports after chmod instead + # of assuming that every OS can represent 0644 exactly. + expected_mode = stat.S_IMODE(os.stat(p).st_mode) write_text_atomic(p, '{"x": 1}') - assert (os.stat(p).st_mode & 0o777) == 0o644 + assert stat.S_IMODE(os.stat(p).st_mode) == expected_mode -def test_write_text_atomic_new_file_respects_umask(tmp_path): - # A brand-new file must land at the umask default (e.g. 0644), NOT mkstemp's - # 0600 — otherwise every fresh graph.json would be owner-only. +def test_write_text_atomic_new_file_matches_platform_default_mode(tmp_path): + # A brand-new atomic file should have the same mode as an ordinary new file + # on this platform, NOT mkstemp's owner-only default. This also avoids + # assuming Windows can represent a POSIX umask-derived mode exactly. + reference = tmp_path / "reference.json" + reference.write_text("{}", encoding="utf-8") + expected_mode = stat.S_IMODE(os.stat(reference).st_mode) + p = tmp_path / "new.json" write_text_atomic(p, "{}") - umask = os.umask(0) - os.umask(umask) - assert (os.stat(p).st_mode & 0o777) == (0o666 & ~umask) + assert stat.S_IMODE(os.stat(p).st_mode) == expected_mode def test_write_text_atomic_writes_through_symlink(tmp_path): @@ -61,7 +70,15 @@ def test_write_text_atomic_writes_through_symlink(tmp_path): target = tmp_path / "real.json" target.write_text("old", encoding="utf-8") link = tmp_path / "link.json" - link.symlink_to(target) + try: + link.symlink_to(target) + except OSError as exc: + if os.name == "nt" and getattr(exc, "winerror", None) == 1314: + pytest.skip( + "Windows symlink creation requires Developer Mode, " + "administrator rights, or SeCreateSymbolicLinkPrivilege" + ) + raise write_text_atomic(link, "new") assert link.is_symlink() assert target.read_text() == "new" diff --git a/tests/test_cpp_preprocess.py b/tests/test_cpp_preprocess.py index 75ca8de5f..a6026576c 100644 --- a/tests/test_cpp_preprocess.py +++ b/tests/test_cpp_preprocess.py @@ -4,6 +4,8 @@ terminator, so _cpp_preprocess passes an absolute path which can never be parsed as a cpp option. """ +from pathlib import Path + from graphify import extract @@ -28,5 +30,8 @@ class _Result: out = extract._cpp_preprocess(f) assert out == b"preprocessed" last_arg = captured["argv"][-1] - assert last_arg.startswith("/"), f"path arg must be absolute, got {last_arg!r}" + assert Path(last_arg).is_absolute(), f"path arg must be absolute, got {last_arg!r}" assert not last_arg.startswith("-"), "path arg must never look like an option" + assert not last_arg.startswith("\\\\?\\"), ( + "transport-only Windows extended prefix must not cross into cpp" + ) diff --git a/tests/test_long_path_hashing.py b/tests/test_long_path_hashing.py index 8e4b3bd1a..f9fa13806 100644 --- a/tests/test_long_path_hashing.py +++ b/tests/test_long_path_hashing.py @@ -1,50 +1,49 @@ r"""#1655 — files whose absolute path exceeds Windows MAX_PATH (260) must still -be hashed, or their manifest entry never stabilizes and detect_incremental +be hashed, or their manifest entry never stabilizes and ``detect_incremental`` re-flags them as changed on every run. The plain file APIs reject long paths on win32 unless prefixed with the -extended-length marker `\\?\`. _os_path adds it (for I/O), the mirror of -cache._normalize_path which strips it (for stable keys). +extended-length marker ``\\?\``. :func:`graphify.paths.io_path` adds it for I/O, +while cache keys and manifests retain ordinary paths. """ from __future__ import annotations from pathlib import Path from graphify import detect +from graphify.paths import io_path def test_os_path_noop_on_posix(monkeypatch): monkeypatch.setattr("sys.platform", "linux") - p = Path("/home/user/deep/file.py") - assert detect._os_path(p) == str(p) + path = Path("/home/user/deep/file.py") + assert io_path(path) == str(path) def test_os_path_adds_prefix_on_win32(monkeypatch): monkeypatch.setattr("sys.platform", "win32") - # os.path.abspath is posix here, so exercise the already-qualified branch: - # a value that abspath leaves intact still gets the prefix. - out = detect._os_path(Path("/already/abs/file.py")) - assert out.startswith("\\\\?\\") + assert io_path(Path(r"C:\already\abs\file.py")) == r"\\?\C:\already\abs\file.py" def test_os_path_idempotent_on_win32(monkeypatch): monkeypatch.setattr("sys.platform", "win32") - already = "\\\\?\\C:\\a\\file.py" - assert detect._os_path(Path(already)) == already + already = r"\\?\C:\a\file.py" + assert io_path(Path(already)) == already def test_hashing_still_works_and_stabilizes(tmp_path): - # End-to-end (posix): a hashed file must produce a stable, non-empty hash so - # its manifest entry doesn't churn. Guards against the _os_path indirection - # breaking normal hashing. - f = tmp_path / "deep" / "nested" / "module.py" - f.parent.mkdir(parents=True) - f.write_text("def x():\n return 1\n") - h1 = detect._md5_file(f) - h2 = detect._md5_file(f) - assert h1 and h1 == h2 - - got = detect._stat_and_hash(str(f)) - assert got is not None - assert got[0] == str(f) - assert got[2] == h1 + # End-to-end (POSIX): a hashed file must produce a stable, non-empty hash so + # its manifest entry does not churn. This guards against the I/O adapter + # breaking ordinary Linux/macOS hashing while fixing Windows long paths. + source = tmp_path / "deep" / "nested" / "module.py" + source.parent.mkdir(parents=True) + source.write_text("def x():\n return 1\n", encoding="utf-8") + + first = detect._md5_file(source) + second = detect._md5_file(source) + assert first and first == second + + stat_and_hash = detect._stat_and_hash(str(source)) + assert stat_and_hash is not None + assert stat_and_hash[0] == str(source) + assert stat_and_hash[2] == first diff --git a/tests/test_windows_long_paths.py b/tests/test_windows_long_paths.py new file mode 100644 index 000000000..698716a42 --- /dev/null +++ b/tests/test_windows_long_paths.py @@ -0,0 +1,239 @@ +r"""Regression coverage for Windows extended-length path handling. + +Graphify keeps normal drive/UNC spellings as its public path identity and uses +``\\?\`` only when crossing an operating-system I/O boundary. These tests run +on every host; the deep-path integrations exercise native long paths on Windows +and verify that Linux/macOS remain unchanged. +""" +from __future__ import annotations + +import json +import shutil +import sys +from pathlib import Path + +import pytest + +from graphify.cache import file_hash +from graphify.detect import FileType, detect, load_manifest, save_manifest +from graphify.extract import collect_files, extract +from graphify.extractors.markdown import extract_markdown +from graphify.paths import ( + glob_paths, + io_path, + iterdir_path, + logical_path, + make_dirs, + path_exists, + path_is_file, + path_stat, + read_bytes, + read_text, + walk_path, + write_text, +) + + +def _deep_parent(root: Path, filename: str, *, minimum_length: int = 320) -> Path: + """Return a parent whose ordinary child path exceeds ``minimum_length``.""" + parent = root + index = 0 + while len(str(parent / filename)) <= minimum_length: + parent /= f"level_{index:02d}_{'x' * 28}" + index += 1 + return parent + + +def _remove_tree(path: Path) -> None: + """Clean up a deep Windows tree without relying on pytest's plain path I/O.""" + shutil.rmtree(io_path(path), ignore_errors=True) + + +def test_io_path_is_a_noop_off_windows(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(sys, "platform", "linux") + value = "/tmp/a/deep/file.py" + assert io_path(value) == value + assert logical_path(value) == value + + +@pytest.mark.parametrize( + ("ordinary", "extended"), + [ + (r"C:\repo\src\module.py", r"\\?\C:\repo\src\module.py"), + ( + r"\\server\share\manuals\deep\file.pdf", + r"\\?\UNC\server\share\manuals\deep\file.pdf", + ), + ], +) +def test_io_path_converts_windows_drive_and_unc_paths( + monkeypatch: pytest.MonkeyPatch, + ordinary: str, + extended: str, +) -> None: + monkeypatch.setattr(sys, "platform", "win32") + assert io_path(ordinary) == extended + assert logical_path(extended) == ordinary + assert io_path(extended) == extended + + +def test_io_path_normalizes_before_prefixing(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(sys, "platform", "win32") + assert io_path(r"C:\repo\one\..\two/file.py") == r"\\?\C:\repo\two\file.py" + + +def test_walk_path_uses_extended_unc_and_yields_logical_paths( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(sys, "platform", "win32") + root = r"\\server\share\root" + extended = r"\\?\UNC\server\share\root" + called_with: list[str] = [] + + def fake_walk(top, *, topdown, followlinks, onerror): + called_with.append(top) + assert topdown is True + assert followlinks is False + assert onerror is not None + yield top, ["deep"], ["root.py"] + yield top + r"\deep", [], ["nested.py"] + + monkeypatch.setattr("graphify.paths.os.walk", fake_walk) + rows = list(walk_path(root, onerror=lambda _error: None)) + + assert called_with == [extended] + assert rows == [ + (root, ["deep"], ["root.py"]), + (root + r"\deep", [], ["nested.py"]), + ] + + +def test_walk_path_preserves_relative_windows_spelling( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(sys, "platform", "win32") + root = r"relative\root" + + def fake_walk(top, *, topdown, followlinks, onerror): + del topdown, followlinks, onerror + yield top, ["deep"], ["root.py"] + yield top + r"\deep", [], ["nested.py"] + + monkeypatch.setattr("graphify.paths.os.walk", fake_walk) + rows = list(walk_path(root)) + + assert rows == [ + (root, ["deep"], ["root.py"]), + (root + r"\deep", [], ["nested.py"]), + ] + + +def test_walk_path_removes_prefix_from_reported_errors( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(sys, "platform", "win32") + root = r"\\server\share\root" + reported: list[OSError] = [] + + def fake_walk(top, *, topdown, followlinks, onerror): + del topdown, followlinks + assert onerror is not None + onerror(FileNotFoundError(3, "not found", top + r"\too\deep")) + return [] + + monkeypatch.setattr("graphify.paths.os.walk", fake_walk) + assert list(walk_path(root, onerror=reported.append)) == [] + assert len(reported) == 1 + assert reported[0].filename == root + r"\too\deep" + + +def test_deep_path_discovery_hash_manifest_and_markdown_extraction(tmp_path: Path) -> None: + root = tmp_path / "corpus" + cache_root = tmp_path / "cache" + parent = _deep_parent(root, "notes.md") + source = parent / "notes.md" + manifest_path = parent / "graphify-out" / "manifest.json" + + try: + make_dirs(parent, exist_ok=True) + write_text(source, "# Deep manual\n\nLong-path content.\n", encoding="utf-8") + assert len(str(source)) > 300 + assert path_exists(source) + assert path_is_file(source) + assert path_stat(source).st_size > 0 + assert source in set(iterdir_path(parent)) + assert source in set(glob_paths(parent, "*.md")) + assert source in set(glob_paths(root, "**/*.md")) + assert read_bytes(source, limit=13) == b"# Deep manual" + + result = detect(root, cache_root=cache_root) + assert result["walk_errors"] == [] + assert str(source) in result["files"][FileType.DOCUMENT] + assert source in collect_files(root, root=root) + + first_hash = file_hash(source, root, cache_root=cache_root) + second_hash = file_hash(source, root, cache_root=cache_root) + assert first_hash and first_hash == second_hash + + save_manifest( + {FileType.DOCUMENT: [str(source)]}, + str(manifest_path), + root=root, + ) + loaded = load_manifest(str(manifest_path), root=root) + assert str(source) in loaded + assert "\\\\?\\" not in read_text(manifest_path, encoding="utf-8") + for cache_file in glob_paths(cache_root, "**/*.json"): + assert "\\\\?\\" not in read_text(cache_file, encoding="utf-8") + + extracted = extract_markdown(source) + assert any(node.get("label") == "Deep manual" for node in extracted["nodes"]) + + serialized = json.dumps({"detect": result, "extract": extracted}, default=str) + assert "\\\\?\\" not in serialized + finally: + _remove_tree(root) + + +def test_deep_python_path_full_extraction(tmp_path: Path) -> None: + pytest.importorskip("tree_sitter") + pytest.importorskip("tree_sitter_python") + + root = tmp_path / "corpus" + cache_root = tmp_path / "cache" + parent = _deep_parent(root, "deep_module.py") + source = parent / "deep_module.py" + + try: + make_dirs(parent, exist_ok=True) + write_text( + source, + "class DeepThing:\n" + " def answer(self):\n" + " return 42\n", + encoding="utf-8", + ) + + first = extract( + [source], + root=root, + cache_root=cache_root, + parallel=False, + ) + # Exercise the warm-cache path as well as first-pass parsing. The 0.9.30 + # cache portability code resolves both the corpus root and source path; + # those lookups must cross the same Windows I/O boundary. + second = extract( + [source], + root=root, + cache_root=cache_root, + parallel=False, + ) + + for result in (first, second): + assert any(node.get("label") == "DeepThing" for node in result["nodes"]) + assert "\\\\?\\" not in json.dumps(result, default=str) + for cache_file in glob_paths(cache_root, "**/*.json"): + assert "\\\\?\\" not in read_text(cache_file, encoding="utf-8") + finally: + _remove_tree(root) From 5c8ee1fe5d9a1f99417fb0a89b7682f85524a96c Mon Sep 17 00:00:00 2001 From: Nicholas M Denny Date: Sat, 1 Aug 2026 14:36:34 -0500 Subject: [PATCH 2/5] Refactor output handling and add coverage detection [fix to merge with 0932] Removed GRAPHIFY_OUT_NAME constant and updated related logic to prevent treating the default output directory as source input. Added coverage artifact detection functionality. --- graphify/detect.py | 63 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 60 insertions(+), 3 deletions(-) diff --git a/graphify/detect.py b/graphify/detect.py index 2e3a1984b..e17788188 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -17,7 +17,6 @@ ) from graphify.paths import ( GRAPHIFY_OUT, - GRAPHIFY_OUT_NAME, io_path as _os_path, make_dirs as _make_dirs, out_path, @@ -812,9 +811,11 @@ def count_words(path: Path) -> int: "site-packages", "lib64", ".pytest_cache", ".mypy_cache", ".ruff_cache", ".tox", ".nox", ".eggs", "*.egg-info", # nox is tox's successor, same .nox/ venv shape (#1804) - "graphify-out", GRAPHIFY_OUT_NAME, # never treat own output as source input (#524); honour GRAPHIFY_OUT (#1423) + "graphify-out", # never treat the default output as source input (#524) # Coverage/test-artefact dirs — generated, never architecturally meaningful - "coverage", "lcov-report", # Vitest/Istanbul/nyc HTML reports (#870) + "lcov-report", # Vitest/Istanbul/nyc HTML reports (#870); + # bare "coverage" is gated on report + # artefacts below (#2339) "visual-tests", "visual-test", # Playwright/visual-regression bundles (#869) "__snapshots__", # Jest/Vitest snapshot dir (unambiguous) "storybook-static", # Storybook production build output @@ -843,6 +844,39 @@ def count_words(path: Path) -> int: # unconditionally pruned above; only the ambiguous bare name is gated here. _JS_SNAPSHOT_TEST_ROOTS = frozenset({"__tests__", "__test__"}) +# Files a coverage tool writes into its own output dir. Any one of them is proof +# the directory is generated: lcov (lcov.info), nyc/Istanbul (coverage-final.json, +# clover.xml, the lcov-report/ subtree), coverage.py (coverage.xml, .coverage), +# JaCoCo/Cobertura (jacoco.xml, cobertura-coverage.xml). +_COVERAGE_ARTIFACT_FILES = frozenset({ + "lcov.info", "coverage-final.json", "coverage-summary.json", + "clover.xml", "coverage.xml", "cobertura-coverage.xml", "jacoco.xml", + ".coverage", "index.html", +}) +_COVERAGE_ARTIFACT_DIRS = frozenset({"lcov-report", "html-report"}) + + +def _has_coverage_artifacts(d: "Path") -> bool: + """True only when *d* holds files a coverage tool actually generated. + + ``coverage`` is a legitimate package name (a Python package, a Go/Rust module, + a domain namespace), so pruning it by name alone silently drops real source — + an entire 5-module package in #2339, with its dependents left in the graph so + queries still returned plausible neighbours. Prune it only on real evidence, + mirroring the ``snapshots``/``env`` gating (#1666/#2058): a coverage report + file, or an Istanbul/lcov HTML report subtree. + """ + try: + for name in _COVERAGE_ARTIFACT_FILES: + if _path_is_file(d / name): + return True + for name in _COVERAGE_ARTIFACT_DIRS: + if _path_is_dir(d / name): + return True + except OSError: + pass + return False + def _has_venv_markers(d: "Path") -> bool: """True only when *d* has actual virtualenv/conda structure on disk. @@ -883,6 +917,12 @@ def _is_noise_dir(part: str, parent: "Path | None" = None) -> bool: if parent is None: return False # cannot verify; keep a possibly-real code dir return _has_venv_markers(parent / part) + if part == "coverage": + # Ambiguous: a generated report dir OR a real package named coverage. + # Prune only on actual coverage-artefact evidence (#2339). + if parent is None: + return False # cannot verify; keep a possibly-real code dir + return _has_coverage_artifacts(parent / part) if part == "snapshots": # Prune only when it looks like an actual JS/Vitest snapshot dir. if parent is None: @@ -1209,6 +1249,13 @@ def _resolves_under_root(path: Path, root: Path) -> bool: def detect(root: Path, *, follow_symlinks: bool | None = None, google_workspace: bool | None = None, extra_excludes: list[str] | None = None, cache_root: Path | None = None, gitignore: bool = True) -> dict: root = _resolve_path(root) + configured_out_dir = root / GRAPHIFY_OUT + configured_out_names = {configured_out_dir.name} + try: + configured_out_dir = _resolve_path(configured_out_dir) + except (OSError, RuntimeError): + configured_out_dir = configured_out_dir.absolute() + configured_out_names.add(configured_out_dir.name) # .graphifyinclude support was removed (#2112): its loader and matchers had # no consumers, so the file has been a silent no-op since dot directories # became indexed by default (#873). Surface that once per scan so a @@ -1328,6 +1375,16 @@ def _on_walk_error(err: OSError) -> None: # repos for no correctness gain. kept_dirs: list[str] = [] for d in dirnames: + child = dp / d + is_configured_out = False + if d in configured_out_names: + try: + is_configured_out = _resolve_path(child) == configured_out_dir + except (OSError, RuntimeError): + pass + if is_configured_out: + pruned_noise.append(str(child) + os.sep) + continue if _is_noise_dir(d, dp): # Record pruned-as-noise dirs so a wrongly-pruned real # source dir is at least traceable in the output rather From 68246dda69408c050ea721177681da65aae27f93 Mon Sep 17 00:00:00 2001 From: Nicholas M Denny Date: Sat, 1 Aug 2026 14:37:02 -0500 Subject: [PATCH 3/5] Refactor _git_head and enhance graph extraction logic [fix to merge with 0932] Refactor _git_head function to accept cwd parameter for better repo context. Update comments and logic to improve clarity on graph extraction and node preservation. --- graphify/watch.py | 101 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 77 insertions(+), 24 deletions(-) diff --git a/graphify/watch.py b/graphify/watch.py index fb3b55891..5531ec571 100644 --- a/graphify/watch.py +++ b/graphify/watch.py @@ -255,11 +255,21 @@ def _apply_resource_limits() -> None: pass -def _git_head() -> str | None: - """Return current git HEAD commit hash, or None outside a repo.""" +def _git_head(cwd: Path | str | None = None) -> str | None: + """Return current git HEAD commit hash, or None outside a repo. + + ``cwd`` selects the repository to ask (#2316). Without it the command + inherits the caller's working directory, so `graphify update ` + stamped the *invoking* repo's commit into the target's graph.json — the + same CWD-anchoring mistake as the manifest path, but writing wrong + provenance rather than a misplaced file. + """ import subprocess as _sp try: - r = _sp.run(["git", "rev-parse", "HEAD"], capture_output=True, text=True, timeout=3) + r = _sp.run( + ["git", "rev-parse", "HEAD"], capture_output=True, text=True, timeout=3, + cwd=str(cwd) if cwd is not None else None, + ) return r.stdout.strip() if r.returncode == 0 else None except Exception: return None @@ -494,6 +504,16 @@ def _reconcile_existing_graph( existing = json.loads(_read_text(existing_graph, encoding="utf-8")) existing_graph_data = existing + # Backfill tier provenance on legacy items (#2334), mirroring + # build._load_existing_graph (this reconcile path loads the raw dict + # separately, so the backfill there does not reach it). Stamping preserved + # items means the graph self-heals on this write. + from graphify.build import _is_ast_tier + for _bucket in ("nodes", "links", "edges"): + for _item in existing.get(_bucket, []): + if isinstance(_item, dict): + _item.setdefault("_origin", "ast" if _is_ast_tier(_item) else "semantic") + try: from graphify.build import _norm_source_file as _nsf from graphify.extract import _get_extractor @@ -587,15 +607,20 @@ def _reconcile_existing_graph( "Run a full re-extraction to purge them if the exclusion is intentional." ) - # A full re-extraction owns every AST node under watch_root. Incremental - # extraction owns only nodes from rebuilt or deleted sources. Semantic - # nodes lack the AST origin marker and remain preserved. + # A full re-extraction owns the AST nodes of every source it actually + # re-extracted (extract_targets, via rebuilt_source_identities) — NOT + # every AST node under watch_root: a semantic-backed doc is excluded + # from extract_targets (#1915), so its existing AST layer is not + # regenerated this run and dropping it would be data loss (#2333, + # COEXIST — the AST and semantic layers of a file coexist). + # Incremental extraction owns only nodes from rebuilt or deleted + # sources. Semantic-tier nodes (per _is_ast_tier) remain preserved. preserved_nodes = [ node for node in existing.get("nodes", []) if node["id"] not in new_ast_ids and not ( - node.get("_origin") == "ast" + _is_ast_tier(node) and ( ( not node.get("source_file") @@ -603,7 +628,7 @@ def _reconcile_existing_graph( ) or ( full_rebuild - and source_paths.in_watch_root(node.get("source_file")) + and source_paths.is_evicted(node, rebuilt_source_identities) ) ) ) @@ -624,7 +649,7 @@ def _reconcile_existing_graph( and edge.get("target") in all_ids and not source_paths.is_evicted(edge, edge_evicted_source_identities) and not ( - edge.get("_origin") == "ast" + _is_ast_tier(edge) and source_paths.is_evicted(edge, rebuilt_source_identities) ) ] @@ -688,6 +713,13 @@ def _node_community_map(graph_data: dict) -> dict[str, int]: def _canonical_graph_for_compare(graph_data: dict) -> dict: canonical = dict(graph_data) canonical.pop("built_at_commit", None) + # A missing "directed" key means the same thing as "directed": false + # everywhere else in the codebase (#2342's --no-cluster path only started + # writing the key once it began inheriting it from the existing graph). + # Normalise so an old graph.json without the key doesn't register as + # "changed" against a freshly-written candidate that now carries + # "directed": false explicitly. + canonical["directed"] = bool(canonical.get("directed", False)) for key in ("nodes", "links", "edges", "hyperedges"): if key in canonical and isinstance(canonical[key], list): canonical[key] = sorted( @@ -960,12 +992,16 @@ def _rebuild_code( return ok watch_root = _resolve_path(watch_path) + # project_root stays CWD-anchored for a relative invocation on purpose: the + # persisted graph rehomes source_file across invocation styles against it + # (tests/test_watch.py:1389, :1428). The manifest is a different artifact + # with a different anchor — see the save_manifest calls below. project_root = _resolve_path(Path.cwd()) if not watch_path.is_absolute() else watch_root report_root = _report_root_label(watch_path) try: from graphify.extract import extract, _get_extractor from graphify.detect import detect - from graphify.build import build_from_json, _norm_source_file as _nsf + from graphify.build import build_from_json, _is_ast_tier, _norm_source_file as _nsf from graphify.cluster import cluster, remap_communities_to_previous, score_all from graphify.analyze import god_nodes, surprising_connections, suggest_questions from graphify.report import generate @@ -1000,10 +1036,12 @@ def _rebuild_code( # existing graph must not ALSO be AST-quick-scanned — otherwise every # rebuild mints heading nodes on top of the preserved semantic nodes # and the doc is represented twice (~4x graph bloat vs the CLI update - # path, which AST-extracts only code). Semantic supersedes AST per doc - # source: the quick-scan stays as a fallback for docs with no semantic - # layer (the no-LLM doc-structure feature, #09b33b7) and for brand-new - # docs the graph has never seen. These docs stay in ``code_files`` so + # path, which AST-extracts only code). A semantic-backed doc is never + # re-quick-scanned, and any AST layer it already carries coexists and + # is preserved rather than regenerated (#2333, COEXIST); the + # quick-scan stays as a fallback for docs with no semantic layer (the + # no-LLM doc-structure feature, #09b33b7) and for brand-new docs the + # graph has never seen. These docs stay in ``code_files`` so # corpus membership (#1795 fail-closed deletion evidence) and the # shrink accounting below still cover them — a previously-bloated # graph must be allowed to self-heal on a full rebuild without the @@ -1036,7 +1074,11 @@ def _rebuild_code( # "document" nodes (extractors/markdown.py). "image" stays out. semantic_doc_identities: set[str] = set() for node in prior.get("nodes", []): - if node.get("_origin") == "ast": + # _is_ast_tier, not a raw _origin check (#2334): a legacy + # unstamped AST heading node (source_location "L") must + # not fake a semantic layer, or the doc would be excluded + # from the AST quick-scan forever. + if _is_ast_tier(node): continue if node.get("file_type") not in ( "document", "concept", "rationale", "paper", "code" @@ -1124,13 +1166,14 @@ def _add_deleted_source(path: Path) -> None: extract_targets = wanted else: # Full rebuild: skip the AST quick-scan for semantic-backed docs - # (#1915). They remain in code_files, so stale _origin=="ast" - # heading nodes from a previously-bloated graph are dropped by the - # full-rebuild AST ownership rule while the shrink accounting - # below still counts the doc as a rebuilt source. + # (#1915). They remain in code_files for corpus membership and + # shrink accounting, but because they are not extract targets the + # full-rebuild AST ownership rule (scoped to + # rebuilt_source_identities, #2333 COEXIST) leaves their existing + # AST heading layer intact alongside the semantic layer. extract_targets = [p for p in code_files if p not in semantic_doc_files] - commit = _git_head() + commit = _git_head(cwd=watch_root) result = extract(extract_targets, cache_root=watch_root) if extract_targets else { "nodes": [], "edges": [], "hyperedges": [], "input_tokens": 0, "output_tokens": 0, @@ -1194,6 +1237,10 @@ def _add_deleted_source(path: Path) -> None: **{k: v for k, v in result.items() if k not in ("edges", "nodes")}, "nodes": _dedupe_nodes(result.get("nodes", [])), "links": _dedupe_edges(result.get("edges", [])), + # Inherit the existing graph's directed flag (#2342) so + # `graphify update --no-cluster` can't silently drop it - + # `result` (the raw merged extraction) never carries one. + "directed": bool((existing_graph_data or {}).get("directed", False)), } candidate_graph_text = _json_text(candidate_graph_data) same_graph = False @@ -1246,7 +1293,8 @@ def _add_deleted_source(path: Path) -> None: # scan but still exist on disk (newly excluded) are pruned # instead of surviving as phantom "deleted" entries (#1908). save_manifest( - detected["files"], kind="ast", root=project_root, + detected["files"], manifest_path=str(out / "manifest.json"), + kind="ast", root=watch_root, scan_corpus={f for _fl in detected["files"].values() for f in _fl}, ) except Exception: @@ -1274,7 +1322,10 @@ def _add_deleted_source(path: Path) -> None: "total_words": detected.get("total_words", 0), } - G = build_from_json(result) + # Inherit the existing graph's directed flag (#2342) so `graphify + # update` can't silently downgrade a directed graph to undirected - + # build_from_json defaults to directed=False otherwise. + G = build_from_json(result, directed=bool((existing_graph_data or {}).get("directed", False))) candidate_topology = _topology_from_graph(G) if existing_graph_data: try: @@ -1289,7 +1340,8 @@ def _add_deleted_source(path: Path) -> None: from graphify.detect import save_manifest # Full-scan save: prune excluded-but-alive rows (#1908). save_manifest( - detected["files"], kind="ast", root=project_root, + detected["files"], manifest_path=str(out / "manifest.json"), + kind="ast", root=watch_root, scan_corpus={f for _fl in detected["files"].values() for f in _fl}, ) except Exception: @@ -1442,7 +1494,8 @@ def _add_deleted_source(path: Path) -> None: from graphify.detect import save_manifest # Full-scan save: prune excluded-but-alive rows (#1908). save_manifest( - detected["files"], kind="ast", root=project_root, + detected["files"], manifest_path=str(out / "manifest.json"), + kind="ast", root=watch_root, scan_corpus={f for _fl in detected["files"].values() for f in _fl}, ) except Exception: From 8043b5df4c906ea7ae9fadaafffa5d6efbc9ac4e Mon Sep 17 00:00:00 2001 From: Nicholas M Denny Date: Tue, 11 Aug 2026 16:20:48 -0500 Subject: [PATCH 4/5] Updating to align with 0.9.40 --- CHANGELOG.md | 77 +- README.md | 898 +------ cache.py | 927 +++++++ cli.py | 2044 +++++++++++++++ dedup.py | 3017 ++++++++++++++++++++++ detect.py | 642 +++++ paths.py | 6670 +++++++++++++++++++++++++++++++++++++++++++++++++ resolution.py | 1955 +++++++++++++++ watch.py | 1455 +++++++++++ 9 files changed, 16816 insertions(+), 869 deletions(-) create mode 100644 cache.py create mode 100644 cli.py create mode 100644 dedup.py create mode 100644 detect.py create mode 100644 paths.py create mode 100644 resolution.py create mode 100644 watch.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 761b8f6e4..35262289d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,16 +2,73 @@ Full release notes with details on each version: [GitHub Releases](https://github.com/safishamsi/graphify/releases) -## Unreleased - -- Fix: Windows scans no longer skip deeply nested local or UNC files when their paths - cross the legacy `MAX_PATH` boundary. Filesystem calls now use extended-length paths - only at the I/O boundary while graph IDs, manifests, cache keys, and diagnostics retain - ordinary paths; discovery, hashing, extraction, incremental rebuilds, and document/media - readers share the same cross-platform adapter. A focused Ubuntu/macOS/Windows CI matrix - exercises 300+ character paths and verifies that the adapter remains a no-op off Windows. - -## 0.9.33 (unreleased) +## 0.9.40 (unreleased) + +- Fix: Windows scans no longer skip deeply nested local or UNC files when their paths cross the legacy `MAX_PATH` boundary. Filesystem calls now use extended-length paths only at the I/O boundary while graph IDs, manifests, cache keys, and diagnostics retain ordinary paths; discovery, hashing, extraction, incremental rebuilds, and document/media readers share the same cross-platform adapter. A focused Ubuntu/macOS/Windows CI matrix exercises 300+ character paths and verifies that the adapter remains a no-op off Windows. +- Fix: the 0.9.37 partial-parse warning no longer fires on valid TypeScript/TSX (#2610, #2599, thanks @Sid-AutoWisdom and @atlasplatformu-ai). tree-sitter-typescript sets an error flag on tiny fully-recovered constructs (a `&` in a JSX string attribute, a semicolon-less `in_*` interface member) that still extract completely; the warning now fires only when recovery plausibly cost symbols (the file yielded at most the file node, or an error region spans multiple lines), so the genuine Kotlin one-line-body and Luau cases still warn. +- Fix: `file_hash()`'s stat fastpath no longer serves a stale digest when a file is rewritten to the same size within one mtime tick (#2612, thanks @rajarshidattapy); a racily-clean guard falls back to a content hash for recently-modified files. +- Fix: stored-path absoluteness is now detected cross-platform, so a POSIX-absolute `source_file` from a Linux/CI-built graph no longer leaks into node ids on Windows (#2618, thanks @rajarshidattapy). +- Fix: `normalize_id()` is now idempotent for Turkish `İ` and similar codepoints by casefolding before the non-word filter; no ASCII identifier ids change (#2614, thanks @rajarshidattapy). +- Fix: `graph.json` collection order is now deterministic across runs (#2582, thanks @hjotha). +- Fix: `explain`/`_find_node` resolve node ids containing punctuation or non-ASCII characters (#2467, thanks @sean-soomgo). +- Fix: `.graphifyignore` patterns match paths regardless of Unicode NFC/NFD normalization, so an accented ignore rule works on macOS (#2544, thanks @bruno-growthsales). +- Fix: Obsidian vault metadata directories (`.obsidian`, `.smart-env`) are skipped during detection (#2493, thanks @rohit-jsfreaky). +- Fix: a single unparenthesised arrow parameter (`x => f(x)`) is now shadowed, so it no longer fabricates an `indirect_call` edge to an unrelated same-named callable (thanks @imagineers-tyler); follows the 0.9.38 sibling-closure fix (#2568). +- Fix: Python extraction no longer crashes resolving an over-deep relative import (`from ....x import y` above the package root) (#2605, thanks @SinghAman21). +- Fix: a Go qualified type (`pkg.Type`) resolves by import path instead of losing the qualifier and binding by bare name to an unrelated same-named symbol (#2608, thanks @gnukeno). +- Fix: `graph.html`'s document title no longer embeds the generator's absolute host path (#2598, thanks @michaelxer); it keeps the path from the output-dir marker onward. + +## 0.9.39 (2026-08-10) + +- Fix: `affected` now traverses a dynamic `import('…')` made inside a function or at module scope (#2584, thanks @phudayyy). The 0.9.38 dedupe keyed only on the target, so an in-function dynamic import (whose symbol-level edge is anchored on the enclosing function) suppressed the file-level edge `affected` follows; the dedupe now keys on the importing file, emitting one file-level `dynamic_import` edge per file/target while keeping the call-site edge. +- Fix: a Python member call on an untyped receiver (`x.get(...)`) no longer binds by name alone to a same-named module-level function, fabricating a false high-confidence `calls` edge and a god node (#2417, #2586, thanks @EZZEASY). Such a call is now resolved only with receiver-type, import, or `self`/`cls`/`super` evidence, matching the TypeScript fix from 0.9.37; `super().method()` still resolves. +- Fix: fuzzy dedup no longer over-merges two distinct entities in the same file whose long labels differ by a content word (#2576, thanks @wilyan09007). A one-token difference is judged on the differing tokens rather than the prefix-weighted whole-label similarity, so `asset contribution flow` and `asset consumption flow` stay separate while genuine typo and whitespace/case variants still collapse. +- Fix: `graphify watch` now rebuilds on a documentation-only deletion batch instead of only flagging it (#2580, thanks @angmeng), so a deleted doc's nodes are evicted immediately rather than waiting for the next code-file event. (The general deleted-file leak was already fixed in 0.9.10; this closes the live-watcher residual.) +- Fix: Objective-C member-call resolution (#2589, #2590, #2591, thanks @xiongjianxu). A `@protocol` declaration is no longer treated as a receiver type (it collided with a same-named class); a category or class-extension interface (`@interface Foo (Bar)`) now folds into the base class instead of minting a duplicate node; and a message send to a `@property` or ivar receiver (`[self.svc run]`, `[_svc run]`) now resolves through the property/ivar's declared type. + +## 0.9.38 (2026-08-09) + +- Fix: the 0.9.37 callback-body fix (#2552) no longer lets a local declared in one callback suppress a call in a sibling callback (#2568, thanks @imagineers-tyler). Each callback body's local names are now scoped to that body instead of unioned under the shared declaration, so a real `indirect_call` in one sibling closure is no longer dropped because another sibling declared a same-named local. This can only restore dropped edges, never fabricate. +- Fix: Kotlin calls in a property initializer are now collected (#2565, thanks @kskchaitanya1993). A class property (`val repo = createRepo()`), a `by lazy { ... }` delegate, a companion-object property, and a top-level property initializer now produce `calls` edges attributed to the enclosing class (or file), including fully-qualified calls. A plain literal initializer produces no edge. +- Fix: Swift receiver-type inference now handles `@Environment(Store.self)` properties and factory-initialised bindings (#2561, thanks @fakewaffle). A member call on a receiver typed only through an `@Environment(Type.self)` attribute, or bound to an in-corpus factory whose return type is known (`let x = ServiceFactory.make()`), now resolves. Ambiguous or non-concrete returns (opaque `some P`, arrays, out-of-corpus) stay unresolved rather than guessing. +- Fix: the SQL extractor no longer emits a `reads_from` edge to a CTE name (#2577, thanks @wilyan09007). A `WITH cte AS (...)` name is scoped to its query and is no longer treated as a table, so it no longer mints a bare stub that could bind to an unrelated same-named symbol; an outer real table sharing a subquery-CTE's name still resolves. +- Fix: a dynamic `await import('…')` inside a nested function or at module scope now produces an edge (#2575, thanks @phudayyy), and `dynamic_import` edges are now included in `affected`. Calls inside a nested named function are also collected now. A dynamic import already captured as a deferred `imports_from` is not double-counted. +- Fix: `explain` resolves node ids that contain punctuation or non-ASCII text (#2467). An id was only ever compared against the `\w+`-tokenized query, so `concept:domain:x`, every `merge-graphs` `::` id, and every Hangul id failed to resolve; the id printed by `explain` could not be fed back into `explain`, and the ambiguity hint "Retry with […] the full node id" named a remedy that could not work. The exact tier now also compares the diacritic-folded id, and the trigram index carries the folded form so a non-ASCII id survives the prefilter. Only ids that previously failed to resolve can now resolve — label queries are unchanged, and an all-ASCII graph indexes byte-identically to before. + +## 0.9.37 (2026-08-08) + +- Fix: TypeScript member calls no longer fabricate a high-confidence `calls` edge by matching a receiver type by name alone (#2553, thanks @Earthfreedom). A member call now resolves only when the receiver's type is defined in the same file or actually imported by the caller's file, so a third-party `import type { Repo }` can no longer bind to an unrelated local `class Repo`; table-inferred receivers are tiered to INFERRED rather than EXTRACTED. +- Fix: TypeScript/JavaScript calls inside a callback body passed to another call (for example `export const handler = wrapper(async (req) => { helper() })`) are no longer dropped (#2552, thanks @Earthfreedom). The callback body is now walked and its calls attributed to the declaration, through the same import-gated resolution so it cannot fabricate edges. +- Fix: Kotlin imports, fully-qualified calls, and one-line type bodies (#2526, #2550, #2551, thanks @spaceBrownie, @thomasrengot-hub, and @Mustaqeem66 for #2531). The extractor now matches the bundled grammar's `import` node (imports were silently dropped) and resolves each import to the real target node; a fully-qualified call like `com.example.Foo.bar()` now produces a `calls` edge; and a file with syntax the bundled grammar cannot parse (such as a one-line `class C { val x }`) now emits a warning instead of silently extracting nothing, and declarations recovered inside an error span keep their enclosing class. +- Fix: `graphify update` now retries a file whose extractor failed on a previous run instead of stamping it up-to-date forever (#2543, thanks @michaelxer). A failed extraction is no longer recorded in the manifest as processed, a manifest already poisoned by the old behavior is healed on the next run, and the fix avoids re-processing a genuinely-unchanged file. +- Fix: the claude-cli backend now surfaces an API error carried in the stdout envelope (for example a rate limit returned with a zero exit code) instead of treating it as an empty success (#2554, thanks @annieyii). The error is raised on both the zero and non-zero exit paths. + +## 0.9.36 (2026-08-07) + +- Fix: four commands that failed silently while exiting 0 now surface the problem (#2534, thanks @elecnix). `cluster-only` warns when `--backend`/`--model`/`--batch-size` are ignored because saved labels are being reused; the community-label prompt no longer collides with the discard sentinel (a model echoing the key back is no longer silently dropped); `tree --root` exits non-zero when the root matches no source file instead of silently flattening the tree; and `cluster-only` stamps `built_at_commit` from the analysed graph rather than the shell's working directory. Also folds in the `cluster-only` refused-write guard from #2522 (thanks @aniJani). +- Fix: a Swift `extension Foo` in a different file from `Foo` no longer drops static and singleton call edges into the type (#2538, thanks @pawelo446). The extension node id is now remapped consistently so the extension merges onto its base type before call resolution, and the merge is gated so it never absorbs a same-named type from another language. +- Fix: node-id collision resolution is now deterministic and prefers active over archived paths (#2532, thanks @michaelxer for the active/archived idea in #2540). Two files that mint the same id (for example `plans/_done/x.md` vs `plans/in-progress/x.md`) are ranked by a lifecycle penalty computed on the root-relative path and a reversed-segment tie-break, so the winner no longer depends on ASCII filename order, absolute-vs-relative path form, or the checkout directory name. +- Fix: the Windows skill now runs on PowerShell (#2528, thanks @tannermosher2015-debug). The Windows skill variant's steps were bash-only (`$(cat ...)`, `rm -f`, `find -delete`); they are now emitted as PowerShell (here-string interpreter invocations and `Remove-Item` cleanup), with POSIX skills unchanged and step parity enforced by a generator check. + +## 0.9.35 (2026-08-06) + +- Fix: the `build_merge` #479 shrink guard is no longer effectively dead (#2497, thanks @sortakool). It read the post-replace node count, so a broken partial re-extract could silently destroy nodes without tripping the guard, and the guard was skipped entirely under `prune_sources`. The guard now diffs the on-disk baseline by node identity and refuses any loss from a source that was neither re-extracted nor pruned this run (active even under `prune_sources`, skipped only under `dedup`), and reports how many nodes a re-extract replaced. +- Fix: `build_merge`/`merge_raw_extraction` `prune_sources` now prunes correctly when given absolute paths under a non-standard layout, deriving the scan root by suffix-matching stored source paths, and warns (instead of reporting "already clean") when a prune matches nothing (#2446, thanks @AI-invest). +- Fix: `graphify update` now removes newly-ignored files from an existing graph (#2495, thanks @alisson-acioli). A file that was added to `.graphifyignore`/`--exclude` (or a skip rule) is evicted even though it still exists on disk; `.gitignore`-driven eviction applies on an explicit full `update`. Files that merely changed are still preserved, and a file that leaves the corpus without matching any live ignore rule stays (fail-closed, #1795). +- Fix: a Java local class and a same-named external annotation (e.g. a local `class Component` and Spring's `@Component`) no longer collapse into one node (#2504, thanks @te7ina-honey). The Java type resolver now runs before the unique-label stub rewire and parks an imported-but-external type on its fully-qualified name, and cross-file import resolution checks the package. In-corpus annotation resolution is unchanged. +- Fix: `graphify callflow` now respects edge direction, so the caller/callee columns are correct (#2508, thanks @Tomaskobel). The call-flow HTML loads the graph directed and recovers direction from the stored `_src`/`_tgt` markers (consistent with the `path` fix), and indirect calls are counted. +- Fix: relational-intent verbs in a `query` ("calls", "uses", "extends", ...) no longer seat spurious seeds (#2507, thanks @filipechagas). Such a verb is excluded from the per-term seed guarantee, so a decoy matching only the verb no longer becomes a traversal root, while a verb that is a genuine symbol name can still be seeded on merit. + +## 0.9.34 (2026-08-05) + +- Fix: C# receiver typing no longer drops a true call when a same-named variable is declared untypeably elsewhere in the method (#2472, thanks @JensD-git). Receiver types are now tracked per lexical declaration scope and resolved by the call's position, so a typed `static` local-function parameter keeps resolving even when an `out var` reuses the name in the enclosing body. This fixes a regression from 0.9.32 (#2346). Cross-method independence (#2299) and field-conflict poisoning are unchanged; an `out var` receiver itself remains untyped. +- Fix: `graphify path` (and the MCP `shortest_path` tool) now respect edge direction by default instead of running on an undirected view, so a returned path no longer traverses edges backwards (#2487, thanks @luliaz0601). Direction is recovered from the stored `_src`/`_tgt` markers. Pass `--undirected` (CLI) or `undirected=true` (MCP) to search ignoring direction; when no directed path exists the command says so instead of silently returning a reversed one. +- Fix: semantic extraction no longer aborts at merge with a `TypeError` when a hyperedge carries dict-shaped members (#2486, thanks @adminwat). Members are normalized to ids (or dropped with a warning) so a malformed hyperedge can no longer destroy a completed extraction. +- Fix: `graphify merge-graphs` no longer drops hyperedges (#2484, thanks @sortakool, and @oleksii-tumanov for the approach in #1691). Hyperedge member ids and ids are now relabeled with the per-repo prefix, both inputs' hyperedges are unioned instead of one clobbering the other, and they are written to both the top-level and nested slots. +- Fix: `build_from_json` now reads hyperedges from both the top-level and nested `graph` slots, so label and re-cluster runs no longer silently empty a graph's hyperedge set (#2485, thanks @sortakool); a full validation wipeout is now reported loudly. +- Fix: the skill flow now passes the curated community labels to `to_json`, so `graph.json` ships with `community_name` on nodes instead of dropping it (#2490, thanks @PapiScholz). + +## 0.9.33 (2026-08-05) - Fix: the C# `partial class` merge (#2332) no longer conflates two same-named classes that live in different assemblies (#2411, thanks @JensD-git). The merge now keys on assembly (nearest ancestor directory containing a `.csproj`/`.fsproj`/`.vbproj`) in addition to namespace and name, so genuine partial halves within one project still merge while same-name types in separate projects stay distinct. A corpus with no project file keeps merging by namespace and name as before. - Fix: `graphify update` no longer drops member-call and `indirect_call` edges from a changed file into an unchanged target (#2437, #2438, thanks @aryanbonigala). Incremental re-resolution now sees the unchanged corpus (its nodes, `contains`/`method` edges, and the `_callable` markers, which now persist to `graph.json` like `_origin`), so cross-file calls survive an incremental rebuild while edges to a genuinely removed target are still evicted. diff --git a/README.md b/README.md index 36a2235ba..1eb0070e7 100644 --- a/README.md +++ b/README.md @@ -1,872 +1,52 @@ -

- Graphify -

+# Graphify 0.9.40 long-path PR merge files -

- Graphify-Labs%2Fgraphify | Trendshift -

+These are complete replacement files based on the supplied Graphify 0.9.40 +snapshot. They preserve current upstream behavior while retaining the proposed +cross-platform filesystem I/O boundary. -
-
Read this in other languages +## GitHub-reported conflict files -🇺🇸 English | 🇨🇳 简体中文 | 🇯🇵 日本語 | 🇰🇷 한국어 | 🇩🇪 Deutsch | 🇫🇷 Français | 🇪🇸 Español | 🇮🇳 हिन्दी | 🇧🇷 Português | 🇷🇺 Русский | 🇸🇦 العربية | 🇮🇷 فارسی | 🇮🇹 Italiano | 🇵🇱 Polski | 🇳🇱 Nederlands | 🇹🇷 Türkçe | 🇺🇦 Українська | 🇻🇳 Tiếng Việt | 🇮🇩 Bahasa Indonesia | 🇸🇪 Svenska | 🇬🇷 Ελληνικά | 🇷🇴 Română | 🇨🇿 Čeština | 🇫🇮 Suomi | 🇩🇰 Dansk | 🇳🇴 Norsk | 🇭🇺 Magyar | 🇹🇭 ภาษาไทย | 🇺🇿 Oʻzbekcha | 🇹🇼 繁體中文 | 🇵🇭 Filipino | 🇮🇱 עברית +Replace these files to resolve the reported conflicts: -
-
+- `CHANGELOG.md` +- `graphify/build.py` +- `graphify/cache.py` +- `graphify/extract.py` +- `graphify/extractors/resolution.py` +- `graphify/paths.py` -

- PyPI - Downloads - Discord - LinkedIn - YC S26 -

+## Additional compatibility files -

- Early access to the graphify platform is open before the public v1 launch: app.graphify.com -

+Also replace these files. Upstream 0.9.33-0.9.40 added new filesystem calls in +these code paths without producing textual merge conflicts; leaving them as-is +would bypass the long-path adapter in parts of detection, incremental update, +watch reconciliation, dynamic-import resolution, and collision ranking. -Type `/graphify` in your AI coding assistant and it maps your entire project (code, docs, PDFs, images, videos) into a **knowledge graph** you can **query instead of grepping** through files. +- `graphify/cli.py` +- `graphify/dedup.py` +- `graphify/detect.py` +- `graphify/watch.py` -- **Code maps for free, fully local.** Code is parsed with tree-sitter AST: deterministic, no LLM, nothing leaves your machine. (Docs, PDFs, images and video use your assistant's model, or a configured API key, for a semantic pass.) -- **Every edge is explained.** Each connection is tagged `EXTRACTED` (explicit in the source) or `INFERRED` (resolved by graphify), so you can tell what was read directly from what was inferred. -- **Not a vector index.** No embeddings, no vector store: a real graph you traverse. Ask a question, trace the path between two things, or explain one concept. +The files preserve repository-relative paths. Copy them into the same locations +in the PR branch; do not add them at the repository root. -> Want this always-on, updating in the background across your code, docs, and meetings rather than only on demand? That is what we are building at **[graphify.com](https://graphify.com)**, and early access is open now at **[app.graphify.com](https://app.graphify.com/login)**. +## Validation performed -

- graphify's interactive graph.html showing the FastAPI codebase as a force-directed knowledge graph with a legend of detected communities -

-

- The FastAPI codebase mapped by graphify. Every node is a concept, colors are detected communities, and the whole thing is clickable in graph.html. -

+- Python compilation: passed +- `git diff --check`: passed +- Conflict-marker scan: passed +- Focused filesystem/path suite: 87 passed, 1 skipped +- Dependency-independent merge/regression suite: 480 passed, 1 skipped +- Sensitive-path scan: passed -**Get started** (30 seconds): +The full dependency installation was unavailable in the analysis environment +because outbound package download/DNS failed. `graphify update .` was attempted +but could not run because tree-sitter was unavailable. Native Windows and the +repository's normal CI remain the authoritative final gates. -```bash -uv tool install graphifyy # install the CLI (or: pipx install graphifyy) -graphify install # register the skill with your AI assistant -``` +Source snapshot: -Then, in your AI assistant: - -``` -/graphify . -``` - -That's it. You get **three files**: - -``` -graphify-out/ -├── graph.html open in any browser — click nodes, filter, search -├── GRAPH_REPORT.md the highlights: key concepts, surprising connections, suggested questions -└── graph.json the full graph — query it anytime without re-reading your files -``` - -**Works in** Claude Code, Cursor, Codex, Gemini CLI, GitHub Copilot, and 15+ more — [pick your platform](#install). - ---- - -## See it in action - -

- graphify path query: a terminal asks for the shortest path between FastAPI and ModelField, and the answer lights up hop by hop across the knowledge graph -

- -Once the graph is built you query it instead of reading files. Real output, graphify run on the FastAPI codebase shown above: - -```text -$ graphify explain "APIRouter" -Node: APIRouter - Source: routing.py L2210 - Community: 2 - Degree: 47 - -Connections (47): - --> RequestValidationError [uses] [INFERRED] - --> Dependant [uses] [INFERRED] - --> .get() [method] [EXTRACTED] - <-- __init__.py [imports] [EXTRACTED] - ... - -$ graphify path "FastAPI" "ModelField" -Shortest path (3 hops): - FastAPI --uses--> DefaultPlaceholder <--references-- get_request_handler() --references--> ModelField -``` - -Every edge carries a **confidence tag** (`EXTRACTED` = explicit in the source, `INFERRED` = derived by resolution), so you can tell what was read directly from what was inferred. `graphify query ""` returns a scoped subgraph for a plain-language question, and `graphify path A B` traces how any two things connect. - ---- - -## What it does - -What you get out of the box: - -| Capability | What you get | -|---|---| -| **God nodes** | The most-connected concepts, so you see what everything flows through | -| **Communities** | The graph split into subsystems (Leiden), with LLM-free labels | -| **Cross-file links** | `calls` / `imports` / `inherits` / `mixes_in` resolved across ~40 languages via tree-sitter AST | -| **Query, path, explain** | Ask a question, trace the path between two things, or explain one concept, all against `graph.json` | -| **Rationale + doc refs** | `# NOTE:` / `# WHY:` comments and ADR/RFC citations become first-class nodes linked to the code | -| **Beyond code** | Docs, PDFs, images, and video/audio all map into the same graph | -| **Local-first** | Code is parsed locally with tree-sitter (no LLM, nothing leaves your machine); only the semantic pass over docs/media calls a backend, and only if you configure one | - ---- - -## Benchmarks - -| Benchmark | Metric | graphify | Field | -|---|---|---|---| -| LOCOMO (n=300) | recall@10 | **0.497** | mem0 0.048, supermemory 0.149 | -| LOCOMO (n=300) | QA accuracy | 45.3% | supermemory 49.7%, mem0 27.3% | -| LongMemEval-S (n=50) | QA accuracy | **76%** | tied with dense RAG | -| Graph build | LLM credits | **0** | per-token for most systems | - -Every system ran on the same harness with the same model and budgets, scored by a judge blind-validated against a second judge (90.6% agreement, Cohen's kappa 0.81). Full per-system tables, the code-intelligence result, and reproduction commands: **[BENCHMARKS.md](./BENCHMARKS.md)**. - ---- - -## Prerequisites - -| Requirement | Minimum | Check | Install | -|---|---|---|---| -| Python | 3.10+ | `python --version` | [python.org](https://www.python.org/downloads/) | -| uv *(recommended)* | any | `uv --version` | `curl -LsSf https://astral.sh/uv/install.sh \| sh` | -| pipx *(alternative)* | any | `pipx --version` | `pip install pipx` | - -**macOS quick install (Homebrew):** -```bash -brew install python@3.12 uv -``` - -**Windows quick install:** -```powershell -winget install astral-sh.uv -``` - -**Ubuntu/Debian:** -```bash -sudo apt install python3.12 python3-pip pipx -# or install uv: -curl -LsSf https://astral.sh/uv/install.sh | sh -``` - ---- - -## Install - -> **Official package:** The PyPI package is `graphifyy` (double-y). Other `graphify*` packages on PyPI are not affiliated. The CLI command is still `graphify`. - -**Step 1 — install the package:** - -```bash -# Recommended (isolated env; if 'graphify' isn't found after, run: uv tool update-shell): -uv tool install graphifyy - -# Alternatives: -pipx install graphifyy -pip install graphifyy # may need PATH setup — see note below -``` - -**Step 2 — register the skill with your AI assistant:** - -```bash -graphify install -``` - -That's it. Open your AI assistant and type `/graphify .` - -To install the assistant skill into the current repository instead of your user -profile, add `--project`: - -```bash -graphify install --project -graphify install --project --platform codex -``` - -Project-scoped installs write under the current directory, for example -`.claude/skills/graphify/SKILL.md` or `.agents/skills/graphify/SKILL.md` (plus a -`references/` sidecar the skill loads on demand), and -print a `git add` hint for files that can be committed. -Per-platform commands that support project-scoped installs accept the same flag, -for example `graphify claude install --project` or `graphify codex install --project`. - -> **PowerShell note:** Use `graphify .` not `/graphify .` — the leading slash is a path separator in PowerShell. - -> **`graphify: command not found`?** `uv tool install` / `pipx install` put the `graphify` command in their tool bin dir (`~/.local/bin`). If your shell can't find it right after install — common on a fresh macOS + zsh setup — that dir isn't on your `PATH` yet: run `uv tool update-shell` (or `pipx ensurepath`), then open a new terminal. With plain `pip`, add `~/.local/bin` (Linux) or `~/Library/Python/3.x/bin` (Mac) to your PATH, or run `python -m graphify`. - -> **Running with `uvx` / `uv tool run` instead of installing?** Name the package, not the command: `uvx --from graphifyy graphify install`. Plain `uvx graphify …` fails (`No solution found … no versions of graphify`) because `uv tool run` reads the first word as a *package*, and the package is `graphifyy` — the `graphify` command lives inside it. - -> **Avoid `pip install` on Mac/Windows** if possible. The skill resolves Python at runtime from `graphify-out/.graphify_python`; if that points to a different environment than where `pip` installed the package, you'll get `ModuleNotFoundError: No module named 'graphify'`. `uv tool install` and `pipx install` isolate the package in their own env and avoid this entirely. - -> **Git hooks and uv tool / pipx:** `graphify hook install` embeds the current interpreter path directly into the hook scripts at install time, so the post-commit hook fires correctly even in GUI git clients and CI runners where `~/.local/bin` is not on PATH. If you reinstall or upgrade graphify, re-run `graphify hook install` to refresh the embedded path. - -> **Strict mode (Claude Code):** `graphify install --project --strict` makes the assistant actually use the graph. The default install *nudges* it to run `graphify query` before reading files; strict mode *blocks* the first raw source read of a session and redirects it to the graph, then reverts to the nudge (so it fires at most once per session and never gets stuck). Toggle at runtime with `GRAPHIFY_HOOK_STRICT=1`/`0`; the default install is unchanged (soft nudge). - -
-Pick your platform (20+ assistants, click to expand) - -| Platform | Install command | -|----------|----------------| -| Claude Code (Linux/Mac) | `graphify install` | -| Claude Code (Windows) | `graphify install` (auto-detected) or `graphify install --platform windows` | -| CodeBuddy | `graphify install --platform codebuddy` | -| Codex | `graphify install --platform codex` | -| OpenCode | `graphify install --platform opencode` | -| Kilo Code | `graphify install --platform kilo` | -| GitHub Copilot CLI | `graphify install --platform copilot` | -| VS Code Copilot Chat | `graphify vscode install` | -| Aider | `graphify install --platform aider` | -| OpenClaw | `graphify install --platform claw` | -| Factory Droid | `graphify install --platform droid` | -| Trae | `graphify install --platform trae` | -| Trae CN | `graphify install --platform trae-cn` | -| Gemini CLI | `graphify install --platform gemini` | -| Hermes | `graphify install --platform hermes` | -| Kimi Code | `graphify install --platform kimi` | -| Amp | `graphify amp install` | -| Agent Skills (cross-framework) | `graphify install --platform agents` (alias `--platform skills`) | -| Kiro IDE/CLI | `graphify kiro install` | -| Pi coding agent | `graphify install --platform pi` | -| Cursor | `graphify cursor install` | -| Devin CLI | `graphify devin install` | -| Google Antigravity | `graphify antigravity install` | - -Codex users also need `multi_agent = true` under `[features]` in `~/.codex/config.toml` for parallel extraction. CodeBuddy uses the same Agent tool and PreToolUse hook mechanism as Claude Code. Factory Droid uses the `Task` tool for parallel subagent dispatch. OpenClaw and Aider use sequential extraction (parallel agent support is still early on those platforms). Trae uses the Agent tool for parallel subagent dispatch and does **not** support `PreToolUse` hooks, so AGENTS.md is the always-on mechanism. - -`--platform agents` (alias `--platform skills`) targets the generic cross-framework [Agent-Skills](https://github.com/anthropics/skills) locations: the spec's user-global `~/.agents/skills/` (read by `npx skills` and spec-compliant frameworks) for a global install, and `./.agents/skills/` for a project (`--project`) install. The bare `graphify install` stays single-platform (Claude Code) by design — use the named `agents` platform when you want the skill discoverable by any framework that reads `.agents/skills`. - -> Codex uses `$graphify` instead of `/graphify`. - -
- -
-Optional extras (install only what you need) - -| Extra | What it adds | Install | -|---|---|---| -| `pdf` | PDF extraction | `uv tool install "graphifyy[pdf]"` | -| `office` | `.docx` and `.xlsx` support | `uv tool install "graphifyy[office]"` | -| `google` | Google Sheets rendering | `uv tool install "graphifyy[google]"` | -| `video` | Video/audio transcription (faster-whisper + yt-dlp) | `uv tool install "graphifyy[video]"` | -| `mcp` | MCP stdio server | `uv tool install "graphifyy[mcp]"` | -| `neo4j` | Neo4j push support | `uv tool install "graphifyy[neo4j]"` | -| `falkordb` | FalkorDB push support | `uv tool install "graphifyy[falkordb]"` | -| `svg` | SVG graph export | `uv tool install "graphifyy[svg]"` | -| `leiden` | Leiden community detection (Python < 3.13 only) | `uv tool install "graphifyy[leiden]"` | -| `ollama` | Ollama local inference | `uv tool install "graphifyy[ollama]"` | -| `openai` | OpenAI / OpenAI-compatible APIs | `uv tool install "graphifyy[openai]"` | -| `gemini` | Google Gemini API | `uv tool install "graphifyy[gemini]"` | -| `anthropic` | Anthropic Claude API (`--backend claude`, uses `ANTHROPIC_API_KEY`) | `uv tool install "graphifyy[anthropic]"` | -| `bedrock` | AWS Bedrock (uses IAM, no API key) | `uv tool install "graphifyy[bedrock]"` | -| `azure` | Azure OpenAI Service (`--backend azure`, uses `AZURE_OPENAI_API_KEY` + `AZURE_OPENAI_ENDPOINT`) | `uv tool install "graphifyy[openai]"` | -| `sql` | SQL schema extraction | `uv tool install "graphifyy[sql]"` | -| `postgres` | Live PostgreSQL introspection (`--postgres DSN`) | `uv tool install "graphifyy[postgres]"` | -| `dm` | BYOND DreamMaker `.dm`/`.dme` AST extraction (may need a C compiler + `python3-dev` if no wheel matches your platform) | `uv tool install "graphifyy[dm]"` | -| `terraform` | Terraform / HCL `.tf`/`.tfvars`/`.hcl` AST extraction | `uv tool install "graphifyy[terraform]"` | -| `pascal` | Pascal / Delphi `.pas`/`.dpr`/`.dpk`/`.inc` AST extraction (more accurate `calls`/`inherits` edges; falls back to a regex extractor when absent) | `uv tool install "graphifyy[pascal]"` | -| `chinese` | Chinese query segmentation (jieba) | `uv tool install "graphifyy[chinese]"` | -| `all` | Everything above | `uv tool install "graphifyy[all]"` | - -
- ---- - -## Make your assistant always use the graph - -Run this once in your project after building a graph: - -| Platform | Command | -|----------|---------| -| Claude Code | `graphify claude install` | -| CodeBuddy | `graphify codebuddy install` | -| Codex | `graphify codex install` | -| OpenCode | `graphify opencode install` | -| Kilo Code | `graphify kilo install` | -| GitHub Copilot CLI | `graphify copilot install` | -| VS Code Copilot Chat | `graphify vscode install` | -| Aider | `graphify aider install` | -| OpenClaw | `graphify claw install` | -| Factory Droid | `graphify droid install` | -| Trae | `graphify trae install` | -| Trae CN | `graphify trae-cn install` | -| Cursor | `graphify cursor install` | -| Gemini CLI | `graphify gemini install` | -| Hermes | `graphify hermes install` | -| Kimi Code | `graphify install --platform kimi` | -| Amp | `graphify amp install` | -| Agent Skills (cross-framework) | `graphify agents install` (alias `graphify skills install`) | -| Kiro IDE/CLI | `graphify kiro install` | -| Pi coding agent | `graphify pi install` | -| Devin CLI | `graphify devin install` | -| Google Antigravity | `graphify antigravity install` | - -This writes a small config file that tells your assistant to consult the knowledge graph for codebase questions, preferring scoped queries like `graphify query ""` over reading the full report or grepping raw files. - -- **Hook platforms** (Claude Code, Gemini CLI): a hook fires automatically before search-style tool calls (and, on Claude Code, before reading source files one by one via the Read/Glob tools) and nudges your assistant toward the graph path. -- **Instruction-file platforms** (Codex, OpenCode, Cursor, etc.): persistent instruction files (`AGENTS.md`, `.cursor/rules/`, etc.) provide the same query-first guidance. - -`GRAPH_REPORT.md` is still available for broad architecture review. - -**CodeBuddy** does the same two things as Claude Code: writes a `CODEBUDDY.md` section telling CodeBuddy to read `graphify-out/GRAPH_REPORT.md` before answering architecture questions, and installs `PreToolUse` hooks (`.codebuddy/settings.json`) that fire before Bash search commands and file reads, nudging toward `graphify query` instead. - -**Codex** writes to `AGENTS.md`, which is what actually carries the always-on graph guidance on this platform. `graphify codex install` also registers a `PreToolUse` hook in `.codex/hooks.json` (`graphify hook-check`), but that entry is deliberately a **no-op**: Codex Desktop rejects `hookSpecificOutput.additionalContext` on `PreToolUse`, so emitting a nudge there would break Bash tool calls. Unlike Claude Code, where the hook (`graphify hook-guard`) does the nudging, on Codex the hook fires and intentionally does nothing, and `AGENTS.md` is the always-on mechanism. - -**Kilo Code** installs the Graphify skill to `~/.config/kilo/skills/graphify/SKILL.md` and a native `/graphify` command to `~/.config/kilo/command/graphify.md`. `graphify kilo install` also writes `AGENTS.md` plus a native `tool.execute.before` plugin (`.kilo/plugins/graphify.js` + `.kilo/kilo.json` or `.kilo/kilo.jsonc` registration) so Kilo gets the same always-on graph reminder behavior through native `.kilo` config. - -**Cursor** writes `.cursor/rules/graphify.mdc` with `alwaysApply: true`, so Cursor includes it in every conversation automatically, no hook needed. - -To remove graphify from all platforms at once: `graphify uninstall` (add `--purge` to also delete `graphify-out/`). Or use the per-platform command (e.g. `graphify claude uninstall`). - ---- - -## What's in the report - -- **God nodes** — the most-connected concepts in your project. Everything flows through these. -- **Surprising connections** — links between things that live in different files or modules. Ranked by how unexpected they are. -- **The "why"** — inline comments (`# NOTE:`, `# WHY:`, `# HACK:`), docstrings, and design rationale from docs are extracted as separate nodes linked to the code they explain. -- **Suggested questions** — 4–5 questions the graph is uniquely positioned to answer. -- **Confidence tags** — every inferred relationship is marked `EXTRACTED`, `INFERRED`, or `AMBIGUOUS`. You always know what was found vs guessed. - ---- - -## What files it handles - -| Type | Extensions | -|------|-----------| -| Code (36 tree-sitter grammars) | `.py .ts .mts .cts .js .jsx .tsx .mjs .go .rs .java .c .cpp .cc .cxx .h .hpp .cu .cuh .metal .rb .cs .kt .kts .scala .php .swift .lua .luau .toc .zig .ps1 .psm1 .psd1 .ex .exs .m .mm .jl .vue .svelte .astro .groovy .gradle .dart .v .sv .svh .sql .f .f90 .f95 .f03 .f08 .pas .pp .dpr .dpk .lpr .inc .dfm .lfm .lpk .sh .bash .json .dm .dme .dmi .dmm .dmf .sln .slnx .csproj .fsproj .vbproj .xaml .razor .cshtml` (`.dm`/`.dme` requires `uv tool install graphifyy[dm]`; `.mts`/`.cts` reuse the TypeScript grammar, `.cc`/`.cxx` and CUDA `.cu`/`.cuh` and Metal `.metal` reuse the C++ grammar) | -| Salesforce Apex | `.cls .trigger` (regex-based; classes, interfaces, enums, methods, triggers, SOQL/DML edges) | -| Terraform / HCL | `.tf .tfvars .hcl` (requires `uv tool install graphifyy[terraform]`) | -| MCP configs | `.mcp.json` `mcp.json` `mcp_servers.json` `claude_desktop_config.json` — extracts server nodes, package refs, env var requirements | -| Package manifests | `apm.yml` `pyproject.toml` `go.mod` `pom.xml` — one canonical package node per package (by name) plus `depends_on` edges, so a package referenced from many manifests is a single hub | -| Docs | `.md .mdx .qmd .html .txt .rst .yaml .yml` (markdown `[text](./other.md)` links and `[[wikilinks]]` become `references` edges between docs) | -| Office | `.docx .xlsx` (requires `uv tool install graphifyy[office]`) | -| Google Workspace | `.gdoc .gsheet .gslides` (opt-in; requires `gws` auth and `--google-workspace`; Sheets need `uv tool install graphifyy[google]`) | -| PDFs | `.pdf` | -| Images | `.png .jpg .webp .gif` | -| Video / Audio | `.mp4 .mov .mp3 .wav` and more (requires `uv tool install graphifyy[video]`) | -| YouTube / URLs | any video URL (requires `uv tool install graphifyy[video]`) | - -Code is extracted **locally with no API calls** (AST via tree-sitter). Everything else goes through your AI assistant's model API. - -Google Drive for desktop `.gdoc`, `.gsheet`, and `.gslides` files are shortcut -pointers, not document content. To include native Google Docs, Sheets, and Slides -in a headless extraction, install and authenticate the -[`gws` CLI](https://github.com/googleworkspace/cli), then run: - -```bash -uv tool install "graphifyy[google]" # needed for Google Sheets table rendering -gws auth login -s drive -graphify extract ./docs --google-workspace -``` - -You can also set `GRAPHIFY_GOOGLE_WORKSPACE=1`. Graphify exports shortcuts into -`graphify-out/converted/` as Markdown sidecars, then extracts those files. - ---- - -## Common commands - -```bash -/graphify . # build graph for current folder -/graphify ./docs --update # re-extract only changed files -/graphify . --cluster-only # rerun clustering without re-extracting -/graphify . --cluster-only --resolution 1.5 # more granular communities -/graphify . --cluster-only --exclude-hubs 99 # suppress utility super-hubs from god-node rankings -/graphify . --no-viz # skip the HTML, just the report + JSON -/graphify . --wiki # build a markdown wiki from the graph -graphify export callflow-html # Mermaid architecture/call-flow HTML (auto-regenerates on every git commit if hook is installed) - -/graphify query "what connects auth to the database?" -/graphify path "UserService" "DatabasePool" -/graphify explain "RateLimiter" - -/graphify add https://arxiv.org/abs/1706.03762 # fetch a paper and add it -/graphify add # transcribe and add a video - -graphify hook install # auto-rebuild on git commit -graphify merge-graphs a.json b.json # combine two graphs - -graphify prs # PR dashboard: CI state, review status, worktree mapping -graphify prs 42 # deep dive on PR #42 with graph impact -graphify prs --triage # AI ranks your review queue (uses whatever backend is configured) -graphify prs --conflicts # PRs sharing graph communities — merge-order risk -``` - -See the [full command reference](#full-command-reference) below. - ---- - -## Ignoring files - -Create a `.graphifyignore` in your project root — same syntax as `.gitignore`, including `!` negation. - -**`.gitignore` is respected automatically.** graphify reads the `.gitignore` in each directory. If a `.graphifyignore` is also present, the two are **merged** — `.graphifyignore` patterns are evaluated last, so they win on conflicts (including `!` negations). Adding a `.graphifyignore` only ever excludes more; it never re-includes a file your `.gitignore` already excluded. Subdirectory scoping works the same way as git — an ignore file only affects its own subtree. - -Pass `--no-gitignore` to `graphify extract` when git-ignored generated or transpiled code belongs in the graph. This disables `.gitignore` and `.git/info/exclude`; `.graphifyignore` still applies. - -``` -# .graphifyignore -node_modules/ -dist/ -*.generated.py - -# only index src/, ignore everything else -* -!src/ -!src/** -``` - ---- - -## Team setup - -`graphify-out/` is meant to be committed to git so everyone on the team starts with a map. - -**Recommended `.gitignore` additions:** -``` -graphify-out/cost.json # local only -# graphify-out/cache/ # optional: commit for speed, skip to keep repo small -``` - -> `manifest.json` is now portable — keys are stored as relative paths and re-anchored on load, so committing it is safe and avoids a full rebuild on first checkout. - -**Workflow:** -1. One person runs `/graphify .` and commits `graphify-out/`. -2. Everyone pulls — their assistant reads the graph immediately. -3. Run `graphify hook install` to auto-rebuild after each commit (AST only, no API cost). This also sets up a git merge driver so `graph.json` is never left with conflict markers — two devs committing in parallel get their graphs union-merged automatically. -4. When docs or papers change, run `/graphify --update` to refresh those nodes. - ---- - -## Using the graph directly - -```bash -# query the graph from the terminal -graphify query "show the auth flow" -graphify query "what connects DigestAuth to Response?" --graph graphify-out/graph.json - -# expose the graph as an MCP server (for repeated tool-call access) -python -m graphify.serve graphify-out/graph.json -python -m graphify.serve --graph graphify-out/graph.json # --graph flag also accepted - -# register with Kimi Code: -kimi mcp add --transport stdio graphify -- python -m graphify.serve graphify-out/graph.json - -# or serve over HTTP so a whole team points at one URL (no local graphify needed): -python -m graphify.serve graphify-out/graph.json --transport http --port 8080 -python -m graphify.serve graphify-out/graph.json --transport http --host 0.0.0.0 --api-key "$SECRET" -``` - -The MCP server gives your assistant structured access: `query_graph`, `get_node`, `get_neighbors`, `shortest_path`, `list_prs`, `get_pr_impact`, `triage_prs`. - -### Shared HTTP server - -`--transport stdio` (the default) spawns one local server per developer. `--transport http` serves the same tools over the MCP Streamable HTTP transport, so a single shared process can serve the graph for the whole team — clients point their IDE MCP config at `http://:8080/mcp` instead of running graphify locally. - -| Flag | Default | Purpose | -|---|---|---| -| `--transport {stdio,http}` | `stdio` | Transport to serve on | -| `--host` | `127.0.0.1` | HTTP bind host (use `0.0.0.0` to expose beyond localhost) | -| `--port` | `8080` | HTTP bind port | -| `--api-key` | env `GRAPHIFY_API_KEY` | Require `Authorization: Bearer ` (or `X-API-Key`) | -| `--path` | `/mcp` | HTTP mount path | -| `--json-response` | off | Return plain JSON instead of SSE streams | -| `--stateless` | off | No per-session state (for load-balanced / CI deployments) | -| `--session-timeout` | `3600` | Reap idle stateful sessions after N seconds (`0` disables) | - -The default `127.0.0.1` bind is loopback-only. Set `--host 0.0.0.0` **and** `--api-key` together when exposing on a shared host. Run it in a container: - -```bash -docker build -t graphify . -docker run -p 8080:8080 -v "$(pwd)/graphify-out:/data" graphify \ - /data/graph.json --transport http --host 0.0.0.0 --api-key "$SECRET" -``` - -> **WSL / Linux note:** Ubuntu ships `python3`, not `python`. Use a venv to avoid conflicts: -> ```bash -> python3 -m venv .venv && .venv/bin/pip install "graphifyy[mcp]" -> ``` - ---- - -## Environment variables - -These are only needed for **headless / CI extraction** (`graphify extract`). When running via the `/graphify` skill inside your IDE, the model API is provided by your IDE session — no extra keys needed. - -| Variable | Used for | When required | -|---|---|---| -| `ANTHROPIC_API_KEY` | Claude (Anthropic) backend | `--backend claude` | -| `ANTHROPIC_BASE_URL` | Anthropic-compatible endpoint URL (LiteLLM proxy, gateways, ...) | `--backend claude` (default: `https://api.anthropic.com`) | -| `ANTHROPIC_MODEL` | Model name for the Claude backend — for custom endpoints, use the model name/alias your server exposes | `--backend claude` (default: `claude-sonnet-4-6`) | -| `GEMINI_API_KEY` or `GOOGLE_API_KEY` | Google Gemini backend | `--backend gemini` | -| `OPENAI_API_KEY` | OpenAI or OpenAI-compatible APIs | `--backend openai` (local servers accept any non-empty value) | -| `OPENAI_BASE_URL` | OpenAI-compatible server URL (llama.cpp, vLLM, LM Studio, ...) | `--backend openai` (default: `https://api.openai.com/v1`) | -| `OPENAI_MODEL` | Model name for the OpenAI backend — for self-hosted servers, use the model name/alias your server exposes (check its `/v1/models` endpoint), e.g. `LFM2.5-8B-A1B-UD-Q4_K_XL` for llama.cpp | `--backend openai` (default: `gpt-4.1-mini`) | -| `DEEPSEEK_API_KEY` | DeepSeek backend | `--backend deepseek` | -| `MOONSHOT_API_KEY` | Kimi Code backend | `--backend kimi` | -| `OLLAMA_BASE_URL` | Ollama local inference URL | `--backend ollama` (default: `http://localhost:11434`) | -| `OLLAMA_MODEL` | Ollama model name | `--backend ollama` (default: auto-detect) | -| `GRAPHIFY_OLLAMA_NUM_CTX` | Override Ollama KV-cache window size | optional — auto-sized by default | -| `GRAPHIFY_OLLAMA_KEEP_ALIVE` | Minutes to keep Ollama model loaded | optional — set `0` to unload after each chunk | -| `AZURE_OPENAI_API_KEY` | Azure OpenAI Service backend | `--backend azure` | -| `AZURE_OPENAI_ENDPOINT` | Azure resource endpoint URL | `--backend azure` (required alongside API key) | -| `AZURE_OPENAI_API_VERSION` | Azure API version override | optional — default `2024-12-01-preview` | -| `AZURE_OPENAI_DEPLOYMENT` or `GRAPHIFY_AZURE_MODEL` | Azure deployment name | optional — default `gpt-4o` | -| `AWS_*` / `~/.aws/credentials` | AWS Bedrock — standard credential chain | `--backend bedrock` (no API key, uses IAM) | -| `GRAPHIFY_MAX_WORKERS` | AST parallelism thread count | optional — also `--max-workers` flag | -| `GRAPHIFY_MAX_OUTPUT_TOKENS` | Raise output cap for dense corpora | optional — e.g. `32768` for large files | -| `GRAPHIFY_API_TIMEOUT` | Per-call timeout in seconds for HTTP, claude-cli, Anthropic SDK, and Bedrock backends (default: 600) | optional — also `--api-timeout` flag | -| `GRAPHIFY_MAX_RETRIES` | How many times to retry a rate-limited (429) request before giving up (default: 6; honors `Retry-After`) | optional — raise for strict per-org limits (e.g. kimi); `0` disables | -| `GRAPHIFY_FORCE` | Force graph rebuild even with fewer nodes | optional — also `--force` flag | -| `GRAPHIFY_GOOGLE_WORKSPACE` | Auto-enable Google Workspace export | optional — set to `1` | -| `GRAPHIFY_TRIAGE_BACKEND` | Backend for `graphify prs --triage` | optional — auto-detected from available keys | -| `GRAPHIFY_TRIAGE_MODEL` | Model override for triage | optional — e.g. `claude-opus-4-7` | -| `GRAPHIFY_QUERY_LOG_ENABLE` | Set to `1` to turn on the local query log at `~/.cache/graphify-queries.log` (records each query/path/explain question + corpus path). Off by default — nothing is written unless you opt in (#1797) | optional | -| `GRAPHIFY_QUERY_LOG` | Enable the query log and write it to this path instead of the default | optional — off unless this or `_ENABLE` is set | -| `GRAPHIFY_QUERY_LOG_DISABLE` | Set to `1` to force the query log off (wins over the enable vars) | optional | -| `GRAPHIFY_QUERY_LOG_RESPONSES` | When the log is enabled, also record full subgraph responses (off by default) | optional | -| `GRAPHIFY_MAX_GRAPH_BYTES` | Override the 512 MiB graph.json size cap — e.g. `700MB`, `2GB`, or plain bytes | optional — useful for very large corpora | -| `GRAPHIFY_MAX_CONTEXTS` | Maximum number of non-default project graphs retained by one multi-project MCP server | optional — default: `8`; invalid values use `8`, and values below `1` use `1` | -| `GRAPHIFY_LLM_TEMPERATURE` | Override LLM temperature for semantic extraction — e.g. `0.7`, or `none` to omit | optional — auto-omitted for o1/o3/o4/gpt-5 reasoning models | - ---- - -## Privacy - -- **Code files** — processed locally via tree-sitter. Nothing leaves your machine. A code-only corpus requires no API key — `graphify extract` runs fully offline. On a mixed repo, add `--code-only` to index just the code and skip the docs/PDFs/images that would otherwise need an LLM. -- **Video / audio** — transcribed locally with faster-whisper. Nothing leaves your machine. -- **Docs, PDFs, images** — sent to your AI assistant for semantic extraction (via the `/graphify` skill, using whatever model your IDE session runs). Headless `graphify extract` requires `GEMINI_API_KEY` / `GOOGLE_API_KEY` (Gemini), `MOONSHOT_API_KEY` (Kimi), `ANTHROPIC_API_KEY` (Claude), `OPENAI_API_KEY` (OpenAI), `DEEPSEEK_API_KEY` (DeepSeek), a running Ollama instance (`OLLAMA_BASE_URL`), AWS credentials via the standard provider chain (Bedrock - no API key needed, uses IAM), or the `claude` CLI binary (Claude Code - no API key needed, uses your Claude subscription). The `--dedup-llm` flag uses the same key. -- **Data residency** — `graphify extract` auto-detects which provider to use based on which API key is set (priority: Gemini → Kimi → Claude → OpenAI → DeepSeek → Azure → Bedrock → Ollama). For code with data-residency requirements, use `--backend ollama` (fully local) or pass an explicit `--backend` flag. Kimi (`MOONSHOT_API_KEY`) routes to Moonshot AI servers in China. -- **No telemetry**, no usage tracking, no analytics. -- **Query logging** — every `graphify query`, `graphify path`, `graphify explain`, and MCP `query_graph` call is logged to `~/.cache/graphify-queries.log` in JSON Lines format (timestamp, question, corpus, nodes returned, duration). Full subgraph responses are **not** stored by default. Set `GRAPHIFY_QUERY_LOG_DISABLE=1` to opt out, or `GRAPHIFY_QUERY_LOG=/dev/null` to silence without disabling the code path. - ---- - -## Troubleshooting - -**`graphify: command not found` after installing** -The CLI is installed but its bin directory isn't on your shell's `PATH`. Pick the fix for how you installed: -- **uv** (`uv tool install graphifyy`): the command lands in uv's tool bin dir (`~/.local/bin`), which a fresh macOS/zsh setup often doesn't have on `PATH`. Run `uv tool update-shell`, then open a new terminal. (Find the dir with `uv tool dir --bin`.) -- **pipx** (`pipx install graphifyy`): run `pipx ensurepath`, then open a new terminal. -- **pip** (`pip install graphifyy`): pip installs scripts to a user bin dir that may not be on `PATH` — add `~/Library/Python/3.x/bin` (macOS) or `~/.local/bin` (Linux) to your `PATH` in `~/.zshrc`/`~/.bashrc`, or just run `python -m graphify`. - -**`uvx graphify …` or `uv tool run graphify …` fails to resolve `graphify`** -The PyPI package is `graphifyy`; `graphify` is only the command it provides. `uv tool run` treats the first word as a *package name*, so it looks for a package called `graphify` and reports `No solution found … no versions of graphify`. Name the package explicitly: `uvx --from graphifyy graphify install` (same as `uv tool run --from graphifyy graphify install`). Or `uv tool install graphifyy` once and then call `graphify` directly. - -**`uv run --with graphifyy python -m graphify` silently runs an older install** -`uv run` uses your *system* Python, so if an older `graphifyy` also lives there (e.g. a past `pip install graphifyy`), Python can find that copy first on `sys.path` and `--with graphifyy` won't override it. It runs with no error, but you get the *old* version's behavior — e.g. env overrides like `OPENAI_BASE_URL` are silently ignored, so requests hit the default endpoint and fail with a 401 that looks like a bad key. The fingerprint is a `warning: skill is from graphify , package is ` line — that means a different install was loaded, not just a stale skill. Check which copy actually loaded: -```bash -python -c "import graphify; print(graphify.__file__)" -``` -Then run the installed command directly (it uses the uv-managed copy), or drop the stale system copy: -```bash -uvx --from graphifyy graphify extract . --backend openai # names the package explicitly -pip uninstall graphifyy # or remove the old system install -``` - -**`python -m graphify` works but `graphify` command doesn't** -Your shell's `PATH` doesn't include the bin directory the command was installed to. Prefer `uv tool install` / `pipx install` over plain `pip`, then run `uv tool update-shell` / `pipx ensurepath` and open a new terminal (see the install notes above). - -**`/graphify .` causes "path not recognized" in PowerShell** -PowerShell treats a leading `/` as a path separator. Use `graphify .` (no slash) on Windows. - -**Graph has fewer nodes after `--update` or rebuild** -If a refactor deleted files, the old nodes linger. Pass `--force` (or set `GRAPHIFY_FORCE=1`) to overwrite even when the rebuild has fewer nodes. - -**`extract` exits with "extraction was incomplete ... refusing to overwrite"** -When an extraction pass crashes or a walk can't fully read the corpus, the run would be smaller than a complete one, so `graphify extract` refuses to overwrite a larger existing graph with the partial result (protecting your `graph.json`). Fix the underlying failure and re-run, or pass `--allow-partial` to overwrite anyway. - -**Graph has duplicate nodes for the same entity (ghost duplicates)** -Ghost duplicates (same symbol appearing twice — once from AST extraction with a source location, once from semantic extraction without) are now automatically merged at build time. If you see this in a graph built before v0.8.33, run a full re-extract to clean up: -```bash -graphify extract . --force -``` - -**Ollama runs out of VRAM / context window exceeded** -The KV-cache window is auto-sized but may be too large for your GPU. Reduce it: -```bash -GRAPHIFY_OLLAMA_NUM_CTX=8192 graphify extract ./docs --backend ollama --token-budget 4000 -``` - -**`LLM returned invalid JSON` / `Unterminated string` warnings** -The model's JSON response hit its output-token limit and was cut off mid-string. graphify auto-recovers (it splits the chunk and re-extracts the halves, and an oversized single document is first sliced at heading/paragraph boundaries so the whole file is still covered), so these warnings are noisy but not data loss. To reduce the churn, raise the output cap or shrink each chunk's output: -```bash -GRAPHIFY_MAX_OUTPUT_TOKENS=16384 graphify extract . --mode deep # lift the cap -graphify extract . --mode deep --token-budget 4000 # smaller input chunks -> smaller output -``` -With a cloud gateway like OpenRouter, prefer `--backend openai` (set `OPENAI_BASE_URL`) over the Ollama shim — it's a cleaner OpenAI-compatible path. If the model has its own max-output ceiling, lowering `--token-budget` is the reliable lever. - -**Graph HTML is too large to open in a browser (>5000 nodes)** -Skip HTML generation and use the JSON directly: -```bash -graphify cluster-only ./my-project --no-viz -graphify query "..." -``` - -**`graph.json` has conflict markers after two devs commit at once** -Run `graphify hook install` — it sets up a git merge driver that union-merges `graph.json` automatically so conflicts never happen. - -**Extraction returns empty nodes/edges for docs or PDFs** -Docs, PDFs, and images require an LLM call — code-only corpora need no key. Check that your API key is set and the backend is correct: -```bash -ANTHROPIC_API_KEY=sk-... graphify extract ./docs --backend claude -``` - -**Skill version mismatch warning in your IDE** -Your installed graphify version is different from the skill file. Update: -```bash -uv tool upgrade graphifyy -graphify install # overwrites the skill file -``` - -**Claude Code prompt cache invalidated after every `graphify extract`** -Graphify writes output files (`graph.json`, `graphify-out/`) into the workspace. If those paths aren't ignored, every write invalidates Claude Code's prompt cache, forcing a full re-upload at cache-write rates on the next turn. Add them to `.claudeignore`: -```text -# .claudeignore -graph.json -graphify-out/ -``` - ---- - -## Full command reference - -``` -/graphify # run on current directory -/graphify ./raw # run on a specific folder -/graphify ./raw --mode deep # more aggressive relationship extraction -graphify extract ./raw --code-only # index code only — local AST, no API key (skips docs/PDFs/images); an `extract` flag, not a skill flag -/graphify ./raw --update # re-extract only changed files -/graphify ./raw --directed # preserve edge direction -/graphify ./raw --cluster-only # rerun clustering on existing graph -/graphify ./raw --no-viz # skip HTML visualization -/graphify ./raw --obsidian # generate Obsidian vault -/graphify ./raw --obsidian --obsidian-dir ~/vault # write into an existing vault (never overwrites your own notes or .obsidian config) -/graphify ./raw --wiki # build agent-crawlable markdown wiki -/graphify ./raw --svg # export graph.svg -/graphify ./raw --graphml # export for Gephi / yEd -/graphify ./raw --neo4j # generate cypher.txt for Neo4j -/graphify ./raw --neo4j-push bolt://localhost:7687 -/graphify ./raw --falkordb # generate cypher.txt for FalkorDB -/graphify ./raw --falkordb-push falkordb://localhost:6379 -/graphify ./raw --watch # auto-sync as files change -/graphify ./raw --mcp # start MCP stdio server - -/graphify add https://arxiv.org/abs/1706.03762 -/graphify add -/graphify add https://... --author "Name" --contributor "Name" - -/graphify query "what connects attention to the optimizer?" -/graphify query "..." --dfs --budget 1500 -/graphify path "DigestAuth" "Response" -/graphify explain "SwinTransformer" - -graphify save-result --question "Q" --answer "A" --nodes Foo Bar --outcome useful # record how a Q&A turned out (work memory; outcome ∈ useful|dead_end|corrected) -graphify reflect # aggregate graphify-out/memory/ outcomes into reflections/LESSONS.md -graphify reflect --if-stale # no-op when LESSONS.md is already newer than every input (cheap to run each session) -graphify reflect --out docs/LESSONS.md # write the lessons doc somewhere else -graphify reflect --graph graphify-out/graph.json # group lessons by community + write the work-memory overlay (.graphify_learning.json) - # the overlay tags nodes preferred/tentative/contested (recency-weighted, with provenance); - # graphify explain / query then show a "Lesson:" hint, flagged "code changed — re-verify" when the source moved on - -graphify uninstall # remove from all platforms in one shot -graphify uninstall --purge # also delete graphify-out/ -graphify uninstall --project --platform codex # remove project-scoped install files only - -graphify hook install # post-commit + post-checkout hooks -graphify hook uninstall -graphify hook status - -# always-on assistant instructions - platform-specific -graphify claude install # CLAUDE.md + PreToolUse hook (Claude Code) -graphify claude uninstall -graphify codebuddy install # CODEBUDDY.md + PreToolUse hook (CodeBuddy) -graphify codebuddy uninstall -graphify codex install # AGENTS.md + PreToolUse hook in .codex/hooks.json (Codex) -graphify opencode install # AGENTS.md + tool.execute.before plugin (OpenCode) -graphify kilo install # native Kilo skill + /graphify command + AGENTS.md + .kilo plugin -graphify kilo uninstall -graphify cursor install # .cursor/rules/graphify.mdc (Cursor) -graphify cursor uninstall -graphify gemini install # GEMINI.md + BeforeTool hook (Gemini CLI) -graphify gemini uninstall -graphify copilot install # skill file (GitHub Copilot CLI) -graphify copilot uninstall -graphify aider install # AGENTS.md (Aider) -graphify aider uninstall -graphify claw install # AGENTS.md (OpenClaw) -graphify claw uninstall -graphify droid install # AGENTS.md (Factory Droid) -graphify droid uninstall -graphify trae install # AGENTS.md (Trae) -graphify trae uninstall -graphify trae-cn install # AGENTS.md (Trae CN) -graphify trae-cn uninstall -graphify hermes install # AGENTS.md + ~/.hermes/skills/ (Hermes) -graphify hermes uninstall -graphify amp install # skill file (Amp) -graphify amp uninstall -graphify agents install # ~/.agents/skills/ + AGENTS.md (cross-framework; alias: graphify skills) -graphify agents uninstall -graphify kiro install # .kiro/skills/ + .kiro/steering/graphify.md (Kiro IDE/CLI) -graphify kiro uninstall -graphify pi install # skill file (Pi coding agent) -graphify pi uninstall -graphify devin install # skill file + .windsurf/rules/graphify.md (Devin CLI) -graphify devin uninstall -graphify antigravity install # .agents/rules + .agents/workflows (Google Antigravity) -graphify antigravity uninstall - -graphify extract ./docs # headless LLM extraction for CI (no IDE needed) -graphify extract ./docs --backend gemini # explicit backend: gemini, kimi, claude, openai, deepseek, ollama, bedrock, or claude-cli -graphify extract ./docs --backend gemini --model gemini-3.1-pro-preview -graphify extract ./docs --backend ollama # local Ollama (set OLLAMA_BASE_URL / OLLAMA_MODEL) - no API key needed for loopback -OPENAI_BASE_URL=http://localhost:8080/v1 OPENAI_MODEL=my-model graphify extract ./docs --backend openai # any OpenAI-compatible server (llama.cpp, vLLM, LM Studio) -ANTHROPIC_BASE_URL=http://localhost:4000 ANTHROPIC_MODEL=my-model graphify extract ./docs --backend claude # any Anthropic-compatible endpoint (LiteLLM proxy, gateways) -GRAPHIFY_OLLAMA_NUM_CTX=32768 graphify extract ./docs --backend ollama # override KV-cache window (auto-sized by default) -GRAPHIFY_OLLAMA_KEEP_ALIVE=0 graphify extract ./docs --backend ollama # unload model after each chunk (saves VRAM on small GPUs) -graphify extract ./docs --backend bedrock # AWS Bedrock via IAM - no API key, uses AWS credential chain -graphify extract ./docs --backend claude-cli # route through Claude Code CLI - no API key, uses your Claude subscription -graphify extract ./docs --backend azure # Azure OpenAI (set AZURE_OPENAI_API_KEY + AZURE_OPENAI_ENDPOINT) -graphify extract ./docs --max-workers 16 # AST parallelism (also GRAPHIFY_MAX_WORKERS) -graphify extract --postgres "postgresql://user:pass@host/db" # introspect live PostgreSQL schema directly -graphify extract ./my-workspace --cargo # introspect Rust Cargo workspace dependencies directly -graphify extract ./docs --token-budget 30000 # smaller semantic chunks for local/small models -graphify extract ./docs --max-concurrency 2 # fewer parallel LLM calls (useful for local inference) -graphify extract ./docs --api-timeout 900 # longer HTTP timeout for slow local models (default 600s) -graphify extract ./docs --google-workspace # export .gdoc/.gsheet/.gslides via gws before extraction -graphify extract ./src --no-gitignore # include git-ignored source; still honor .graphifyignore -graphify extract ./docs --mode deep # richer semantic extraction via extended system prompt -graphify extract ./docs --no-cluster # raw extraction only, skip clustering -graphify extract ./docs --timing # print per-stage wall-clock timings to stderr (also works on cluster-only) -graphify extract ./docs --force # overwrite graph.json even if new graph has fewer nodes (use after refactors or to clear ghost duplicates) -graphify extract ./docs --dedup-llm # LLM tiebreaker for ambiguous entity pairs (uses same API key) -graphify extract ./docs --global --as myrepo # extract and register into the cross-project global graph -GRAPHIFY_MAX_OUTPUT_TOKENS=32768 graphify extract ./docs --backend claude # raise output cap for dense corpora - -graphify export callflow-html # graphify-out/-callflow.html -graphify export callflow-html --max-sections 8 # cap generated architecture sections -graphify export callflow-html --output docs/arch.html -graphify export callflow-html ./some-repo/graphify-out - -graphify global add graphify-out/graph.json --as myrepo # register a project graph into ~/.graphify/global-graph.json -graphify global remove myrepo # remove a project from the global graph -graphify global list # show all registered repos + node/edge counts -graphify global path # print path to the global graph file - -graphify prs # PR dashboard: CI, review, worktree, graph impact -graphify prs 42 # deep dive on PR #42 -graphify prs --triage # AI triage ranking (auto-detects backend from env) -graphify prs --worktrees # worktree → branch → PR mapping -graphify prs --conflicts # PRs sharing graph communities (merge-order risk) -graphify prs --base main # filter to PRs targeting a specific base branch -graphify prs --repo owner/repo # run against a different GitHub repo -GRAPHIFY_TRIAGE_BACKEND=kimi graphify prs --triage # use a specific backend for triage - -graphify clone https://github.com/karpathy/nanoGPT -graphify merge-graphs a.json b.json --out merged.json -graphify --version # print installed version -graphify watch ./src -graphify check-update ./src -graphify update ./src -graphify update ./src --no-cluster # skip reclustering, write raw AST graph only -graphify update ./src --force # overwrite even if new graph has fewer nodes -graphify cluster-only ./my-project -graphify cluster-only ./my-project --graph path/to/graph.json # custom graph location -graphify cluster-only ./my-project --max-concurrency 16 --batch-size 200 # parallel community labeling (large graphs) -graphify cluster-only ./my-project --resolution 1.5 # more, smaller communities -graphify cluster-only ./my-project --exclude-hubs 99 # exclude p99 degree nodes from partitioning -graphify cluster-only ./my-project --no-label # keep "Community N" placeholders -graphify cluster-only ./my-project --backend=gemini # backend for community naming -graphify cluster-only ./my-project --backend=gemini --model gemini-2.5-pro # specific model -graphify label ./my-project # (re)name communities with the configured backend -graphify label ./my-project --backend=openai --model gpt-4o # force a specific backend and model -``` - -> **Community names:** inside an agent (Claude Code, Gemini CLI) the agent names communities itself. When you run the bare CLI, `cluster-only` auto-names them with the configured backend (built-in or custom OpenAI-compatible provider) — pass `--no-label` to keep `Community N`, or run `graphify label` to (re)generate names on demand. - ---- - -## Learn more - -- [How it works](docs/how-it-works.md) — the extraction pipeline, community detection, confidence scoring, benchmarks -- [ARCHITECTURE.md](ARCHITECTURE.md) — module breakdown, how to add a language -- [Optional integrations](docs/docker-mcp-sqlite.md) — Docker MCP Toolkit + SQLite -- [The Memory Layer](https://safishamsi.gumroad.com/l/qetvlo) — the book on the ideas behind graphify, the architecture end to end - ---- - -## graphify Enterprise - -[**graphify Enterprise**](https://graphify.com) is the always-on layer built on top of graphify — it applies the same graph approach to your entire working context: meetings, files, docs, and code, updating continuously in the background. - -Built for people and teams whose work lives across hundreds of conversations and documents they can never fully reconstruct. - -**[Join the waitlist at graphify.com](https://graphify.com).** Free trial launching soon. - ---- - -
-Contributing - -### Development setup - -The project uses [uv](https://docs.astral.sh/uv/) for dev workflow. Install it once, then: - -```bash -git clone https://github.com/safishamsi/graphify.git -cd graphify -git checkout v8 # active development branch - -# Create the project venv and install graphify + all extras + the dev group -# (pytest). uv installs the dev dependency group by default; pass --no-dev to -# skip it. -uv sync --all-extras -``` - -Verify the editable install: -```bash -uv run graphify --version -uv run python -c "import graphify; print(graphify.__file__)" -``` - -### Running tests - -```bash -uv run pytest tests/ -q # run the full suite -uv run pytest tests/test_extract.py -q # one module -uv run pytest tests/ -q -k "python" # filter by name -``` - -> macOS note: the test suite includes both `sample.f90` and `sample.F90` fixtures. These collide on case-insensitive HFS+ / APFS file systems. Run on Linux or in a Docker container if you need to test both Fortran variants simultaneously. - -### Git workflow - -- Active development happens on the `v8` branch. -- Commit style: `fix: ` / `feat: ` / `docs: ` -- Before opening a PR, run `uv run pytest tests/ -q` and confirm it passes. -- Add a fixture file to `tests/fixtures/` and tests to `tests/test_languages.py` for any new language extractor. - -### What to contribute - -**Worked examples** are the most useful contribution. Run `/graphify` on a real corpus, save the output to `worked/{slug}/`, write an honest `review.md` covering what the graph got right and wrong, and open a PR. - -**Extraction bugs** — open an issue with the input file, the cache entry (`graphify-out/cache/`), and what was missed or wrong. - -See [ARCHITECTURE.md](ARCHITECTURE.md) for module responsibilities and how to add a language. - -
- ---- - -## Community and links - -

- Website - Discord - X - Sponsor - The Memory Layer -

+- Graphify version: 0.9.40 (unreleased) +- Supplied ZIP SHA-256: d8470c798610624ceb24bd45775a5046f8151b61d9f734c173a19a860851b342 +- Supplied upstream tree: 2a912ac905b5cc8a7a7518b5a4f3c3879ac3ae10 +- Proposed merged tree: 60e69faf6d955d74a314fda22712f218b88873e6 diff --git a/cache.py b/cache.py new file mode 100644 index 000000000..06da2d754 --- /dev/null +++ b/cache.py @@ -0,0 +1,927 @@ +"""Entity deduplication pipeline for graphify knowledge graphs. + +Pipeline: exact normalization → entropy gate → MinHash/LSH blocking → +Jaro-Winkler verification → same-community boost → union-find merge. +""" +from __future__ import annotations +import math +import re +import sys +import unicodedata +from collections import defaultdict +from pathlib import Path + +from graphify._minhash import MinHash, MinHashLSH +from graphify.paths import resolve_path as _resolve_path +from rapidfuzz.distance import DamerauLevenshtein, Jaro, JaroWinkler + + +# ── helpers ─────────────────────────────────────────────────────────────────── + +def _norm(label: str | None) -> str: + """Lowercase + collapse non-alphanumeric runs to space (Unicode-aware).""" + if not isinstance(label, str): + label = "" if label is None else str(label) + label = unicodedata.normalize("NFKC", label) + return re.sub(r"[\W_]+", " ", label.casefold(), flags=re.UNICODE).strip() + + +def _entropy(label: str) -> float: + """Shannon entropy in bits/char of the normalised label.""" + s = _norm(label) + if not s: + return 0.0 + freq: dict[str, int] = defaultdict(int) + for ch in s: + freq[ch] += 1 + n = len(s) + return -sum((c / n) * math.log2(c / n) for c in freq.values()) + + +def _shingles(text: str, k: int = 3) -> set[str]: + """Return k-gram character shingles of text.""" + if len(text) < k: + return {text} + return {text[i : i + k] for i in range(len(text) - k + 1)} + + +def _make_minhash(text: str, num_perm: int = 128) -> MinHash: + # Strip spaces so "graph extractor" and "graphextractor" share shingles + m = MinHash(num_perm=num_perm) + for shingle in _shingles(text.replace(" ", "")): + m.update(shingle.encode("utf-8")) + return m + + +# Matches labels whose trailing token is a version/variant suffix: +# digits optionally followed by letters (chip SKUs: ASR1603, M1, Cortex-A55) +# or 2+ letters (codename revisions: cranelr vs cranel). +# Requires the stem to end in a letter so plain words don't accidentally match. +_VARIANT_SUFFIX = re.compile(r"^(.*[a-z])([0-9]+[a-z]*|[a-z]{2,})$") + + +def _is_variant_pair(a: str, b: str) -> bool: + """True if a and b are sibling model/SKU variants (same stem, different suffix). + + Only applied to short labels (< 12 chars); long labels go through JW normally. + """ + if a == b: + return False + if max(len(a), len(b)) >= 12: + return False + ma, mb = _VARIANT_SUFFIX.match(a), _VARIANT_SUFFIX.match(b) + if not (ma and mb): + return False + return ma.group(1) == mb.group(1) and ma.group(2) != mb.group(2) + + +def _short_label_blocked(a: str, b: str, jw_score: float) -> bool: + """Block fuzzy merge for short labels unless it's a same-length single-char substitution. + + Insertions/deletions on short strings (cranel/cranelr, M1/M1 Pro) produce + high Jaro-Winkler scores due to the prefix bonus but are almost never true + duplicates — they're abbreviations or variants. + """ + if max(len(a), len(b)) >= 12: + return False + from rapidfuzz.distance import DamerauLevenshtein + # Allow only same-length single-char substitutions (true typos like "Extractor"/"Extractar"). + # Block length-differing pairs regardless of score. + if jw_score >= 97.0 and len(a) == len(b) and DamerauLevenshtein.distance(a, b) <= 1: + return False + return True + + +_DIGIT_RUN = re.compile(r"\d+") + + +def _numeric_tokens_differ(a: str, b: str) -> bool: + """True when two labels carry different embedded numbers (#1284). + + Long labels that differ only in their digit runs ("ADR 0011 §D5" vs + "ADR 0013 D4", "3.1 Product Goals" vs "1.1 Product Goals", "block3" vs + "block13", "40%+ retention" vs "<20% retention") are numbered/versioned + siblings, not duplicates -- but the long shared boilerplate keeps + Jaro-Winkler above _MERGE_THRESHOLD, and _is_variant_pair only covers + short trailing suffixes. Digit runs are compared as multisets with + leading zeros stripped, so zero-padding ("09" vs "9") does not count as + a difference. (String comparison, not int(): a pathological label with a + >4300-digit run would crash int() on Python's conversion limit.) Labels + with identical numbers, or none at all, are unaffected. + """ + if a == b: + return False + return sorted(t.lstrip("0") or "0" for t in _DIGIT_RUN.findall(a)) != \ + sorted(t.lstrip("0") or "0" for t in _DIGIT_RUN.findall(b)) + + +# Function words. A restatement of one entity is what inserts or swaps these +# ("export a read-only ..." vs "export the read-only ..."); a content word +# carries the entity's identity and swapping one names something else. +_STOPWORDS = frozenset({ + "a", "an", "the", "and", "or", "of", "for", "to", "in", "on", "at", "by", + "with", "from", "as", "is", "are", "be", "this", "that", "its", +}) + + +def _same_word_variant(x: str, y: str) -> bool: + """True when tokens x and y read as one word misspelt, not two words (#2576). + + A same-length pair within one substitution/transposition is a typo + ("manager"/"nanager") -- the same rationale _short_label_blocked applies + to whole short labels, and unlike Jaro-Winkler it holds at position 0, + where the prefix bonus gives no help (JW scores "manager"/"nanager" at + 84.92, below threshold, yet it is as much a typo as "managr"). Below 6 + chars JW cannot separate two words from a typo ("pane"/"plane" scores + 94.0), so short length-differing pairs never read as variants. Longer + pairs fall back to Jaro-Winkler on the merge threshold, so + "manager"/"managr" (97.14) still reads as one word. Accepted trade, per + the never-merge-two-distinct-entities bar: "colour"/"color" (5 chars, + lengths differ) now reads as two words and stays unmerged -- a spelling + variant kept separate beats a fabricated merge. + """ + if len(x) == len(y) and DamerauLevenshtein.distance(x, y) <= 1: + return True # same-length 1-sub/transposition = typo, even at position 0 + if min(len(x), len(y)) < 6: + return False # short tokens: JW can't separate pane/plane (94.0) from a typo + return JaroWinkler.normalized_similarity(x, y) * 100 >= _MERGE_THRESHOLD + + +def _content_token_swap(a: str, b: str) -> bool: + """True when two equal-token-count labels differ in at least one swapped + content word rather than only typos or function words (#2576, adopted + from @wilyan09007's PR #2587 and generalized from exactly-one to any + number of differing positions). + + Whole-string scoring cannot separate a legit restatement from a + distinguishing-token swap: both edit one short run in the middle of a long + shared string, so both land in the same Jaro band (#1243). Which token + differs does separate them. Structured prose names sibling sections from a + template ("Asset Contribution Flow" / "Asset Consumption Flow", four + consecutive headings of one operations doc), and those siblings are densest + inside a single file -- exactly where Jaro-Winkler's prefix bonus still + applies, and where the shared affixes it rewards are boilerplate. + + Each same-position differing pair is judged on its own: a function word on + either side is what a restatement swaps, a _same_word_variant pair is one + word misspelt, and anything else is a distinct content word naming a + different entity -- one such pair blocks the merge. A restatement differs + only in stopwords/typos at every position; a template sibling differs in + at least one distinct content word ("... Contribution Flow Handler" vs + "... Consumption Flows Handler" blocks on either position). Pairs with + different token counts are left to the prefix-extension guard (#1201) and + whole-label scoring. Known gap, out of scope here: fused camelCase labels + ("AssetContributionFlow" vs "AssetConsumptionFlow") normalize to single + tokens whose only differing "position" is the whole label, so this guard + reduces to whole-token _same_word_variant and long fused pairs can still + clear the JW fallback. + """ + tokens_a, tokens_b = a.split(), b.split() + if len(tokens_a) != len(tokens_b): + return False + for x, y in zip(tokens_a, tokens_b): + if x == y: + continue + if x in _STOPWORDS or y in _STOPWORDS: + continue # restatement: a function word swapped in or out + if _same_word_variant(x, y): + continue # one word misspelt/inflected, not a different word + return True + return False + + +# file_type values whose identity is anchored to their source location, not +# their label text. Like code (#1205), these must not be label-merged across +# files: rationale = module/class docstrings, document = headings/positional +# content. `concept` is intentionally excluded -- it is the type meant to unify +# across files (protected from over-merge by the numeric/Jaro guards instead). +_FILE_ANCHORED_NONCODE = frozenset({"rationale", "document"}) + + +def _crossfile_fileanchored_blocked(node: dict, neighbor: dict) -> bool: + """Block label-based merging of file-anchored non-code nodes across files (#1284). + + rationale/document nodes are docstring- and heading-derived and as + file-anchored as the code they describe (#1205's reasoning, one layer up): + parallel modules carry near-identical boilerplate ("Django app config for + apps.. No business logic here...") that differs by one word and sails + past the JW threshold. Same-file duplicates of these types may still merge. + """ + if (node.get("file_type") not in _FILE_ANCHORED_NONCODE + and neighbor.get("file_type") not in _FILE_ANCHORED_NONCODE): + return False + return (node.get("source_file") or "") != (neighbor.get("source_file") or "") + + +# ── union-find ──────────────────────────────────────────────────────────────── + +class _UF: + def __init__(self) -> None: + self._parent: dict[str, str] = {} + + def find(self, x: str) -> str: + self._parent.setdefault(x, x) + while self._parent[x] != x: + self._parent[x] = self._parent[self._parent[x]] + x = self._parent[x] + return x + + def union(self, x: str, y: str) -> None: + self._parent.setdefault(x, x) + self._parent.setdefault(y, y) + rx, ry = self.find(x), self.find(y) + if rx != ry: + self._parent[ry] = rx + + def components(self) -> dict[str, list[str]]: + groups: dict[str, list[str]] = defaultdict(list) + for x in self._parent: + groups[self.find(x)].append(x) + return dict(groups) + + +# ── constants ───────────────────────────────────────────────────────────────── + +_ENTROPY_THRESHOLD = 2.5 +_LSH_THRESHOLD = 0.7 +_MERGE_THRESHOLD = 92.0 # rapidfuzz normalized_similarity * 100 +_COMMUNITY_BOOST = 5.0 # score bonus when both nodes share community +_NUM_PERM = 128 +_CHUNK_SUFFIX = re.compile(r"_c\d+$") + + +def _is_code(node: dict) -> bool: + """True for AST-extracted code symbols. + + Code-node identity is the node ID (which already encodes the fully + qualified path: module/class/symbol). The label is only a display name + (e.g. a bare ``.draw()`` method name, or a function name shared by two + parallel backends), so label-based merging conflates distinct symbols + (#1205). Genuine duplicates — the same symbol re-extracted — share an ID + and are already collapsed by the exact-ID ``seen_ids`` pre-dedup above, + so code never needs label-based merging. + """ + return node.get("file_type") == "code" + + +# ── ID collisions ───────────────────────────────────────────────────────────── + +_ID_SEGMENT = re.compile(r"[^a-z0-9]+") +_EXTENSION = re.compile(r"\.[^./]+$") + + +def _id_prefixes(source_file: str) -> set[str]: + """The ID prefixes a node extracted from ``source_file`` may legitimately mint. + + An ID is ``_``, where the path is the extension-stripped source + path, each segment slugified and joined with ``_``. Every trailing slice of the + path counts as a prefix: the stored path may be absolute or repo-relative, and + graphs built under the pre-#1504 scheme keyed off the bare filename stem. + """ + stem = _EXTENSION.sub("", source_file.replace("\\", "/")) + segments = [s for s in (_ID_SEGMENT.sub("_", p.casefold()).strip("_") + for p in stem.split("/")) if s] + return {"_".join(segments[i:]) for i in range(len(segments))} + + +def _defines_id(node: dict) -> bool: + """True when the node's own source_file is the file its ID encodes. + + A doc that *references* an entity mints the ID of the entity's own file, not one + derived from the doc's path — so the referencing node collides with the defining + node by construction. This separates the two: the definer owns the ID. + """ + nid = node.get("id") or "" + source_file = node.get("source_file") or "" + if not nid or not source_file: + return False + # `nid == prefix` covers a bare file-level node whose id is exactly the + # slugified path with no `_entity` suffix (a semantic node for the file + # itself); `startswith(prefix + "_")` covers the usual `_` id. + return any(nid == prefix or nid.startswith(f"{prefix}_") + for prefix in _id_prefixes(source_file)) + + +# Path-segment lifecycle markers used by _collision_rank (#2532). Lower penalty +# wins. Without them, pure lexical source_file order makes ``plans/_done/…`` +# beat ``plans/in-progress/…`` because "_" < "i" in ASCII. Active-vs-archived +# marker idea by @michaelxer (#2540); matched against ROOT-RELATIVE directory +# segments only, so a checkout directory that happens to be named ``wip`` or +# ``done`` never leaks into the ranking. +_ACTIVE_PATH_SEGMENTS = frozenset({ + "in-progress", + "in_progress", + "active", + "current", + "wip", +}) +_ARCHIVED_PATH_SEGMENTS = frozenset({ + "_done", + "done", + "archive", + "archived", + "backup", + "bak", + "old", + "attic", + "graveyard", + "completed", +}) + + +def _lifecycle_penalty(rank_path: str) -> int: + """0 for active/in-progress paths, 2 for archived/done paths, 1 otherwise. + + Judged on the DIRECTORY segments of the root-relative rank path — a file + literally named ``done.md`` is not a marker. Among mixed markers the best + (lowest) score wins so an active segment is not drowned out by an unrelated + archive directory higher in the tree (#2532). + """ + segments = [s for s in rank_path.casefold().split("/") if s] + marked = [ + 0 if s in _ACTIVE_PATH_SEGMENTS else 2 + for s in segments[:-1] # directories only, never the basename + if s in _ACTIVE_PATH_SEGMENTS or s in _ARCHIVED_PATH_SEGMENTS + ] + return min(marked) if marked else 1 + + +def _rank_path(source_file: str, root: Path | None) -> str: + """The root-relative form of ``source_file`` used for collision ranking. + + Mirrors ``_source_key`` in extractors/resolution.py: with a scan root, an + absolute stored path and its repo-relative twin rank identically, and the + checkout location's own segments never participate (#2532). Without a root + (or when relativizing fails) the normalized stored path is used as-is. + """ + normalized = source_file.replace("\\", "/") + if root is not None and normalized: + try: + return _resolve_path(normalized).relative_to(root).as_posix() + except Exception: + pass + return normalized + + +def _collision_rank(node: dict, root: Path | None = None) -> tuple: + """A total order for choosing the survivor of an ID collision, independent of + the order the colliding nodes arrive in. + + The winner is the node with the SMALLEST rank. A node whose ``source_file`` + defines the ID always outranks a mere reference; among equally-(non-)defining + nodes an active/in-progress path outranks an archived/done one (#2532); then + it prefers the shorter, more canonical label over a longer qualified variant, + then breaks any remaining tie lexically on label and finally on the REVERSED + segments of the root-relative path. Basename-first comparison decides two + in-repo colliders by segments present in both path forms, so absolute and + repo-relative spellings of the same layout order identically — fully + deterministic regardless of arrival order (#1851) or checkout location. + """ + label = node.get("label") or "" + rank_path = _rank_path(node.get("source_file") or "", root) + return ( + not _defines_id(node), # definers (False) sort before references (True) + _lifecycle_penalty(rank_path), # active paths beat archived ones (#2532) + len(label), # shorter, more canonical label first + label, # lexical tiebreak + tuple(reversed([s for s in rank_path.split("/") if s and s != "."])), + ) + + +def _same_source_entity(survivor: dict, duplicate: dict) -> bool: + """True when exact-ID records came from the same source file. + + Exact IDs can also collide across files through references or slugged-path + ambiguity (#1504). Keep those records isolated rather than importing + attributes whose provenance belongs to another file. + """ + keep_file = survivor.get("source_file") or "" + lose_file = duplicate.get("source_file") or "" + # Require a non-empty source_file: two provenance-less records ("" == "") + # are NOT proof of the same symbol (#1178), and merging their attributes + # would be a cross-pollination bug in the opposite direction (#2091 review). + return bool(keep_file) and keep_file == lose_file + + +def _merge_missing_attributes(survivor: dict, duplicate: dict) -> dict: + """Fill the survivor's absent/None attributes from a same-source duplicate, + without overriding values the survivor already has (#2091).""" + merged = dict(survivor) + for key, value in duplicate.items(): + # Never inherit a provenance tag from a dropped record: a false + # _origin="ast" on an LLM survivor is read as an authority signal by the + # ghost-merge (#2068) and watch deletion logic (#2091 review). + if key == "_origin": + continue + if value is None: + continue + # Treat an explicit None on the survivor as absent — the codebase emits + # `source_location: None`, and that is exactly the attribute #2091 loses. + if merged.get(key) is None: + merged[key] = value + return merged + + +def _report_id_collision(nid: str, survivor: dict, losers: list[dict]) -> None: + """Report an ID collision in proportion to what dropping the loser actually costs. + + Cross-reference to a defining node: the structural entity and its edges survive; + foreign-file attributes stay isolated, so no collision warning is needed. Same + file, different labels: the extractor emitted two labels for one entity and one is + discarded — note it. Two files that both encode this ID: they are distinct entities + and one is genuinely lost — warn, and point at the extraction split that keeps them + apart (#1504). + """ + keep_file = survivor.get("source_file") or "" + keep_label = survivor.get("label") or "" + for loser in losers: + lose_file = loser.get("source_file") or "" + lose_label = loser.get("label") or "" + if lose_file == keep_file: + if _norm(lose_label) != _norm(keep_label): + print( + f"[graphify] note: node '{nid}' was extracted twice from " + f"'{keep_file}' under different labels — keeping '{keep_label}', " + f"dropping '{lose_label}'.", + file=sys.stderr, + ) + elif _defines_id(survivor) and not _defines_id(loser): + continue # the loser only references the entity the survivor defines + else: + print( + f"[graphify] WARNING: node '{nid}' is minted by two different files — " + f"keeping '{keep_label}' from '{keep_file}', dropping '{lose_label}' " + f"from '{lose_file}'. An ID is derived from the source path plus the " + f"entity name, so this one does not identify a single entity and the " + f"dropped node is lost. To keep them distinct, run 'graphify extract' " + f"per subfolder and merge with 'graphify merge-graphs'.", + file=sys.stderr, + ) + + +# ── main entry point ────────────────────────────────────────────────────────── + +def deduplicate_entities( + nodes: list[dict], + edges: list[dict], + *, + communities: dict[str, int], + dedup_llm_backend: str | None = None, + root: str | Path | None = None, +) -> tuple[list[dict], list[dict]]: + """Deduplicate near-identical entities in a knowledge graph. + + Args: + nodes: list of node dicts with at minimum {"id": str, "label": str} + edges: list of edge dicts with {"source": str, "target": str, ...} + communities: mapping of node_id -> community_id (from cluster()) + dedup_llm_backend: if set, use LLM to resolve ambiguous pairs + root: scan root; ID-collision ranking judges source paths relative to + it so path form and checkout location cannot flip the survivor (#2532) + + Returns: + (deduped_nodes, deduped_edges) with edges rewired to survivors + """ + # Guard: cross-project dedup is not supported — nodes from different repos + # share label names by coincidence and must never be merged by string similarity. + # If you need to dedup a global graph, run deduplicate_entities per-repo first. + repos_seen = {n.get("repo") for n in nodes if n.get("repo")} + if len(repos_seen) > 1: + raise ValueError( + f"deduplicate_entities: nodes span multiple repos {sorted(repos_seen)!r}. " + f"Cross-project dedup is disabled — run dedup per-repo before merging." + ) + + if len(nodes) <= 1: + return nodes, edges + + # Resolve the scan root once: _collision_rank ranks each node's source_file + # relative to it, so an absolute stored path and its repo-relative twin rank + # identically and lifecycle markers in the checkout location's own segments + # cannot flip the survivor (#2532). + try: + root_resolved: Path | None = _resolve_path(root) if root else None + except Exception: + root_resolved = None + + # Pre-deduplicate: one node per ID. The survivor is the node that *defines* the + # ID (its source_file is the file the ID encodes), not merely the first seen — + # otherwise chunk order decides whether an entity keeps its own attributes or a + # passing cross-reference's. Missing attributes from same-source records are + # retained so AST structure and semantic enrichment can coexist (#2091). + # Genuine cross-file ID collisions stay isolated and are reported below (#1504). + seen_ids: dict[str, dict] = {} + dropped: dict[str, list[dict]] = defaultdict(list) + for node in nodes: + nid = node.get("id", "") + if not nid: + continue + incumbent = seen_ids.get(nid) + if incumbent is None: + seen_ids[nid] = node + elif _collision_rank(node, root_resolved) < _collision_rank(incumbent, root_resolved): + # Smallest-ranked node wins; the min over a total order is independent + # of the order nodes arrive in, so the survivor no longer depends on + # chunk ordering (#1851). + seen_ids[nid] = node + dropped[nid].append(incumbent) + else: + dropped[nid].append(node) + + # Gap-fill each survivor from its SAME-SOURCE losers, applied in deterministic + # _collision_rank order (best loser first). Merging here — not incrementally in + # the loop above — keeps the merged attributes independent of chunk arrival + # order with 3+ colliding records, preserving the #1851 order-independence + # contract (#2091 review). + for nid, losers in dropped.items(): + survivor = seen_ids[nid] + same_source = sorted( + (l for l in losers if _same_source_entity(survivor, l)), + key=lambda l: _collision_rank(l, root_resolved), + ) + for loser in same_source: + survivor = _merge_missing_attributes(survivor, loser) + seen_ids[nid] = survivor + + for nid, losers in dropped.items(): + _report_id_collision(nid, seen_ids[nid], losers) + + unique_nodes = list(seen_ids.values()) + + if len(unique_nodes) <= 1: + return unique_nodes, edges + + # ── pass 1: exact normalization ─────────────────────────────────────────── + norm_to_nodes: dict[str, list[dict]] = defaultdict(list) + for node in unique_nodes: + # Code symbols are keyed by ID, never by label — skip them entirely so + # distinct same-named symbols are never merged by string similarity (#1205). + if _is_code(node): + continue + key = _norm(node.get("label", node.get("id", ""))) + if key: + norm_to_nodes[key].append(node) + + uf = _UF() + exact_merges = 0 + for key, group in norm_to_nodes.items(): + if len(group) <= 1: + continue + # Partition by source_file — same-file exact matches always merge here. + # Cross-file exact matches are handled just below, gated to `concept` + # nodes only: Pass 2 cannot form them because its candidate list keeps a + # single node per normalized label (#2182). + by_file: dict[str, list[dict]] = defaultdict(list) + for node in group: + sf = node.get("source_file") or "" + by_file[sf].append(node) + for sf, file_group in by_file.items(): + if not sf: + # No source_file — cannot prove same symbol; skip to avoid + # collapsing distinct nodes that happen to share a label (#1178). + continue + if len(file_group) > 1: + winner = _pick_winner(file_group) + for node in file_group: + uf.union(winner["id"], node["id"]) + exact_merges += len(file_group) - 1 + # Cross-file residue: union exact matches across files, but only where + # it is provably safe (#2182). `concept` is the one file_type meant to + # unify across files (#1284) — code is keyed by ID (#1205), rationale/ + # document are file-anchored (#1284), and image/paper labels are often + # shared basenames (logo.png). Provenance is required (#1178), and the + # entropy gate mirrors Pass 2 so short generic labels ("API") stay + # distinct. Sorting by id keeps the winner order-independent. + mergeable = sorted( + (n for n in group + if n.get("file_type") == "concept" + and (n.get("source_file") or "") + and _entropy(n.get("label", "")) >= _ENTROPY_THRESHOLD), + key=lambda n: n["id"], + ) + if len(mergeable) > 1: + winner = _pick_winner(mergeable) + for node in mergeable: + if uf.find(winner["id"]) != uf.find(node["id"]): + uf.union(winner["id"], node["id"]) + exact_merges += 1 + + # ── pass 2: MinHash/LSH + Jaro-Winkler (high-entropy nodes only) ───────── + candidates: list[dict] = [] + seen_norms: set[str] = set() + for node in unique_nodes: + # Code symbols are excluded from fuzzy matching too: two functions with + # similar long names in different files (parallel backends, sibling + # classes) must not be fuzzy-merged, and a code↔concept fuzzy match must + # not transitively union two distinct code symbols via a concept (#1205). + if _is_code(node): + continue + key = _norm(node.get("label", node.get("id", ""))) + if key and key not in seen_norms: + seen_norms.add(key) + if _entropy(node.get("label", "")) >= _ENTROPY_THRESHOLD: + candidates.append(node) + + fuzzy_merges = 0 + if len(candidates) >= 2: + lsh = MinHashLSH(threshold=_LSH_THRESHOLD, num_perm=_NUM_PERM) + minhashes: dict[str, MinHash] = {} + # Pre-build O(1) lookup structures so the query loop below doesn't scan + # the candidates list linearly for every LSH neighbor (was O(n²×B)). + candidates_by_id: dict[str, dict] = {} + norm_cache: dict[str, str] = {} + + for node in candidates: + node_id = node["id"] + candidates_by_id[node_id] = node + nl = _norm(node.get("label", node.get("id", ""))) + norm_cache[node_id] = nl + m = _make_minhash(nl) + minhashes[node_id] = m + try: + lsh.insert(node_id, m) + except ValueError: + pass # duplicate key in LSH — already inserted + + for node in candidates: + node_id = node["id"] + norm_label = norm_cache[node_id] + neighbors = lsh.query(minhashes[node_id]) + + for neighbor_id in neighbors: + if neighbor_id == node_id: + continue + if uf.find(node_id) == uf.find(neighbor_id): + continue + + neighbor = candidates_by_id.get(neighbor_id) + if neighbor is None: + continue + + neighbor_norm = norm_cache.get(neighbor_id) or _norm(neighbor.get("label", neighbor.get("id", ""))) + # Cross-file long labels score on plain Jaro (no prefix bonus). + # Jaro-Winkler's leading-prefix bonus lifts pairs that share a + # prefix but diverge in a distinguishing token ("testing-library + # jest-native" vs "react-native") past threshold, fabricating + # destructive cross-file merges; on Jaro alone they fall short + # while true cross-file duplicates still clear it (#1243). Same-file + # near-duplicates keep Jaro-Winkler (low-risk, and a mid-string + # stopword insertion needs the prefix bonus to merge); short labels + # keep Jaro-Winkler too (gated by _short_label_blocked). + _xfile = (node.get("source_file") or "") != (neighbor.get("source_file") or "") + if _xfile and max(len(norm_label), len(neighbor_norm)) >= 12: + score = Jaro.normalized_similarity(norm_label, neighbor_norm) * 100 + else: + score = JaroWinkler.normalized_similarity(norm_label, neighbor_norm) * 100 + + if _is_variant_pair(norm_label, neighbor_norm): + continue + if _short_label_blocked(norm_label, neighbor_norm, score): + continue + # Prefix-extension pairs (getActiveSession / getActiveSessions, + # parseConfig / parseConfigFile) are almost never duplicates — + # one is a strict suffix-extension of the other. Block the merge + # regardless of JW score (#1201). + _lo, _hi = sorted((norm_label, neighbor_norm), key=len) + if _hi.startswith(_lo) and _hi != _lo: + continue + # Numbered/versioned siblings and cross-file file-anchored + # boilerplate (rationale/document) are decisively distinct + # regardless of score (#1284). + if _numeric_tokens_differ(norm_label, neighbor_norm): + continue + # Template-named siblings differing in a content word are + # distinct too, on either path: same-file pairs keep the prefix + # bonus, and a cross-file pair can still reach threshold on the + # community boost alone (#2576). + if _content_token_swap(norm_label, neighbor_norm): + continue + if _crossfile_fileanchored_blocked(node, neighbor): + continue + + c1 = communities.get(node_id) + c2 = communities.get(neighbor_id) + if (c1 is not None and c2 is not None and c1 == c2 + and min(len(norm_label), len(neighbor_norm)) >= 12): + score += _COMMUNITY_BOOST + + if score >= _MERGE_THRESHOLD: + # Belt-and-braces (#1046, narrowed by #2182): candidates are + # norm-unique (`seen_norms` above), so two candidates can + # never share a normalized label and this branch is + # unreachable today. Retained in case candidate selection + # changes. Equal-norm cross-file pairs are handled in Pass 1 + # instead, gated to `concept` nodes — the original #1046 + # rationale (same-named code symbols) was obsoleted by code + # being excluded from label matching entirely (#1205, #1247). + if norm_label == neighbor_norm: + sf_a = node.get("source_file") or "" + sf_b = neighbor.get("source_file") or "" + if sf_a != sf_b: + continue + # Pick the winner from the verified pair only. Selecting it + # from the union of both normalized-label groups pulls + # never-compared nodes (same label, different source_file) + # into the merge, bypassing the #1046/#1178 guards. + winner = _pick_winner([node, neighbor]) + uf.union(winner["id"], node_id) + uf.union(winner["id"], neighbor_id) + fuzzy_merges += 1 + + # ── pass 3: LLM tiebreaker for ambiguous pairs (opt-in) ────────────────── + if dedup_llm_backend is not None: + _llm_tiebreak(candidates, uf, communities, backend=dedup_llm_backend) + + # ── build remap table from union-find components ────────────────────────── + components = uf.components() + remap: dict[str, str] = {} + + # id -> (position, node), built once. Previously each component re-scanned + # the whole unique_nodes list, making remap construction O(nodes x + # components) — 31% of dedup wall-clock on a 50k-node corpus. + # The position is carried so group_nodes keeps unique_nodes order: _pick_winner + # resolves ties (equal chunk-suffix status and equal id length) via min(), + # which returns the first minimum, so reordering here would silently change + # which node survives. + nodes_by_id: dict[str, tuple[int, dict]] = { + n["id"]: (i, n) for i, n in enumerate(unique_nodes) + } + + for root, members in components.items(): + if len(members) == 1: + continue + group_nodes = [ + n for _, n in sorted( + (nodes_by_id[m] for m in members if m in nodes_by_id), + key=lambda pair: pair[0], + ) + ] + winner = _pick_winner(group_nodes) if group_nodes else {"id": root} + winner_id = winner["id"] + for member in members: + if member != winner_id: + remap[member] = winner_id + + # ── apply remap ─────────────────────────────────────────────────────────── + if not remap: + return unique_nodes, edges + + total = len(remap) + msg = f"[graphify] Deduplicated {total} node(s)" + # Both counters are reported when non-zero. Previous form nested the fuzzy + # branch inside `if exact_merges`, silently dropping the fuzzy count on + # doc/semantic-heavy runs where Pass 1 finds nothing (#1857). + parts: list[str] = [] + if exact_merges: + parts.append(f"{exact_merges} exact") + if fuzzy_merges: + parts.append(f"{fuzzy_merges} fuzzy") + if parts: + msg += f" ({', '.join(parts)})" + print(msg + ".", flush=True) + + deduped_nodes = [n for n in unique_nodes if n["id"] not in remap] + deduped_edges = [] + for edge in edges: + e = dict(edge) + # Tolerate "from"/"to" keys from LLM backends that don't follow the + # schema exactly — build_from_json normalises later but dedup runs + # first so bracket access would KeyError here (#803). + # Use explicit key presence check (not `or`) so empty-string src/tgt + # aren't silently replaced by the fallback key. + src = e["source"] if "source" in e else e.get("from") + tgt = e["target"] if "target" in e else e.get("to") + if src is None or tgt is None: + continue + e["source"] = remap.get(src, src) + e["target"] = remap.get(tgt, tgt) + # Remove legacy keys so they don't leak into edge attrs in graph.json. + e.pop("from", None) + e.pop("to", None) + if e["source"] != e["target"]: + deduped_edges.append(e) + + return deduped_nodes, deduped_edges + + +def _pick_winner(nodes: list[dict]) -> dict: + """Pick the canonical survivor: prefer no chunk suffix, then shorter ID.""" + if not nodes: + raise ValueError("Cannot pick winner from empty list") + + def _score(n: dict) -> tuple[int, int]: + has_suffix = bool(_CHUNK_SUFFIX.search(n["id"])) + return (1 if has_suffix else 0, len(n["id"])) + + return min(nodes, key=_score) + + +def _llm_tiebreak( + candidates: list[dict], + uf: _UF, + communities: dict[str, int], + *, + backend: str, + batch_size: int = 30, + low: float = 75.0, + high: float = 92.0, +) -> None: + """Batch-resolve ambiguous pairs (score in [low, high)) via LLM.""" + try: + from graphify.llm import BACKENDS, _format_backend_env_keys, _get_backend_api_key + if backend not in BACKENDS: + print(f"[graphify] --dedup-llm: unknown backend {backend!r}, skipping LLM tiebreaker.", flush=True) + return + if not _get_backend_api_key(backend): + env_keys = _format_backend_env_keys(backend) + print(f"[graphify] --dedup-llm: {env_keys} not set, skipping LLM tiebreaker.", flush=True) + return + except ImportError: + return + + ambiguous: list[tuple[dict, dict, float]] = [] + for i, node in enumerate(candidates): + norm_i = _norm(node.get("label", node.get("id", ""))) + for j in range(i + 1, len(candidates)): + neighbor = candidates[j] + if uf.find(node["id"]) == uf.find(neighbor["id"]): + continue + norm_j = _norm(neighbor.get("label", neighbor.get("id", ""))) + # Mirror pass 2: plain Jaro for cross-file long labels (#1243). + _xfile = (node.get("source_file") or "") != (neighbor.get("source_file") or "") + if _xfile and max(len(norm_i), len(norm_j)) >= 12: + score = Jaro.normalized_similarity(norm_i, norm_j) * 100 + else: + score = JaroWinkler.normalized_similarity(norm_i, norm_j) * 100 + if _is_variant_pair(norm_i, norm_j): + continue + if _short_label_blocked(norm_i, norm_j, score): + continue + _lo, _hi = sorted((norm_i, norm_j), key=len) + if _hi.startswith(_lo) and _hi != _lo: + continue + # Mirror pass 2: decisively-distinct pairs never reach the LLM + # (#1284, #2576). + if _numeric_tokens_differ(norm_i, norm_j): + continue + if _content_token_swap(norm_i, norm_j): + continue + if _crossfile_fileanchored_blocked(node, neighbor): + continue + c1 = communities.get(node["id"]) + c2 = communities.get(neighbor["id"]) + if (c1 is not None and c2 is not None and c1 == c2 + and min(len(norm_i), len(norm_j)) >= 12): + score += _COMMUNITY_BOOST + if low <= score < high: + ambiguous.append((node, neighbor, score)) + + if not ambiguous: + return + + try: + from graphify.llm import _call_llm + except ImportError as exc: + # F-038: previously this silent fallback hid the fact that `_call_llm` + # didn't exist in `graphify.llm` at all, so `--dedup-llm` was a no-op. + # Surface the import failure so future regressions are visible. + print( + f"[graphify] --dedup-llm: cannot import _call_llm ({exc}); skipping LLM tiebreaker.", + flush=True, + ) + return + + for batch_start in range(0, len(ambiguous), batch_size): + batch = ambiguous[batch_start : batch_start + batch_size] + pairs_text = "\n".join( + f"{i+1}. \"{a['label']}\" vs \"{b['label']}\"" + for i, (a, b, _) in enumerate(batch) + ) + prompt = ( + "For each pair below, answer only 'yes' or 'no': are they the same real-world concept?\n\n" + f"{pairs_text}\n\n" + "Reply with one line per pair: '1. yes', '2. no', etc." + ) + try: + response = _call_llm(prompt, backend=backend, max_tokens=200) + lines = response.strip().splitlines() + for line in lines: + line = line.strip() + if not line: + continue + parts = line.split(".", 1) + if len(parts) != 2: + continue + try: + idx = int(parts[0].strip()) - 1 + except ValueError: + continue + if 0 <= idx < len(batch): + answer = parts[1].strip().lower() + if answer.startswith("yes"): + a, b, _ = batch[idx] + winner = _pick_winner([a, b]) + uf.union(winner["id"], a["id"]) + uf.union(winner["id"], b["id"]) + except Exception as exc: + print(f"[graphify] --dedup-llm batch failed: {exc}", flush=True) diff --git a/cli.py b/cli.py new file mode 100644 index 000000000..ef0a18612 --- /dev/null +++ b/cli.py @@ -0,0 +1,2044 @@ +# file discovery, type classification, and corpus health checks +from __future__ import annotations +import fnmatch +import json +import os +import re +import shlex +import unicodedata +from concurrent.futures import ThreadPoolExecutor +from enum import Enum +from functools import lru_cache +from pathlib import Path +from typing import Callable + +from graphify.google_workspace import ( + GOOGLE_WORKSPACE_EXTENSIONS, + convert_google_workspace_file, + google_workspace_enabled, +) +from graphify.paths import ( + GRAPHIFY_OUT, + io_path as _os_path, + make_dirs as _make_dirs, + out_path, + path_exists as _path_exists, + path_is_dir as _path_is_dir, + path_is_file as _path_is_file, + path_is_symlink as _path_is_symlink, + path_stat as _path_stat, + read_text as _read_text, + resolve_path as _resolve_path, + walk_path as _walk_path, + write_text as _write_text, +) + + +class FileType(str, Enum): + CODE = "code" + DOCUMENT = "document" + PAPER = "paper" + IMAGE = "image" + VIDEO = "video" + + +_MANIFEST_PATH = str(out_path("manifest.json")) + +CODE_EXTENSIONS = {'.py', '.ts', '.tsx', '.mts', '.cts', '.js', '.jsx', '.mjs', '.cjs', '.ejs', '.ets', '.go', '.rs', '.java', '.groovy', '.gradle', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.cu', '.cuh', '.metal', '.rb', '.rake', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.luau', '.toc', '.zig', '.ps1', '.psm1', '.psd1', '.ex', '.exs', '.m', '.mm', '.jl', '.vue', '.svelte', '.astro', '.dart', '.v', '.sv', '.svh', '.sql', '.r', '.f', '.F', '.f90', '.F90', '.f95', '.F95', '.f03', '.F03', '.f08', '.F08', '.pas', '.pp', '.dpr', '.dpk', '.lpr', '.inc', '.dfm', '.lfm', '.lpk', '.sh', '.bash', '.json', '.tf', '.tfvars', '.hcl', '.dm', '.dme', '.dmi', '.dmm', '.dmf', '.sln', '.slnx', '.csproj', '.fsproj', '.vbproj', '.xaml', '.razor', '.cshtml', '.cls', '.trigger'} +DOC_EXTENSIONS = {'.md', '.mdx', '.qmd', '.skill', '.txt', '.rst', '.html', '.yaml', '.yml'} +PAPER_EXTENSIONS = {'.pdf'} +IMAGE_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg'} +OFFICE_EXTENSIONS = {'.docx', '.xlsx'} +VIDEO_EXTENSIONS = {'.mp4', '.mov', '.webm', '.mkv', '.avi', '.m4v', '.mp3', '.wav', '.m4a', '.ogg'} + +CORPUS_WARN_THRESHOLD = 50_000 # words - below this, warn "you may not need a graph" +CORPUS_UPPER_THRESHOLD = 500_000 # words - above this, warn about token cost +FILE_COUNT_UPPER = 500 # files - above this, warn about token cost + +# Resource caps for parsing untrusted office/PDF files (F2). A corpus is +# attacker-controllable (graphify runs on cloned/shared folders), and .docx/.xlsx +# are zip+XML containers: a few-KB zip-bomb can decompress to gigabytes and +# OOM-kill the process at load_workbook/Document time. Screen the file before any +# parser touches it. +_OFFICE_MAX_RAW_BYTES = 50 * 1024 * 1024 # 50 MiB on-disk +_OFFICE_MAX_DECOMPRESSED_BYTES = 512 * 1024 * 1024 # 512 MiB total uncompressed +_OFFICE_MAX_COMPRESSION_RATIO = 200 # uncompressed : compressed + + +def _file_within_size_cap(path: Path, cap: int = _OFFICE_MAX_RAW_BYTES) -> bool: + """True if *path* exists and its on-disk size is within *cap*.""" + try: + return _path_stat(path).st_size <= cap + except OSError: + return False + + +def _zip_within_caps(path: Path) -> bool: + """Reject a zip-based office file that is a likely zip/XML bomb. + + Two layers, because the zip central-directory sizes are attacker-controlled: + 1. A cheap pre-filter on the declared sizes (on-disk cap, summed-uncompressed + cap, compression ratio) that rejects an honest bomb without decompressing. + 2. An authoritative pass that stream-decompresses every member with a hard + byte ceiling, so a member that under-declares its size in the central + directory cannot expand past the cap undetected. Decompression is chunked + and bounded, so checking a bomb never materializes more than the ceiling. + """ + import zipfile + if not _file_within_size_cap(path): + return False + try: + with zipfile.ZipFile(_os_path(path)) as zf: + infos = zf.infolist() + compressed = sum(i.compress_size for i in infos) or 1 + declared = sum(i.file_size for i in infos) + if declared > _OFFICE_MAX_DECOMPRESSED_BYTES: + return False + if declared / compressed > _OFFICE_MAX_COMPRESSION_RATIO: + return False + total = 0 + for info in infos: + with zf.open(info) as member: + while True: + chunk = member.read(1024 * 1024) + if not chunk: + break + total += len(chunk) + if total > _OFFICE_MAX_DECOMPRESSED_BYTES: + return False + except (zipfile.BadZipFile, OSError, EOFError): + return False + return True + +# Dedicated credential-store directories: everything beneath them is sensitive, +# with no carve-out — a .py inside ~/.ssh or ~/.aws is tooling for key material, +# not a source package, and keys there are routinely extensionless. +# Both sets are checked against path.parts[:-1] (parents only) so a root-level +# file named "credentials" or "secrets" is not falsely flagged by this stage. +_CREDENTIAL_STORE_DIRS = frozenset({ + ".ssh", ".gnupg", ".aws", ".gcloud", +}) + +# Bare-name directories that are as often legitimate source packages (Go +# internal/secrets, a credentials/ service module) as credential stores. Their +# contents are sensitive EXCEPT genuine programming-language source, mirroring +# the Stage 3 keyword carve-out (#1666) at the directory level (#1943). +_AMBIGUOUS_SENSITIVE_DIRS = frozenset({ + "secrets", ".secrets", "credentials", +}) + +# Files that may contain secrets - skip silently. These patterns are specific +# (extensions, exact credential-store names) and always apply. +_SENSITIVE_PATTERNS = [ + re.compile(r'(^|[\\/])\.(env|envrc)(\.|$)', re.IGNORECASE), + re.compile(r'\.(pem|key|p12|pfx|cert|crt|der|p8)$', re.IGNORECASE), + # SSH/GPG private keys. Left boundary + IGNORECASE so `grid_rsa` (alpha before + # `id_rsa`) and `ID_RSA` are handled correctly, not matched as a substring. + re.compile(r'(^|[^A-Za-z0-9])(id_rsa|id_dsa|id_ecdsa|id_ed25519)(\.pub)?$', re.IGNORECASE), + re.compile(r'^secring(\.(gpg|pgp))?$', re.IGNORECASE), # GPG private keyring + # Auth/credential dotfiles that routinely hold tokens (#2106: .npmrc/.pypirc/ + # .git-credentials/.boto were silently indexed before). + re.compile(r'(\.netrc|\.pgpass|\.htpasswd|\.npmrc|\.pypirc|\.git-credentials|\.boto)$', re.IGNORECASE), + # NOTE: aws_credentials/gcloud_credentials/service_account moved to the + # boundary-checked Stage 3 keyword path (#2106). The old unbounded + # `service.account` substring (regex `.` wildcard) matched real source like + # google/oauth2/service_account.py and prose like aws_credentials_rotation.md. +] + +# Committed dotenv / envrc templates — placeholders only, not live secrets. +# Stage 2's `.env.` regex otherwise treats these like `.env.local` (#2184). +_ENV_TEMPLATE_SUFFIXES = (".example", ".sample", ".template", ".dist") + + +def _is_env_template(name: str) -> bool: + """True for `.env.example` / `.envrc.sample` style committed templates (#2184).""" + lower = name.lower() + if not lower.endswith(_ENV_TEMPLATE_SUFFIXES): + return False + # Basename must still be an .env* / .envrc* file (not e.g. secrets.example). + return bool(re.match(r"\.(env|envrc)\.", lower)) + +# Generic keyword patterns - these only count when the keyword is LOAD-BEARING +# in the filename (see _generic_keyword_hit), because a keyword buried mid-phrase +# in a long descriptive slug names a topic, not a credential store: +# "token-economics-of-recall.md" is a note ABOUT tokens; "api_token.txt" IS one. +# Uses lookarounds instead of \b so underscore-prefixed names like api_token.txt +# match. Both patterns use (?![a-zA-Z]) so that the trailing-underscore behavior +# is consistent: "secret_store.txt" IS flagged, "tokenizer.py" is NOT (because +# "i" after "token" is alpha and blocks the match). +# `token` is kept separate because its longer suffix "izer"/"ize" is the only +# common false-positive; other keywords have no such well-known derivatives. +_GENERIC_KEYWORD_PATTERNS = [ + re.compile(r'(? bool: + """A prose/note file (.md/.rst/...) whose stem is a multi-word topic slug is + exempt from the generic-keyword drop (#2106). A stem that IS exactly a bare + keyword (secrets / token / passwords) is NOT exempt — that still reads as a + credential dump.""" + if path.suffix.lower() not in _PROSE_EXTS: + return False + stem = Path(path.name).stem.lstrip('.') or Path(path.name).stem + return not any(p.fullmatch(stem) for p in _GENERIC_KEYWORD_PATTERNS) + + +def _generic_keyword_hit(name: str) -> bool: + """True if a generic secret keyword appears load-bearing in the filename. + + Secret-store files name their contents, and in English compounds the + content noun is the head, which comes last: "github-personal-access-token", + "api_token", "oauth_token". A keyword that is neither at the end of the + stem nor in a short (<=2 word) name is a topic word in a descriptive slug + ("token-economics-of-recall.md", "password-policy-discussion.md") and must + not cause the file to be silently dropped from the graph (#436, #718). + """ + # Stem = name minus only the FINAL extension (not up to the first dot), so a + # multi-dot topic slug like `token.economics.notes.md` keeps all its words and + # doesn't collapse to a bare `token` (#2106). Leading dots stripped so + # dotfiles like `.token` keep their keyword. + stem = Path(name).stem.lstrip('.') or Path(name).stem + for pat in _GENERIC_KEYWORD_PATTERNS: + hit = False + for m in pat.finditer(stem): + hit = True + if m.end() == len(stem): # keyword ends the stem -> names the contents + return True + if hit and len([w for w in _WORD_SPLIT.split(stem) if w]) <= 2: + return True # short name like token_config.yaml / secret_handler.txt + return False + +# Signals that a .md/.txt file is actually a converted academic paper +_PAPER_SIGNALS = [ + re.compile(r'\barxiv\b', re.IGNORECASE), + re.compile(r'\bdoi\s*:', re.IGNORECASE), + re.compile(r'\babstract\b', re.IGNORECASE), + re.compile(r'\bproceedings\b', re.IGNORECASE), + re.compile(r'\bjournal\b', re.IGNORECASE), + re.compile(r'\bpreprint\b', re.IGNORECASE), + re.compile(r'\\cite\{'), # LaTeX citation + re.compile(r'\[\d+\]'), # Numbered citation [1], [23] (inline) + re.compile(r'\[\n\d+\n\]'), # Numbered citation spread across lines (markdown conversion) + re.compile(r'eq\.\s*\d+|equation\s+\d+', re.IGNORECASE), + re.compile(r'\d{4}\.\d{4,5}'), # arXiv ID like 1706.03762 + re.compile(r'\bwe propose\b', re.IGNORECASE), # common academic phrasing + re.compile(r'\bliterature\b', re.IGNORECASE), # "from the literature" +] +_PAPER_SIGNAL_THRESHOLD = 3 # need at least this many signals to call it a paper + + +def _is_graphable_source(path: Path) -> bool: + """True for genuine programming-language source — the only category exempt + from the ambiguous-dir (Stage 1, #1943) and generic-keyword (Stage 3, #1666) + drops. Data/serialization formats are NOT exempt even though some route + through the CODE path for manifest parsing: credentials.json / secrets.yaml + are exactly the stores those stages must keep catching. + """ + return classify_file(path) == FileType.CODE and path.suffix.lower() not in _SECRET_PRONE_DATA_EXTS + + +def _is_sensitive(path: Path) -> bool: + """Return True if this file likely contains secrets and should be skipped.""" + # Stage 1: any PARENT directory is a known secrets dir (parts[:-1] excludes + # the filename itself so a root-level file named "credentials" is not falsely + # skipped — the name patterns in Stage 2 handle the filename). Dedicated + # credential stores drop everything unconditionally; ambiguous bare-name dirs + # (secrets/, credentials/) spare genuine source (#1943), which still falls + # through so Stages 2-3 screen its filename like anywhere else. + parents = path.parts[:-1] + # Lowercase the segment comparison so `Secrets/`/`SECRETS/` (real on + # case-insensitive macOS/Windows filesystems) are still caught (#2106). + if any(part.lower() in _CREDENTIAL_STORE_DIRS for part in parents): + return True + if any(part.lower() in _AMBIGUOUS_SENSITIVE_DIRS for part in parents) and not _is_graphable_source(path): + return True + # Stage 2: filename pattern match. Template suffixes (.example/.sample/…) + # on .env / .envrc are the usual "safe to commit" convention — keep them + # in the graph without opening a broad Stage 2 allowlist (#2184 / #1921). + name = path.name + if any(p.search(name) for p in _SENSITIVE_PATTERNS) and not _is_env_template(name): + return True + # Stage 3: generic keywords, only when load-bearing in the name. Do NOT let a + # bare name keyword silently drop a genuine programming-language source file: + # a .rb/.py named device_token or passwords_controller is a module, not a secret + # store (#1666). Data/config formats (.json, .yaml, .toml, ...) are deliberately + # NOT exempt even though .json routes through the CODE path for manifest parsing, + # because credentials.json / oauth_token.json / secrets.yaml are exactly the + # secret stores this stage must catch. The specific Stage 2 patterns (.env, .pem, + # id_rsa, ...) still apply to everything regardless of extension. + if _generic_keyword_hit(name): + # Genuine source AND multi-word prose notes are exempt; a bare-keyword + # name (secrets.md, token.txt) still drops (#1666, #2106). + return not (_is_graphable_source(path) or _is_prose_note(path)) + return False + + +def _looks_like_paper(path: Path) -> bool: + """Heuristic: does this text file read like an academic paper?""" + try: + # Only scan first 3000 chars for speed + text = _read_text(path, encoding="utf-8", errors="ignore")[:3000] + hits = sum(1 for pattern in _PAPER_SIGNALS if pattern.search(text)) + return hits >= _PAPER_SIGNAL_THRESHOLD + except Exception: + return False + + +_ASSET_DIR_MARKERS = {".imageset", ".xcassets", ".appiconset", ".colorset", ".launchimage"} + + +_SHEBANG_CODE_INTERPRETERS = { + "python", "python3", "python2", + "ruby", "perl", "node", "nodejs", + "bash", "sh", "dash", "zsh", "fish", "ksh", "tcsh", + "lua", "php", "julia", "Rscript", +} + + +def _split_env_s(value: str, rest: list[str]) -> list[str]: + """Re-tokenize an `env -S`/`--split-string` packed command, prepending the + operand to any trailing args. Returns the unpacked argv.""" + packed = " ".join([value, *rest]).strip() + return shlex.split(packed) + + +def _env_command_args(args: list[str], *, allow_split: bool = True) -> list[str]: + """Strip leading env(1) options and var assignments, return the trailing + command argv. Covers macOS/BSD and GNU coreutils env documented spellings. + + POSIX/macOS short forms: + env [-0iv] [-C workdir] [-P utilpath] [-S string] + [-u name] [name=value ...] [utility [argument ...]] + + GNU coreutils long/compact forms additionally supported: + --argv0=ARG / -a ARG / -aARG + --unset=NAME / --unset NAME / -u NAME / -uNAME + --chdir=DIR / --chdir DIR / -C DIR / -CDIR + --split-string=STRING / --split-string STRING + -S STRING / -SSTRING / -vS STRING / -vSSTRING + --ignore-environment / --null / --debug / --list-signal-handling + --default-signal[=SIG] / --ignore-signal[=SIG] / --block-signal[=SIG] + + `-S` / `--split-string` payloads are themselves env-style argument lists + per the GNU shebang synopsis: + #!/usr/bin/env -[v]S[option]... [name=value]... command [args]... + so after splitting the payload we recursively re-parse it with + `allow_split=False` (a nested -S inside a split payload is rejected to + bound recursion). + + Unknown hyphen-prefixed args yield [] (we refuse to guess whether + their next token is an interpreter or an operand). + """ + i = 0 + while i < len(args): + arg = args[i] + + if arg == "--": + return args[i + 1:] + + # Split-string forms: tokenize the packed payload, then re-parse it + # as env args (so leading assignments/flags inside the payload are + # skipped before the interpreter is identified). + if allow_split: + if arg == "-S": + if i + 1 >= len(args): + return [] + return _env_command_args( + _split_env_s(" ".join(args[i + 1:]), []), + allow_split=False, + ) + if arg.startswith("-S") and len(arg) > 2: + return _env_command_args( + _split_env_s(arg[2:], args[i + 1:]), + allow_split=False, + ) + if arg == "-vS": + if i + 1 >= len(args): + return [] + return _env_command_args( + _split_env_s(" ".join(args[i + 1:]), []), + allow_split=False, + ) + if arg.startswith("-vS") and len(arg) > 3: + return _env_command_args( + _split_env_s(arg[3:], args[i + 1:]), + allow_split=False, + ) + if arg.startswith("--split-string="): + return _env_command_args( + _split_env_s(arg.split("=", 1)[1], args[i + 1:]), + allow_split=False, + ) + if arg == "--split-string": + if i + 1 >= len(args): + return [] + return _env_command_args( + _split_env_s(args[i + 1], args[i + 2:]), + allow_split=False, + ) + + # Options with separate required operand + if arg in {"-u", "-C", "-P", "-a", "--unset", "--chdir", "--argv0"}: + if i + 2 > len(args): + return [] + i += 2 + continue + + # Clumped short option + operand + if ( + arg.startswith(("-u", "-C", "-P", "-a")) + and len(arg) > 2 + and not arg.startswith("--") + ): + i += 1 + continue + + # Long option with `=` operand + if arg.startswith(("--unset=", "--chdir=", "--argv0=")): + i += 1 + continue + + # No-operand flags + if arg in {"-", "-i", "-0", "-v", "--ignore-environment", "--null", + "--debug", "--list-signal-handling"}: + i += 1 + continue + + # Signal-handling long flags (with or without =SIG operand — we treat + # them as no-effect for interpreter-resolution purposes) + if arg.startswith(("--default-signal", "--ignore-signal", "--block-signal")): + i += 1 + continue + + # Unknown hyphen-prefixed: refuse to guess + if arg.startswith("-"): + return [] + + # Inline NAME=value assignment + if "=" in arg: + i += 1 + continue + + # First non-option, non-assignment token starts the command argv + return args[i:] + + return [] + + +def _shebang_interpreter(path: Path) -> str | None: + """Return the interpreter name from a shebang line. + + Handles forms that a naive parser misses: + - `#!/usr/bin/env -S python3 -u` (env -S split-args form, anywhere) + - `#!/usr/bin/env -i bash` (no-operand env flags) + - `#!/usr/bin/env -u VAR python3` (env options with operands) + - `#!/usr/bin/env -C /tmp python3` (env -C workdir) + - `#!/usr/bin/env -P /bin python3` (env -P utilpath) + - `#!/usr/bin/env DEBUG=1 python3` (inline var assignment) + - `#!"/usr/local/bin/python with spaces"` (shlex handles quotes) + + Returns the basename of the resolved interpreter, or None if there is + no shebang / the file is unreadable / parsing fails. + """ + try: + with open(_os_path(path), "rb") as f: + first = f.read(256) + if not first.startswith(b"#!"): + return None + line = first.split(b"\n")[0].decode(errors="replace")[2:].strip() + parts = shlex.split(line) + if not parts: + return None + interp = Path(parts[0]).name + if interp == "env": + env_args = _env_command_args(parts[1:]) + if not env_args: + return None + interp = Path(env_args[0]).name + return interp + except (OSError, ValueError): + return None + + +def _shebang_file_type(path: Path) -> FileType | None: + """Peek at the first line of an extensionless file for a shebang.""" + interp = _shebang_interpreter(path) + if interp in _SHEBANG_CODE_INTERPRETERS: + return FileType.CODE + return None + + +def classify_file(path: Path) -> FileType | None: + # Package manifests (apm.yml, pyproject.toml, go.mod, pom.xml) are parsed + # deterministically, so route them to the AST path (CODE) rather than the LLM + # document path — otherwise apm.yml (a .yml "document") would be LLM-extracted + # and a package would split into duplicate file-anchored nodes (#1377). + from graphify.manifest_ingest import is_package_manifest_path + if is_package_manifest_path(path): + return FileType.CODE + # Compound extensions must be checked before simple suffix lookup + if path.name.lower().endswith(".blade.php"): + return FileType.CODE + ext = path.suffix.lower() + if not ext: + return _shebang_file_type(path) + if ext in CODE_EXTENSIONS: + return FileType.CODE + if ext in PAPER_EXTENSIONS: + # PDFs inside Xcode asset catalogs are vector icons, not papers + if any(part.endswith(tuple(_ASSET_DIR_MARKERS)) for part in path.parts): + return None + return FileType.PAPER + if ext in IMAGE_EXTENSIONS: + return FileType.IMAGE + if ext in DOC_EXTENSIONS: + # Check if it's a converted paper + if _looks_like_paper(path): + return FileType.PAPER + return FileType.DOCUMENT + if ext in OFFICE_EXTENSIONS: + return FileType.DOCUMENT + if ext in GOOGLE_WORKSPACE_EXTENSIONS: + return FileType.DOCUMENT + if ext in VIDEO_EXTENSIONS: + return FileType.VIDEO + return None + + +def extract_pdf_text(path: Path) -> str: + """Extract plain text from a PDF file using pypdf.""" + if not _file_within_size_cap(path): + return "" + try: + from pypdf import PdfReader + reader = PdfReader(_os_path(path)) + pages = [] + for page in reader.pages: + text = page.extract_text() + if text: + pages.append(text) + return "\n".join(pages) + except Exception: + return "" + + +def docx_to_markdown(path: Path) -> str: + """Convert a .docx file to markdown text using python-docx.""" + if not _zip_within_caps(path): + return "" + try: + from docx import Document + from docx.oxml.ns import qn + doc = Document(_os_path(path)) + lines = [] + for para in doc.paragraphs: + style = para.style.name if para.style else "" + text = para.text.strip() + if not text: + lines.append("") + continue + if style.startswith("Heading 1"): + lines.append(f"# {text}") + elif style.startswith("Heading 2"): + lines.append(f"## {text}") + elif style.startswith("Heading 3"): + lines.append(f"### {text}") + elif style.startswith("List"): + lines.append(f"- {text}") + else: + lines.append(text) + # Tables + for table in doc.tables: + rows = [[cell.text.strip() for cell in row.cells] for row in table.rows] + if not rows: + continue + header = "| " + " | ".join(rows[0]) + " |" + sep = "| " + " | ".join("---" for _ in rows[0]) + " |" + lines.extend([header, sep]) + for row in rows[1:]: + lines.append("| " + " | ".join(row) + " |") + return "\n".join(lines) + except ImportError: + return "" + except Exception: + return "" + + +def xlsx_to_markdown(path: Path) -> str: + """Convert an .xlsx file to markdown text using openpyxl.""" + if not _zip_within_caps(path): + return "" + try: + import openpyxl + wb = openpyxl.load_workbook(_os_path(path), read_only=True, data_only=True) + sections = [] + for sheet_name in wb.sheetnames: + ws = wb[sheet_name] + rows = [] + for row in ws.iter_rows(values_only=True): + if all(cell is None for cell in row): + continue + rows.append([str(cell) if cell is not None else "" for cell in row]) + if not rows: + continue + sections.append(f"## Sheet: {sheet_name}") + if len(rows) >= 1: + header = "| " + " | ".join(rows[0]) + " |" + sep = "| " + " | ".join("---" for _ in rows[0]) + " |" + sections.extend([header, sep]) + for row in rows[1:]: + sections.append("| " + " | ".join(row) + " |") + wb.close() + return "\n".join(sections) + except ImportError: + return "" + except Exception: + return "" + + +def xlsx_extract_structure(path: Path) -> dict: + """Extract structural nodes (sheets, named tables, column headers) from an .xlsx file. + + Returns a nodes/edges dict compatible with the graphify extract pipeline. + Used in addition to xlsx_to_markdown so Claude sees both structure and content. + """ + def _nid(*parts: str) -> str: + return re.sub(r"[^a-z0-9_]", "_", "_".join(p.lower() for p in parts).strip("_")) + + try: + import openpyxl + except ImportError: + return {"nodes": [], "edges": []} + + try: + wb = openpyxl.load_workbook(_os_path(path), read_only=False, data_only=True) + except Exception: + return {"nodes": [], "edges": []} + + # F-035: typo fix — was `_re.sub` (NameError, but unreachable because the + # whole xlsx codepath is currently behind a feature flag / not yet wired + # into the dispatcher). Before re-enabling this path, re-audit it for + # zip/XML bombs (openpyxl is built on top of zipfile and lxml-style XML + # parsing — a malicious .xlsx can blow up memory at load_workbook time). + stem = re.sub(r"[^a-z0-9]", "_", path.stem.lower()) + str_path = str(path) + file_nid = _nid(str_path) + nodes: list[dict] = [{"id": file_nid, "label": path.name, "file_type": "document", + "source_file": str_path, "source_location": None}] + edges: list[dict] = [] + seen: set[str] = {file_nid} + + def _add(nid: str, label: str) -> None: + if nid not in seen: + seen.add(nid) + nodes.append({"id": nid, "label": label, "file_type": "document", + "source_file": str_path, "source_location": None}) + + def _edge(src: str, tgt: str, relation: str) -> None: + edges.append({"source": src, "target": tgt, "relation": relation, + "confidence": "EXTRACTED", "source_file": str_path, + "source_location": None, "weight": 1.0}) + + for sheet_name in wb.sheetnames: + ws = wb[sheet_name] + sheet_nid = _nid(stem, sheet_name) + _add(sheet_nid, f"{sheet_name} (sheet)") + _edge(file_nid, sheet_nid, "contains") + + # Named Excel Tables (ListObjects) + if hasattr(ws, "tables"): + for tbl in ws.tables.values(): + tbl_nid = _nid(stem, sheet_name, tbl.name) + _add(tbl_nid, tbl.name) + _edge(sheet_nid, tbl_nid, "contains") + # Column headers from table header row + ref = tbl.ref # e.g. "A1:D10" + if ref: + try: + from openpyxl.utils import range_boundaries + min_col, min_row, max_col, _ = range_boundaries(ref) + header_row = list(ws.iter_rows(min_row=min_row, max_row=min_row, + min_col=min_col, max_col=max_col, + values_only=True)) + if header_row: + for col_name in header_row[0]: + if col_name: + col_nid = _nid(stem, tbl.name, str(col_name)) + _add(col_nid, str(col_name)) + _edge(tbl_nid, col_nid, "contains") + except Exception: + pass + else: + # Fallback: first non-empty row as column headers + for row in ws.iter_rows(max_row=1, values_only=True): + for cell in row: + if cell: + col_nid = _nid(stem, sheet_name, str(cell)) + _add(col_nid, str(cell)) + _edge(sheet_nid, col_nid, "contains") + break + + try: + wb.close() + except Exception: + pass + + return {"nodes": nodes, "edges": edges} + + +def convert_office_file(path: Path, out_dir: Path, root: "Path | None" = None) -> Path | None: + """Convert a .docx or .xlsx to a markdown sidecar in out_dir. + + Returns the path of the converted .md file, or None if conversion failed + or the required library is not installed. + """ + ext = path.suffix.lower() + if ext == ".docx": + text = docx_to_markdown(path) + elif ext == ".xlsx": + text = xlsx_to_markdown(path) + else: + return None + + if not text.strip(): + return None + + _make_dirs(out_dir, exist_ok=True) + # Use a stable name derived from the original path to avoid collisions. + # Hash the path RELATIVE to the scan root, not the absolute path: the + # absolute form salts the name with the checkout location, so the same + # tracked .xlsx in two clones/worktrees emits two differently-named, + # byte-identical sidecars — unbounded duplicates when graphify-out/ is + # committed, each ingested as a distinct source doc (#2059). The relative + # path still disambiguates same-stem files in different directories. + # Normalize to NFC before hashing: on macOS (HFS+/APFS) os.walk/rglob return + # filenames in NFD, while Python string literals and directly-constructed + # Path objects are NFC, so the same source file would otherwise hash to + # different sidecar names across runs — making --update treat every Office + # file as new and re-extract it (#1226). + import hashlib + import unicodedata + if root is None: + # Default layout: out_dir is //converted. + root = out_dir.parent.parent + try: + key = _resolve_path(path).relative_to(_resolve_path(root)).as_posix() + except (ValueError, OSError): + # Not under the scan root (custom GRAPHIFY_OUT layouts, --include + # sources, direct API callers): keep the previous absolute form rather + # than guessing, so behavior is unchanged for those cases. + key = str(_resolve_path(path)) + normalized_path = unicodedata.normalize("NFC", key) + name_hash = hashlib.sha256(normalized_path.encode()).hexdigest()[:8] + out_path = out_dir / f"{path.stem}_{name_hash}.md" + # Skip re-writing only when the sidecar is present AND at least as new as the + # source. detect_incremental tracks the SIDECAR (not the Office source), so a + # sidecar that is never rewritten after the source changes leaves the doc + # reported "unchanged" forever and freezes the graph (#1649). Re-converting + # when the source is newer bumps the sidecar's mtime/content, which the + # incremental hash check then correctly picks up. An unchanged source keeps + # its (newer-or-equal) sidecar untouched so it never churns (#1226). + try: + if ( + _path_exists(out_path) + and _path_stat(out_path).st_mtime >= _path_stat(path).st_mtime + ): + return out_path + except OSError: + if _path_exists(out_path): + return out_path + _write_text( + out_path, + f"\n\n{text}", + encoding="utf-8", + ) + return out_path + + +def count_words(path: Path) -> int: + try: + ext = path.suffix.lower() + if ext == ".pdf": + return len(extract_pdf_text(path).split()) + if ext == ".docx": + return len(docx_to_markdown(path).split()) + if ext == ".xlsx": + return len(xlsx_to_markdown(path).split()) + with open(_os_path(path), encoding="utf-8", errors="ignore") as f: + return len(f.read().split()) + except Exception: + return 0 + + +# Directory names to always skip - venvs, caches, build artifacts, deps +_SKIP_DIRS = { + "venv", ".venv", # "env"/".env"/"*_env" are gated on venv markers below (#2058) + "node_modules", "__pycache__", ".git", + "dist", "build", "target", "out", + "site-packages", "lib64", + ".pytest_cache", ".mypy_cache", ".ruff_cache", + ".tox", ".nox", ".eggs", "*.egg-info", # nox is tox's successor, same .nox/ venv shape (#1804) + "graphify-out", # never treat the default output as source input (#524) + # Coverage/test-artefact dirs — generated, never architecturally meaningful + "lcov-report", # Vitest/Istanbul/nyc HTML reports (#870); + # bare "coverage" is gated on report + # artefacts below (#2339) + "visual-tests", "visual-test", # Playwright/visual-regression bundles (#869) + "__snapshots__", # Jest/Vitest snapshot dir (unambiguous) + "storybook-static", # Storybook production build output + "dist-protected", # Protected dist variants (same noise as dist) + # Framework cache/build dirs — generated, never architecturally meaningful (#873) + ".next", ".nuxt", ".turbo", ".angular", + ".idea", ".cache", ".parcel-cache", ".svelte-kit", ".terraform", ".serverless", + ".graphify", # graphify's own extraction cache — never index self-generated data + ".obsidian", ".smart-env", # Obsidian vault metadata and plugin caches (#2493) + ".worktrees", # git worktree convention (#947) — sibling checkouts, always redundant +} + +# Large generated files that are never useful to extract +_SKIP_FILES = { + "package-lock.json", "yarn.lock", "pnpm-lock.yaml", + "Cargo.lock", "poetry.lock", "Gemfile.lock", + "composer.lock", "go.sum", "go.work.sum", + # Removed allowlist config (#2112) — no longer consumed, so keep a leftover + # file out of the unclassified list instead of surfacing it as scan input. + ".graphifyinclude", +} + +# A bare "snapshots" dir is a Jest/Vitest artifact only when it actually holds +# snapshot files or lives directly under a JS test root. Elsewhere it is often a +# real code namespace (e.g. Rails app/services/snapshots/), so pruning it by name +# silently dropped legitimate source from the graph (#1666). "__snapshots__" stays +# unconditionally pruned above; only the ambiguous bare name is gated here. +_JS_SNAPSHOT_TEST_ROOTS = frozenset({"__tests__", "__test__"}) + +# Files a coverage tool writes into its own output dir. Any one of them is proof +# the directory is generated: lcov (lcov.info), nyc/Istanbul (coverage-final.json, +# clover.xml, the lcov-report/ subtree), coverage.py (coverage.xml, .coverage), +# JaCoCo/Cobertura (jacoco.xml, cobertura-coverage.xml). +_COVERAGE_ARTIFACT_FILES = frozenset({ + "lcov.info", "coverage-final.json", "coverage-summary.json", + "clover.xml", "coverage.xml", "cobertura-coverage.xml", "jacoco.xml", + ".coverage", "index.html", +}) +_COVERAGE_ARTIFACT_DIRS = frozenset({"lcov-report", "html-report"}) + + +def _has_coverage_artifacts(d: "Path") -> bool: + """True only when *d* holds files a coverage tool actually generated. + + ``coverage`` is a legitimate package name (a Python package, a Go/Rust module, + a domain namespace), so pruning it by name alone silently drops real source — + an entire 5-module package in #2339, with its dependents left in the graph so + queries still returned plausible neighbours. Prune it only on real evidence, + mirroring the ``snapshots``/``env`` gating (#1666/#2058): a coverage report + file, or an Istanbul/lcov HTML report subtree. + """ + try: + for name in _COVERAGE_ARTIFACT_FILES: + if _path_is_file(d / name): + return True + for name in _COVERAGE_ARTIFACT_DIRS: + if _path_is_dir(d / name): + return True + except OSError: + pass + return False + + +def _has_venv_markers(d: "Path") -> bool: + """True only when *d* has actual virtualenv/conda structure on disk. + + ``env``/``.env``/``*_env`` is a real source-directory convention (UVM/ASIC + verification trees, and others), so pruning it by name alone silently drops + legitimate source with no trace (#2058). Prune it only on real evidence: a + ``pyvenv.cfg``, an ``activate`` script, a ``lib/python*`` tree, or conda's + ``conda-meta/`` (``conda create -p ./env`` writes no pyvenv.cfg). + """ + try: + if _path_is_file(d / "pyvenv.cfg"): + return True + if _path_is_file(d / "bin" / "activate") or _path_is_file( + d / "Scripts" / "activate" + ): + return True + try: + with os.scandir(_os_path(d / "lib")) as entries: + if any(entry.name.startswith("python") for entry in entries): + return True + except OSError: + pass + if _path_is_dir(d / "conda-meta"): + return True + except OSError: + pass + return False + + +def _is_noise_dir(part: str, parent: "Path | None" = None) -> bool: + """Return True if this directory name looks like a venv, cache, or dep dir.""" + if part in _SKIP_DIRS: + return True + if part in ("env", ".env") or part.endswith("_env"): + # Ambiguous: a real venv OR a real source dir. Prune only on actual venv + # evidence, mirroring the "snapshots" gating (#1666/#2058). + if parent is None: + return False # cannot verify; keep a possibly-real code dir + return _has_venv_markers(parent / part) + if part == "coverage": + # Ambiguous: a generated report dir OR a real package named coverage. + # Prune only on actual coverage-artefact evidence (#2339). + if parent is None: + return False # cannot verify; keep a possibly-real code dir + return _has_coverage_artifacts(parent / part) + if part == "snapshots": + # Prune only when it looks like an actual JS/Vitest snapshot dir. + if parent is None: + return False # cannot verify; keep a possibly-real code dir + snap_dir = parent / part + if parent.name in _JS_SNAPSHOT_TEST_ROOTS: + return True + try: + with os.scandir(_os_path(snap_dir)) as entries: + if any(entry.name.endswith(".snap") for entry in entries): + return True + except OSError: + pass + return False + # Catch *_venv (unambiguous — "venv" is always a virtualenv signal). "*_env" + # is gated on markers above (#2058), not pruned by name. + if part.endswith("_venv"): + return True + if part.endswith(".egg-info"): + return True + # worktrees/ nested inside a dotted dir (e.g. .claude/worktrees/, .git/worktrees/) + if part == "worktrees" and parent is not None and parent.name.startswith("."): + return True + return False + + +_VCS_MARKERS = (".git", ".hg", ".svn", "_darcs", ".fossil") + + +def _nfc(text: str) -> str: + """Normalize text to NFC so ignore matching survives Unicode form drift. + + macOS (APFS/HFS+) returns filenames in NFD: "ç" comes back as "c" + + U+0327 COMBINING CEDILLA. Editors write ignore files in NFC, where the + same "ç" is the single codepoint U+00E7. The two render identically and + compare unequal, so a pattern like `Orçamento/` silently fails to exclude + the directory it names — the files are scanned and, for docs/PDFs, sent + to an LLM despite an explicit rule against it. + + Both sides are normalized to NFC before any fnmatch call. NFC is the form + Linux and Windows already use, so this is a no-op there and only repairs + the macOS mismatch. + """ + return unicodedata.normalize("NFC", text) + + +def _parse_gitignore_line(raw: str) -> str: + """Parse one raw line from a .graphifyignore file per gitignore spec. + + - Strip newline chars + - Strip inline comments (whitespace + # suffix), but only when # is + preceded by whitespace — so path#with#hash.py is preserved + - Unescape \\# to literal # + - Remove trailing spaces unless escaped with backslash + - Strip leading whitespace + - Return empty string for blank lines and full-line comments + """ + line = raw.rstrip("\n\r") + line = line.lstrip() + if not line or line.startswith("#"): + return "" + # Strip inline comments: require whitespace before # (gitignore extension) + line = re.sub(r"\s+#+[^\\].*$", "", line) + # Unescape \# → literal # + line = line.replace("\\#", "#") + # Remove unescaped trailing spaces (per gitignore spec) + line = re.sub(r"(? Path | None: + """Walk upward from start; return the first directory containing a VCS marker.""" + current = _resolve_path(start) + home = Path.home() + while True: + if any(_path_exists(current / m) for m in _VCS_MARKERS): + return current + parent = current.parent + if parent == current or current == home: + return None + current = parent + + +def _git_info_exclude(vcs_root: Path) -> Path | None: + """Resolve ``$GIT_DIR/info/exclude`` for the repo rooted at ``vcs_root``. + + ``info/exclude`` is where git records local-only, uncommitted excludes — and + where ``git worktree add`` writes nested worktree paths — so a repo can ignore + a directory without any ``.gitignore`` entry. graphify only read + ``.gitignore``/``.graphifyignore``, so it walked into those worktree copies and + the graph exploded (#1810). Handles the linked-worktree/submodule case where + ``.git`` is a file (``gitdir: ``) and the real excludes live in the + shared common git dir. Returns None when there is no readable exclude file. + """ + dot_git = vcs_root / ".git" + git_dir: Path | None = None + if _path_is_dir(dot_git): + git_dir = dot_git + elif _path_is_file(dot_git): + try: + content = _read_text(dot_git, encoding="utf-8", errors="ignore").strip() + except OSError: + content = "" + if content.startswith("gitdir:"): + gd = Path(content[len("gitdir:"):].strip()) + if not gd.is_absolute(): + gd = _resolve_path(vcs_root / gd) + git_dir = gd + # A linked worktree's gitdir holds a `commondir` file pointing at the + # shared git dir, where info/exclude actually lives. + commondir = gd / "commondir" + if _path_exists(commondir): + try: + cd_raw = _read_text(commondir, encoding="utf-8", errors="ignore").strip() + except OSError: + cd_raw = "" + if cd_raw: + cd = Path(cd_raw) + git_dir = cd if cd.is_absolute() else _resolve_path(gd / cd) + if git_dir is None: + return None + exclude = git_dir / "info" / "exclude" + return exclude if _path_is_file(exclude) else None + + +def _load_dir_own_ignore(d: Path, *, gitignore: bool = True) -> list[tuple[Path, str]]: + """Read .gitignore/.graphifyignore directly inside *d* (not its ancestors). + + Merges .gitignore and .graphifyignore for this one directory (#1363): + .gitignore is read first and .graphifyignore last, so .graphifyignore + patterns (including `!` negations) win on conflict via last-match-wins; + adding a .graphifyignore can only ever exclude MORE, never re-include a + .gitignore-excluded file (#945 kept: a dir with only a .gitignore still + gets sensible defaults). + + Shared by `_load_graphifyignore` (ancestor chain, loaded once before the + scan) and the live os.walk loop in `detect()` (called per-directory as + each descendant is visited), so nested ignore files *below* the scan + root are honored too — previously only the scan root and its ancestors + were read, so e.g. `vendor/sub/.gitignore` was silently ignored (#1206). + """ + patterns: list[tuple[Path, str]] = [] + for fname in ((".gitignore", ".graphifyignore") if gitignore else (".graphifyignore",)): + ignore_file = d / fname + if _path_exists(ignore_file): + for raw in _read_text(ignore_file, encoding="utf-8-sig", errors="ignore").splitlines(): + line = _parse_gitignore_line(raw) + if line: + patterns.append((d, line)) + return patterns + + +def _load_graphifyignore(root: Path, *, gitignore: bool = True) -> list[tuple[Path, str]]: + """Read .graphifyignore files and return (anchor_dir, pattern) pairs. + + Patterns are returned outer-first so that inner (closer) rules are + appended last and win via last-match-wins semantics — matching gitignore + behavior exactly. + + Walk ceiling: the nearest VCS root if inside a repo, otherwise the scan + root itself (hermetic — no leakage across unrelated sibling projects). + + Covers the scan root and its ancestors only — directories *below* the + scan root are picked up live during the os.walk in `detect()` instead, + since they aren't known until the walk reaches them (#1206). + """ + root = _resolve_path(root) + ceiling = _find_vcs_root(root) or root + + # Collect ancestor dirs from ceiling down to root (outer → inner) + dirs: list[Path] = [] + current = root + while True: + dirs.append(current) + if current == ceiling: + break + current = current.parent + dirs.reverse() # ceiling first, scan root last + + patterns: list[tuple[Path, str]] = [] + + # $GIT_DIR/info/exclude is repo-root-scoped and, per git, ranks below every + # per-directory .gitignore/.graphifyignore — so load it first (lowest priority + # under last-match-wins) anchored at the VCS root, letting a nearer `!` + # re-include still override it (#1810). + info_exclude = _git_info_exclude(ceiling) if gitignore else None + if info_exclude is not None: + for raw in _read_text(info_exclude, encoding="utf-8-sig", errors="ignore").splitlines(): + line = _parse_gitignore_line(raw) + if line: + patterns.append((ceiling, line)) + + for d in dirs: + patterns.extend(_load_dir_own_ignore(d, gitignore=gitignore)) + return patterns + + +def _match_anchored_ignore_pattern(path: str, pattern: str) -> bool: + """Match an anchored gitignore pattern without letting ``*`` cross ``/``.""" + path_parts = tuple(path.split("/")) + pattern_parts = tuple(pattern.split("/")) + + @lru_cache(maxsize=None) + def _matches(path_idx: int, pattern_idx: int) -> bool: + if pattern_idx == len(pattern_parts): + return path_idx == len(path_parts) + + part = pattern_parts[pattern_idx] + if part == "**": + if pattern_idx == len(pattern_parts) - 1: + return path_idx < len(path_parts) + return _matches(path_idx, pattern_idx + 1) or ( + path_idx < len(path_parts) + and _matches(path_idx + 1, pattern_idx) + ) + + return ( + path_idx < len(path_parts) + and fnmatch.fnmatchcase(path_parts[path_idx], part) + and _matches(path_idx + 1, pattern_idx + 1) + ) + + return _matches(0, 0) + + +def _is_ignored( + path: Path, + root: Path, + patterns: list[tuple[Path, str]], + *, + _cache: dict[Path, bool] | None = None, +) -> bool: + """Return True if the path should be ignored per .graphifyignore patterns. + + Uses gitignore last-match-wins semantics: all patterns are evaluated in + order; the final matching pattern determines the result. Negation patterns + (starting with !) un-ignore a previously ignored path. + + Enforces gitignore's parent-exclusion rule: a ! pattern cannot re-include + a file whose ancestor directory is already excluded. + + _cache: optional dict shared across calls within the same scan. Ancestor + directory results are memoised so files under the same subtree don't + re-evaluate the same patterns repeatedly. + """ + if not patterns: + return False + + def _eval(target: Path) -> bool: + """Apply last-match-wins to a single target path.""" + if _cache is not None and target in _cache: + return _cache[target] + def _matches(rel: str, p: str, path_relative: bool) -> bool: + if path_relative: + return _match_anchored_ignore_pattern(rel, p) + parts = rel.split("/") + if fnmatch.fnmatch(rel, p): + return True + if fnmatch.fnmatch(_nfc(target.name), p): + return True + for i, part in enumerate(parts): + if fnmatch.fnmatch(part, p): + return True + if fnmatch.fnmatch("/".join(parts[:i + 1]), p): + return True + return False + + result = False + for anchor, pattern in patterns: + negated = pattern.startswith("!") + raw = pattern[1:] if negated else pattern + directory_only = raw.endswith("/") + path_relative = "/" in raw.rstrip("/") + p = raw.strip("/") + if not p: + continue + + # gitignore semantics: patterns from A/.gitignore apply ONLY to paths + # under A. Matching non-anchored patterns against root-relative paths + # let e.g. .hypothesis/.gitignore's bare "*" ignore the ENTIRE repo + # (detect() returned 0 files). The anchor dir itself is exempt — an + # ignore file governs its directory's contents, not the directory. + matched = False + try: + rel_anchor = _nfc(str(target.relative_to(anchor)).replace(os.sep, "/")) + except ValueError: + continue # target outside this pattern's anchor: cannot match + if rel_anchor != ".": + matched = _matches(rel_anchor, p, path_relative=path_relative) + if matched and directory_only and not _path_is_dir(target): + matched = False + + if matched: + result = not negated # last match wins; ! flips to un-ignore + if _cache is not None: + _cache[target] = result + return result + + # Gitignore parent-exclusion rule: a ! re-include cannot rescue a file + # whose ancestor directory is already excluded. Walk ancestors top-down; + # if any ancestor is excluded, the file is excluded regardless of later + # ! patterns targeting the file or a sub-path. + try: + rel_parts = path.relative_to(root).parts + except ValueError: + return _eval(path) + + ancestor = root + for part in rel_parts[:-1]: + ancestor = ancestor / part + if _eval(ancestor): + return True + return _eval(path) + + +def ignored_predicate( + root: Path, + *, + extra_excludes: list[str] | None = None, + gitignore: bool = True, +) -> Callable[[Path], bool]: + """Build a per-path predicate answering "would detect() exclude this path?". + + Mirrors detect()'s ignore decisions for a single existing path WITHOUT + re-walking the corpus, from the same machinery detect() uses: the ancestor + .graphifyignore/.gitignore chain (_load_graphifyignore), CLI/persisted + ``--exclude`` patterns appended last at the root anchor (#947), nested + per-directory ignore files along the path's own lineage (#1206), the + _is_noise_dir directory pruning, and _SKIP_FILES. The sensitive-file + heuristic (_is_sensitive) is deliberately NOT included: callers use this + predicate as positive evidence of a live ignore RULE (#2495), and a + heuristic match is not user intent. + + Nested patterns are loaded lazily, once per directory, into one shared + pattern list. That accumulation cannot cross-contaminate results — a + pattern only ever matches paths under its anchor directory, so patterns + from a sibling subtree are inert — which is the same invariant detect()'s + live os.walk relies on, and it keeps the shared _is_ignored cache valid. + """ + root = _resolve_path(root) + patterns = _load_graphifyignore(root, gitignore=gitignore) + if extra_excludes: + for pat in extra_excludes: + line = _parse_gitignore_line(pat) + if line: + patterns.append((root, line)) + cache: dict[Path, bool] = {} + # root's own ignore file is the last entry of _load_graphifyignore's chain. + loaded_dirs: set[Path] = {root} + + def _ignored(path: Path) -> bool: + path = Path(os.path.abspath(path)) + try: + rel_parts = path.relative_to(root).parts + except ValueError: + return False # outside the scan root: detect() never considered it + if path.name in _SKIP_FILES: + return True + # Noise-dir pruning: os.walk never descends these, so anything beneath + # one is excluded from the corpus regardless of ignore patterns. + parent = root + for part in rel_parts[:-1]: + if _is_noise_dir(part, parent): + return True + parent = parent / part + # Load ignore files along this path's own lineage — detect()'s walk + # would have loaded exactly these before reaching the file (#1206). + ancestor = root + for part in rel_parts[:-1]: + ancestor = ancestor / part + if ancestor not in loaded_dirs: + loaded_dirs.add(ancestor) + patterns.extend(_load_dir_own_ignore(ancestor, gitignore=gitignore)) + return _is_ignored(path, root, patterns, _cache=cache) + + return _ignored + + +def _auto_follow_symlinks(root: Path) -> bool: + """Return whether ``root`` has any direct symlinked child. + + Kept for callers that import the private helper, but detection no longer + enables symlink following automatically. Following symlinks is now an + explicit opt-in, and out-of-root symlink targets are never indexed. + """ + try: + with os.scandir(_os_path(root)) as entries: + for entry in entries: + if entry.is_symlink(): + return True + except (OSError, PermissionError): + pass + return False + + +def _resolves_under_root(path: Path, root: Path) -> bool: + """True when ``path`` resolves to a target inside ``root``.""" + try: + _resolve_path(path).relative_to(_resolve_path(root)) + except (OSError, RuntimeError, ValueError): + return False + return True + + +def detect(root: Path, *, follow_symlinks: bool | None = None, google_workspace: bool | None = None, extra_excludes: list[str] | None = None, cache_root: Path | None = None, gitignore: bool = True) -> dict: + root = _resolve_path(root) + configured_out_dir = root / GRAPHIFY_OUT + configured_out_names = {configured_out_dir.name} + try: + configured_out_dir = _resolve_path(configured_out_dir) + except (OSError, RuntimeError): + configured_out_dir = configured_out_dir.absolute() + configured_out_names.add(configured_out_dir.name) + # .graphifyinclude support was removed (#2112): its loader and matchers had + # no consumers, so the file has been a silent no-op since dot directories + # became indexed by default (#873). Surface that once per scan so a + # leftover allowlist file is not a silent behavior change. + if _path_is_file(root / ".graphifyinclude"): + import sys as _sys + print( + "[graphify] WARNING: .graphifyinclude is no longer supported " + "(it has been non-functional since dot directories became indexed " + "by default); to re-include ignored paths, use ! negation patterns " + "in .graphifyignore.", + file=_sys.stderr, + ) + if follow_symlinks is None: + follow_symlinks = False + google_workspace = google_workspace_enabled() if google_workspace is None else google_workspace + files: dict[FileType, list[str]] = { + FileType.CODE: [], + FileType.DOCUMENT: [], + FileType.PAPER: [], + FileType.IMAGE: [], + FileType.VIDEO: [], + } + total_words = 0 + + def _wc(path: Path) -> int: + # Cache word counts against each file's stat signature so unchanged + # PDFs/docx aren't re-parsed on every run just to size the corpus (#1656). + # cache_root (when given, e.g. from `extract --out`) keeps this cache out + # of the scanned corpus (#1747). + from graphify import cache as _cache + return _cache.cached_word_count(path, root, count_words, cache_root=cache_root) + + skipped_sensitive: list[str] = [] + unclassified: list[str] = [] + # Files/dirs dropped by a .gitignore/.graphifyignore rule. Recorded so an + # over-broad ignore (or a legitimately-ignored subtree) is visible instead + # of silently vanishing from the graph (#1922). Directory-level entries keep + # this bounded — a pruned `data/` is one entry, not one per contained file. + ignored: list[str] = [] + pruned_noise: list[str] = [] + ignore_patterns = _load_graphifyignore(root, gitignore=gitignore) + ignore_cache: dict[Path, bool] = {} # shared across all _is_ignored calls in this scan + # CLI --exclude patterns are anchored at the scan root and appended last + # so they win over any .graphifyignore/.gitignore rules (#947). + if extra_excludes: + for pat in extra_excludes: + line = _parse_gitignore_line(pat) + if line: + ignore_patterns.append((root, line)) + + # Always include graphify-out/memory/ - query results filed back into the graph + memory_dir = root / GRAPHIFY_OUT / "memory" + scan_paths = [root] + if _path_exists(memory_dir): + scan_paths.append(memory_dir) + + seen: set[Path] = set() + all_files: list[Path] = [] + + # os.walk swallows os.scandir errors by default (no onerror -> the failing + # directory subtree is silently skipped). That turns a transient + # PermissionError, or a directory created/deleted mid-walk (e.g. concurrent + # writes racing the scan), into a partial file list and, downstream, a + # silently partial graph.json. Record and surface every skipped directory + # so an incomplete enumeration is visible rather than silent. + walk_errors: list[str] = [] + + def _on_walk_error(err: OSError) -> None: + import sys as _sys + target = getattr(err, "filename", None) or "" + walk_errors.append(f"{target}: {err}") + print( + f"[graphify] WARNING: could not scan {target} ({err}); " + f"its files are missing from this run's enumeration.", + file=_sys.stderr, + ) + + for scan_root in scan_paths: + in_memory_tree = _path_exists(memory_dir) and str(scan_root).startswith( + str(memory_dir) + ) + for dirpath, dirnames, filenames in _walk_path( + scan_root, followlinks=follow_symlinks, onerror=_on_walk_error + ): + dp = Path(dirpath) + # os.walk must stay in the extended Windows namespace so every + # recursive scandir call can reach long descendants. Convert the + # yielded path back immediately: graph/cache identities must use the + # ordinary UNC/drive spelling, not the transport-only \\?\ prefix. + if follow_symlinks and _path_is_symlink(dp): + real = str(_resolve_path(dp)) + parent_real = str(_resolve_path(dp.parent)) + if parent_real == real or parent_real.startswith(real + os.sep): + dirnames.clear() + continue + if not in_memory_tree: + # dp == root was already loaded by _load_graphifyignore (root is + # the last entry in its ancestor chain); every other directory + # reached by the walk is a descendant below the scan root, whose + # own .gitignore/.graphifyignore is unknown until we get here. + # Load it now, before pruning dp's children, so a nested ignore + # file governs its own subtree the same way git honors it (#1206). + if dp != root: + ignore_patterns.extend(_load_dir_own_ignore(dp, gitignore=gitignore)) + # Prune noise dirs in-place so os.walk never descends into them. + # Dot dirs are allowed — users often want .github/, .claude/, etc. + # Framework caches (.next, .nuxt, …) are caught by _is_noise_dir. + # Negations need no special-casing here: _is_ignored already applies + # last-match-wins (so `!dir/` un-ignores a directory and it won't be + # pruned) and the gitignore parent-exclusion rule (a `!` cannot rescue + # a file beneath an excluded dir), so descending an ignored directory to + # look for a re-included file is never necessary. The previous blanket + # `has_negation` disabled directory pruning for EVERY ignored dir whenever + # any `!` rule existed — e.g. a single `!docs/**` made the walk descend + # bin/, obj/, wwwroot/, generated/, … : a pathological slowdown on large + # repos for no correctness gain. + kept_dirs: list[str] = [] + for d in dirnames: + child = dp / d + is_configured_out = False + if d in configured_out_names: + try: + is_configured_out = _resolve_path(child) == configured_out_dir + except (OSError, RuntimeError): + pass + if is_configured_out: + pruned_noise.append(str(child) + os.sep) + continue + if _is_noise_dir(d, dp): + # Record pruned-as-noise dirs so a wrongly-pruned real + # source dir is at least traceable in the output rather + # than vanishing silently (#2058). + pruned_noise.append(str(dp / d) + os.sep) + continue + if _is_ignored(dp / d, root, ignore_patterns, _cache=ignore_cache): + ignored.append(str(dp / d) + os.sep) + continue + kept_dirs.append(d) + dirnames[:] = kept_dirs + if follow_symlinks: + safe_dirs: list[str] = [] + for d in dirnames: + child = dp / d + if _path_is_symlink(child) and not _resolves_under_root(child, root): + skipped_sensitive.append(str(child) + " [symlink target outside scan root]") + continue + safe_dirs.append(d) + dirnames[:] = safe_dirs + for fname in filenames: + if fname in _SKIP_FILES: + continue + p = dp / fname + if p not in seen: + seen.add(p) + all_files.append(p) + + all_files.sort(key=lambda p: str(p)) + + converted_dir = root / GRAPHIFY_OUT / "converted" + + for p in all_files: + # For memory dir files, skip hidden/noise filtering + in_memory = _path_exists(memory_dir) and str(p).startswith(str(memory_dir)) + if not in_memory: + # Skip files inside our own converted/ dir (avoid re-processing sidecars) + if str(p).startswith(str(converted_dir)): + continue + if not in_memory and _is_ignored(p, root, ignore_patterns, _cache=ignore_cache): + ignored.append(str(p)) + continue + if not _resolves_under_root(p, root): + skipped_sensitive.append(str(p) + " [symlink target outside scan root]") + continue + if _is_sensitive(p): + skipped_sensitive.append(str(p)) + continue + ftype = classify_file(p) + if not ftype: + # Considered but unclassifiable: an extension not in any supported set, + # or an extensionless, non-shebang file (Dockerfile, Gemfile, Makefile, + # Rakefile, LICENSE, ...). Previously these left no trace at all — not + # counted, not listed — so a user couldn't tell they were seen (#1692). + unclassified.append(str(p)) + continue + if ftype: + if p.suffix.lower() in GOOGLE_WORKSPACE_EXTENSIONS: + if not google_workspace: + skipped_sensitive.append( + str(p) + + " [Google Workspace shortcut skipped - pass --google-workspace " + "or set GRAPHIFY_GOOGLE_WORKSPACE=1]" + ) + continue + try: + md_path = convert_google_workspace_file(p, converted_dir, xlsx_to_markdown=xlsx_to_markdown, root=root) + except Exception as exc: + skipped_sensitive.append(str(p) + f" [Google Workspace export failed: {exc}]") + continue + if md_path: + if _is_ignored(md_path, root, ignore_patterns, _cache=ignore_cache): + continue + files[ftype].append(str(md_path)) + total_words += _wc(md_path) + else: + skipped_sensitive.append(str(p) + " [Google Workspace export produced no readable text]") + continue + # Office files: convert to markdown sidecar so subagents can read them + if p.suffix.lower() in OFFICE_EXTENSIONS: + md_path = convert_office_file(p, converted_dir, root=root) + if md_path: + if _is_ignored(md_path, root, ignore_patterns, _cache=ignore_cache): + continue + files[ftype].append(str(md_path)) + total_words += _wc(md_path) + else: + # Conversion failed (library not installed) - skip with note + skipped_sensitive.append(str(p) + " [office conversion failed - pip install graphifyy[office]]") + continue + files[ftype].append(str(p)) + if ftype != FileType.VIDEO: + total_words += _wc(p) + + for ftype in files: + files[ftype].sort() + + total_files = sum(len(v) for v in files.values()) + needs_graph = total_words >= CORPUS_WARN_THRESHOLD + + # Determine warning - lower bound, upper bound, or sensitive files skipped + warning: str | None = None + if not needs_graph: + warning = ( + f"Corpus is ~{total_words:,} words - fits in a single context window. " + f"You may not need a graph." + ) + elif total_words >= CORPUS_UPPER_THRESHOLD or total_files >= FILE_COUNT_UPPER: + warning = ( + f"Large corpus: {total_files} files · ~{total_words:,} words. " + f"Semantic extraction will be expensive (many Claude tokens). " + f"Consider running on a subfolder." + ) + + return { + "files": {k.value: v for k, v in files.items()}, + "total_files": total_files, + "total_words": total_words, + "needs_graph": needs_graph, + "warning": warning, + "skipped_sensitive": skipped_sensitive, + "unclassified": sorted(unclassified), + "walk_errors": walk_errors, + "ignored": sorted(ignored), + "pruned_noise_dirs": sorted(pruned_noise), + "graphifyignore_patterns": len(ignore_patterns), + "scan_root": str(_resolve_path(root)), + } + + +def _md5_file(path: Path) -> str: + """MD5 of file contents streamed in 64KB chunks — for change detection only.""" + import hashlib as _hl + h = _hl.md5(usedforsecurity=False) + try: + with open(_os_path(path), "rb") as f: + for chunk in iter(lambda: f.read(65536), b""): + h.update(chunk) + except OSError: + return "" + return h.hexdigest() + + +def _stat_and_hash(path_str: str) -> tuple[str, float, str] | None: + """Stat + MD5 a single file; returns None on OSError (e.g. deleted mid-run).""" + try: + p = Path(path_str) + return path_str, _path_stat(p).st_mtime, _md5_file(p) + except OSError: + return None + + +def _nfc(s: str) -> str: + """NFC-normalize a path string used as a manifest key. + + On macOS, ``os.walk`` / ``getcwd`` yield NFD paths while path literals + and many skill-substituted roots are NFC. Raw string compare then treats + every file as both deleted and new, forcing a full re-extract (#2221). + Same boundary as the Office sidecar hash fix (#1226). + """ + import unicodedata + return unicodedata.normalize("NFC", s) + + +def _to_relative_for_storage(key: str, root: Path) -> str: + """Return ``key`` as a forward-slash relative path from ``root``. + + Keys outside ``root`` (out-of-tree symlinked sources, external --include + paths) and already-relative keys pass through unchanged — mirrors the + fallback in :func:`graphify.watch._relativize_source_files` so the + on-disk artifact survives the round-trip even when some paths cannot be + portably encoded. + + Only ``root`` is resolved — the key itself is relativized symbolically + so an in-root symlink (e.g. ``alias.py -> sub/target.py``) is stored + under its own name. Resolving the key would point the stored entry at + the symlink target, and the original key would then miss on reload and + re-extract on every incremental run. + + Both sides of ``relpath`` are NFC'd first: stamped keys may already be + NFC while ``Path(root).resolve()`` is NFD on macOS, and a mixed-form + compare would mark an in-root file as ``../…`` and keep it absolute + (#2221 / #777). + """ + p = Path(key) + if not p.is_absolute(): + return key + try: + base = _nfc(str(_resolve_path(root))) + rel = os.path.relpath(_nfc(str(p)), base) + except (ValueError, OSError): + return key # outside root (e.g. Windows cross-drive) + # ``os.path.relpath`` happily produces ``../foo`` for paths outside + # root; mirror the prior ``relative_to``-raises-ValueError semantics by + # keeping out-of-root entries in their absolute form. + if rel == ".." or rel.startswith(".." + os.sep) or rel.startswith("../"): + return key + return rel.replace(os.sep, "/") + + +def _to_absolute_from_storage(key: str, root: Path) -> str: + """Inverse of :func:`_to_relative_for_storage`. + + Re-anchor a stored key against ``root``. Already-absolute keys + (legacy manifests, out-of-root entries) pass through unchanged so + that newly-loaded manifests from before this change remain readable. + Uses ``Path(root).resolve()`` so the produced absolute path matches + what :func:`detect` returns (which also resolves the scan root). + NFC both sides so a relative key and an NFD-resolved root still join + to the same string form the rest of the manifest path uses (#2221). + """ + p = Path(key) + if p.is_absolute(): + return str(p) + # NFC the joined result so an NFD-resolved root + relative key lands on + # the same form load_manifest / detect_incremental compare against. + return _nfc(str(_resolve_path(root) / p)) + + +def load_manifest( + manifest_path: str = _MANIFEST_PATH, + *, + root: Path | None = None, +) -> dict: + """Load the manifest from a previous run. Returns {} on any error. + + When ``root`` is provided, stored relative keys are re-anchored against + it so callers see absolute paths regardless of on-disk format. Legacy + manifests with absolute keys pass through unchanged, so a graphify-out/ + written by an older version (or by a caller that didn't supply ``root`` + to :func:`save_manifest`) remains readable. + + Keys are NFC-normalized on load so a manifest written under one Unicode + form still matches a scan that yields the other (#2221). + """ + try: + raw = json.loads(_read_text(manifest_path, encoding="utf-8")) + except Exception: + return {} + if not isinstance(raw, dict): + return raw + if root is None: + return {_nfc(k): v for k, v in raw.items()} + return {_nfc(_to_absolute_from_storage(k, root)): v for k, v in raw.items()} + + +def save_manifest( + files: dict[str, list[str]], + manifest_path: str = _MANIFEST_PATH, + *, + kind: str = "both", + root: Path | None = None, + scan_corpus: set[str] | list[str] | None = None, + clear_semantic: set[str] | list[str] | None = None, + clear_ast: set[str] | list[str] | None = None, +) -> None: + """Save current file mtimes + content hashes for change detection. + + kind="ast" — written by `graphify update` (AST-only rebuild). Stamps + ast_hash; preserves an existing semantic_hash only when + the file content is unchanged (mtime + hash match). + kind="semantic" — written by `graphify extract` after semantic extraction. + Stamps semantic_hash; preserves existing ast_hash. + kind="both" — full pipeline: stamps both hashes (default). + + When ``root`` is provided, keys are relativized against it before write + (forward-slash, posix-style) so the on-disk manifest is portable across + machines and checkout locations (#777). Out-of-root entries are written + as absolute so they continue to round-trip on the saving machine. + When ``root`` is None the legacy absolute-keyed format is preserved. + + ``scan_corpus`` (#1908): full-scan callers pass the COMPLETE detect + corpus (absolute paths) so seeded rows for in-root files that are still + alive on disk but no longer part of the scan (newly excluded via + .graphifyignore/.gitignore/--exclude) are dropped instead of surviving + forever and masquerading as deletions in detect_incremental. It must be + the RAW detect output, not a stamp-filtered subset — pruning to a + filtered set would erase rows the filter merely omitted (failed chunks, + --code-only doc rows). Out-of-root entries are never pruned. Callers + saving a SUBSET of files (changed_paths hooks, skill runbooks, #917) + must leave this None so their untouched rows are preserved. + + ``clear_semantic`` (#1948): files that were dispatched this run but + produced no stamped output (e.g. the LLM omitted their chunk on a + --force re-run) are absent from ``files``, so the seed loop below would + otherwise copy their prior semantic_hash verbatim — masking the omission + and making detect_incremental(kind="semantic") report them unchanged. + Pass the set of such files (any path form ``scan_corpus`` accepts) to + force their seeded semantic_hash to "" instead of inheriting it. + + ``clear_ast`` (#2543): same idea for AST failures (missing optional extra, + zero-node anomalous extract). Blanks BOTH ``ast_hash`` and + ``semantic_hash`` on the seeded row so either detect_incremental kind + re-queues the file after the failure is fixed, without deleting + graphify-out/. + """ + existing = load_manifest(manifest_path, root=root) + + # Index both raw and NFC forms so scan/clear membership survives the + # same NFC/NFD mismatch that breaks manifest lookups (#2221). + def _path_index(paths: set[str] | list[str] | None) -> set[str] | None: + if paths is None: + return None + indexed: set[str] = set() + for p in paths: + indexed.add(p) + indexed.add(_nfc(p)) + return indexed + + scan_set = _path_index(scan_corpus) + clear_set = _path_index(clear_semantic) + clear_ast_set = _path_index(clear_ast) + try: + root_res: Path | None = _resolve_path(root) if root is not None else None + except (OSError, RuntimeError): + root_res = Path(root) if root is not None else None + + def _in_scan(path_str: str) -> bool: + if path_str in scan_set or _nfc(path_str) in scan_set: + return True + try: + resolved = str(_resolve_path(path_str)) + return resolved in scan_set or _nfc(resolved) in scan_set + except (OSError, RuntimeError): + return False + + def _in_clear(path_str: str) -> bool: + if clear_set is None: + return False + if path_str in clear_set or _nfc(path_str) in clear_set: + return True + try: + resolved = str(_resolve_path(path_str)) + return resolved in clear_set or _nfc(resolved) in clear_set + except (OSError, RuntimeError): + return False + + def _in_clear_ast(path_str: str) -> bool: + if clear_ast_set is None: + return False + if path_str in clear_ast_set or _nfc(path_str) in clear_ast_set: + return True + try: + resolved = str(_resolve_path(path_str)) + return resolved in clear_ast_set or _nfc(resolved) in clear_ast_set + except (OSError, RuntimeError): + return False + + def _in_root(path_str: str) -> bool: + # Without a root we cannot tell in-root from out-of-root; fail open + # (keep the row) so out-of-root corpora are never pruned by accident. + if root_res is None: + return False + p = Path(path_str) + try: + p.relative_to(root_res) + return True + except ValueError: + pass + try: + _resolve_path(p).relative_to(root_res) + return True + except (ValueError, OSError, RuntimeError): + return False + + def _normalise_entry(entry): + if isinstance(entry, (int, float)): + return {"mtime": entry, "ast_hash": "", "semantic_hash": ""} + if isinstance(entry, dict) and "hash" in entry and "ast_hash" not in entry: + return {"mtime": entry.get("mtime", 0), "ast_hash": entry["hash"], "semantic_hash": ""} + if isinstance(entry, dict): + return entry + return None + + # Seed from the existing manifest so incremental callers passing a subset + # of files don't silently erase entries for untouched files (#917). + # Prune entries whose file no longer exists on disk — those are genuine + # deletions that detect_incremental() should treat as gone. When the + # caller supplied the full scan corpus, additionally prune in-root rows + # the scan no longer covers: those files were excluded, not deleted, and + # keeping the row makes them look deleted on every future run (#1908). + manifest: dict[str, dict] = {} + for f, entry in existing.items(): + normalised = _normalise_entry(entry) + if normalised is None: + continue + try: + if not _path_exists(f): + continue + except OSError: + continue + if scan_set is not None and not _in_scan(f) and _in_root(f): + continue # excluded-but-alive: drop the stale row (#1908) + if clear_ast_set is not None and _in_clear_ast(f): + # AST failure this run (missing extra / zero nodes, #2543): blank + # both hashes so either detect_incremental kind re-queues. + normalised = {**normalised, "ast_hash": "", "semantic_hash": ""} + elif clear_set is not None and _in_clear(f): + # Dispatched-but-omitted this run: don't inherit the stale + # semantic_hash, or detect_incremental would call it unchanged (#1948). + normalised = {**normalised, "semantic_hash": ""} + manifest[f] = normalised + + all_files = [f for file_list in files.values() for f in file_list] + with ThreadPoolExecutor() as pool: + raw = pool.map(_stat_and_hash, all_files) + hashed: dict[str, tuple[float, str]] = { + r[0]: (r[1], r[2]) for r in raw if r is not None + } + + for f in all_files: + if f not in hashed: + continue # file deleted between detect() and manifest write + mtime, h = hashed[f] + key = _nfc(f) + prev = _normalise_entry(existing.get(key, {})) or {} + entry: dict = {"mtime": mtime} + if kind in ("ast", "both"): + entry["ast_hash"] = h + else: + entry["ast_hash"] = prev.get("ast_hash", "") + if kind in ("semantic", "both"): + entry["semantic_hash"] = h + else: + # Preserve semantic_hash only when content is unchanged + entry["semantic_hash"] = prev.get("semantic_hash", "") if h == prev.get("ast_hash", "") else "" + manifest[key] = entry + if root is not None: + # Persist in portable form: forward-slash relative paths. Keys outside + # ``root`` (out-of-tree symlinked corpora, --include sources) keep + # their absolute form so the manifest round-trips on the saving + # machine even when not every entry can be portably encoded. + # NFC after relativize so on-disk keys match what load_manifest + # re-anchors and compares against (#2221). + manifest = {_nfc(_to_relative_for_storage(k, root)): v for k, v in manifest.items()} + else: + manifest = {_nfc(k): v for k, v in manifest.items()} + from graphify.paths import write_json_atomic + # Atomic write: a crash mid-write must not leave a truncated manifest that + # detect_incremental then fails to parse. + write_json_atomic(manifest_path, manifest, indent=2) + + +def detect_incremental( + root: Path, + manifest_path: str = _MANIFEST_PATH, + *, + follow_symlinks: bool | None = None, + google_workspace: bool | None = None, + kind: str = "semantic", + extra_excludes: list[str] | None = None, + gitignore: bool = True, +) -> dict: + """Like detect(), but returns only new or modified files since the last run. + + kind="semantic" (default for extract): a file is "changed" when its + semantic_hash is missing or its content has changed since the last + semantic extraction pass. Use this for `graphify extract` so that + files touched by `graphify update` (AST-only) are re-extracted + semantically. + kind="ast": a file is "changed" when its ast_hash is missing or its + content has changed. Use this for `graphify update`. + + Fast path: mtime unchanged + hash matches → unchanged (free, no disk IO + beyond stat). Slow path: mtime bumped → compare MD5 against the relevant + hash field before re-extracting. + + Backwards compatible with legacy manifests storing plain float mtime values + or {mtime, hash} dicts (treated as ast_hash only; semantic_hash = miss). + + The ``follow_symlinks`` flag is forwarded to :func:`detect` so in-root + symlinked sub-trees are scanned consistently between full and incremental + runs. ``None`` (default) does not follow symlinked directories; callers must + opt in explicitly, and resolved targets outside the scan root are skipped. + """ + full = detect( + root, + follow_symlinks=follow_symlinks, + google_workspace=google_workspace, + extra_excludes=extra_excludes, + gitignore=gitignore, + ) + # Pass ``root`` so a manifest written with relative keys (post-#777) is + # re-anchored to the absolute form the rest of this function compares + # against. Legacy absolute-keyed manifests pass through unchanged. + manifest = load_manifest(manifest_path, root=root) + + if not manifest: + # No previous run - treat everything as new + full["incremental"] = True + full["new_files"] = full["files"] + full["unchanged_files"] = {k: [] for k in full["files"]} + full["new_total"] = full["total_files"] + full["deleted_files"] = [] + full["excluded_files"] = [] + return full + + new_files: dict[str, list[str]] = {k: [] for k in full["files"]} + unchanged_files: dict[str, list[str]] = {k: [] for k in full["files"]} + + for ftype, file_list in full["files"].items(): + for f in file_list: + # Manifest keys are NFC; scan paths may arrive NFD (#2221). + stored = manifest.get(_nfc(f)) + try: + current_mtime = _path_stat(f).st_mtime + except Exception: + current_mtime = 0 + + # Legacy manifest: plain float value stores only mtime. + # Compare with `!=` so backwards mtime motion (git checkout of an + # older commit, tarball restore, rsync --times) still triggers a + # re-extract; the previous `>` silently kept the stale cache and + # the graph drifted from disk (#1859). No stored hash means we + # cannot verify content — any mtime delta forces a re-extract, + # and the next save promotes the entry into the dict schema. + if isinstance(stored, (int, float)): + changed = current_mtime != stored + elif isinstance(stored, dict): + # Normalise legacy {mtime, hash} to new schema + if "hash" in stored and "ast_hash" not in stored: + stored = {"mtime": stored.get("mtime", 0), "ast_hash": stored["hash"], "semantic_hash": ""} + hash_key = "semantic_hash" if kind == "semantic" else "ast_hash" + stored_hash = stored.get(hash_key, "") + # Missing semantic_hash means update ran but extract hasn't — always re-extract + if not stored_hash: + changed = True + else: + stored_mtime = stored.get("mtime") + # Schema-drift guard (#1163): tolerate a nested {mtime: ...} + # dict or any non-numeric value without crashing. + if isinstance(stored_mtime, dict): + stored_mtime = stored_mtime.get("mtime") + if not isinstance(stored_mtime, (int, float)): + stored_mtime = None + if stored_mtime is None or current_mtime != stored_mtime: + # mtime bumped — verify with content hash before re-extracting + changed = _md5_file(Path(f)) != stored_hash + else: + changed = False + else: + changed = True # unknown format, re-extract to be safe + + if changed: + new_files[ftype].append(f) + else: + unchanged_files[ftype].append(f) + + # Manifest rows that left the corpus, split by disk existence (#1908): + # a row whose file is gone from DISK is a genuine deletion (its cached + # nodes are ghosts); a row whose file still exists but is out of the + # current scan was EXCLUDED (ignore rules / --exclude changed) and must + # not be reported as deleted. Mirrors the watch-side excluded-vs-deleted + # distinction (#1795). + current_files = {_nfc(f) for flist in full["files"].values() for f in flist} + deleted_files: list[str] = [] + excluded_files: list[str] = [] + for f in manifest: + if _nfc(f) in current_files: + continue + try: + alive = _path_exists(f) + except OSError: + alive = False + (excluded_files if alive else deleted_files).append(f) + + new_total = sum(len(v) for v in new_files.values()) + full["incremental"] = True + full["new_files"] = new_files + full["unchanged_files"] = unchanged_files + full["new_total"] = new_total + full["deleted_files"] = deleted_files + full["excluded_files"] = excluded_files + return full diff --git a/dedup.py b/dedup.py new file mode 100644 index 000000000..ec7c5e0f6 --- /dev/null +++ b/dedup.py @@ -0,0 +1,3017 @@ +"""resolution — moved verbatim from graphify/extract.py.""" +from __future__ import annotations + +from pathlib import Path +from typing import Any, Callable + +from graphify.paths import ( + glob_paths as _glob_paths, + path_exists as _path_exists, + path_is_dir as _path_is_dir, + path_is_file as _path_is_file, + path_stat as _path_stat, + read_bytes as _read_file_bytes, + read_text as _read_file_text, + resolve_path as _resolve_path, +) +from graphify.extractors.models import LanguageConfig, _JS_CACHE_BYPASS_SUFFIXES, _NamespaceExportFact, _StarExportFact, _SymbolAliasFact, _SymbolDeclarationFact, _SymbolExportFact, _SymbolImportFact, _SymbolResolutionFacts, _SymbolUseFact, _WORKSPACE_PACKAGE_CACHE # noqa: E402,F401 +from graphify.extractors.base import ( # noqa: F401 + _LANGUAGE_BUILTIN_GLOBALS, + _file_stem, + _make_id, + _read_text, +) +import hashlib +import json +import os +import re +import sys + + +_TSCONFIG_ALIAS_CACHE: dict[str, dict[str, list[str]]] = {} + +# compilerOptions.baseUrl per config path, as an absolute dir (#2153). +_TSCONFIG_BASEURL_CACHE: "dict[str, Path | None]" = {} + +_WORKSPACE_MANIFEST_NAMES = ("pnpm-workspace.yaml", "package.json") + +_JS_RESOLVE_EXTS = (".ts", ".tsx", ".mts", ".cts", ".svelte", ".js", ".jsx", ".mjs", ".cjs") + +_JS_INDEX_FILES = ("index.ts", "index.tsx", "index.svelte", "index.js", "index.jsx", "index.mjs") + +def _resolve_js_import_path(candidate: Path) -> Path: + """Resolve a JS/TS/Svelte import target to a local file when it exists.""" + candidate = Path(os.path.normpath(candidate)) + if _path_is_file(candidate): + return candidate + + # TS ESM convention: imports often spell .js/.jsx while source is .ts/.tsx. + if candidate.suffix == ".js": + ts_candidate = candidate.with_suffix(".ts") + if _path_is_file(ts_candidate): + return ts_candidate + elif candidate.suffix == ".jsx": + tsx_candidate = candidate.with_suffix(".tsx") + if _path_is_file(tsx_candidate): + return tsx_candidate + + # Append extensions to the full filename, which covers extensionless imports, + # multi-dot helpers, and Svelte 5 rune files like Foo.svelte.ts. + for ext in _JS_RESOLVE_EXTS: + with_ext = candidate.parent / f"{candidate.name}{ext}" + if _path_is_file(with_ext): + return with_ext + + # Only fall back to directory indexes after file candidates lose. + if _path_is_dir(candidate): + for index_name in _JS_INDEX_FILES: + index_candidate = candidate / index_name + if _path_is_file(index_candidate): + return index_candidate + + return candidate + +def _strip_jsonc(text: str) -> str: + """Strip // line comments, /* */ block comments, and trailing commas from JSONC. + + Preserves string contents (including // and /* inside strings) by skipping over + quoted spans first. Required for tsconfig.json files generated by SvelteKit, + NestJS, Vite, T3, Astro, etc., which use JSONC by default (#700). + """ + # Remove block and line comments while leaving string literals untouched. + pattern = re.compile( + r'"(?:\\.|[^"\\])*"' # double-quoted string (with escapes) + r"|/\*.*?\*/" # /* block comment */ + r"|//[^\n]*", # // line comment + re.DOTALL, + ) + + def _replace(match: re.Match) -> str: + token = match.group(0) + if token.startswith('"'): + return token + return "" + + stripped = pattern.sub(_replace, text) + # Remove trailing commas before } or ] (allowing whitespace between). + stripped = re.sub(r",(\s*[}\]])", r"\1", stripped) + return stripped + +def _read_tsconfig_aliases(tsconfig: Path, base_dir: Path, seen: set) -> dict[str, list[str]]: + """Recursively read path aliases from a tsconfig, following extends chains. + + Child config paths override parent. Circular extends are detected via seen set. + npm package configs (e.g. @tsconfig/svelte) are skipped since they're not on disk. + Handles JSONC (comments + trailing commas) which is the default tsconfig format + for SvelteKit, NestJS, Vite, T3, Astro, etc. (#700). + """ + if str(tsconfig) in seen: + return {} + seen.add(str(tsconfig)) + try: + raw = _read_file_text(tsconfig, encoding="utf-8") + except Exception as e: + print(f" warning: could not read {tsconfig} ({type(e).__name__}: {e})", file=sys.stderr, flush=True) + return {} + try: + data = json.loads(raw) + except json.JSONDecodeError: + try: + data = json.loads(_strip_jsonc(raw)) + except json.JSONDecodeError as e: + print(f" warning: failed to parse {tsconfig} as JSON/JSONC ({e.msg} at line {e.lineno} col {e.colno})", file=sys.stderr, flush=True) + return {} + except Exception as e: + print(f" warning: failed to parse {tsconfig} ({type(e).__name__}: {e})", file=sys.stderr, flush=True) + return {} + + aliases: dict[str, list[str]] = {} + # `extends` may be a string or, since TypeScript 5.0, an array of paths. + # For an array, parents are processed in order with later entries + # overriding earlier ones; the extending config (paths below) overrides + # all parents. Without the list branch, an array `extends` raised + # `AttributeError: 'list' object has no attribute 'startswith'`, which + # _safe_extract turned into a skip of the whole file. + extends = data.get("extends") + if isinstance(extends, str): + extends_list = [extends] + elif isinstance(extends, list): + extends_list = [e for e in extends if isinstance(e, str)] + else: + extends_list = [] + for ext in extends_list: + # Skip scoped npm package configs (e.g. @tsconfig/svelte) — not on disk. + if not ext or ext.startswith("@"): + continue + extended_path = _resolve_path(base_dir / ext) + if not extended_path.suffix: + extended_path = extended_path.with_suffix(".json") + if _path_exists(extended_path): + aliases.update(_read_tsconfig_aliases(extended_path, extended_path.parent, seen)) + + # tsconfig `paths` are resolved relative to `baseUrl` (itself relative to + # the tsconfig's directory), not the tsconfig directory directly. Honoring + # baseUrl is required for the common monorepo / NestJS layout where + # baseUrl points at a subdirectory, e.g. baseUrl "./src" with + # "@services/*": ["services/*"] must resolve to /src/services rather + # than /services. Defaults to "." so configs without baseUrl (paths + # relative to the tsconfig dir, the TS 4.1+ behavior) keep working. + compiler_options = data.get("compilerOptions", {}) + base_url = compiler_options.get("baseUrl") or "." + paths_base = base_dir / base_url + paths = compiler_options.get("paths", {}) + for alias, targets in paths.items(): + if not targets: + continue + # Keep ALL targets in declared order — tsc tries each until one resolves + # on disk. Discarding the fallbacks (#1531) misresolved/dropped imports + # whose file lived at a non-first target. Preserve wildcard tokens in + # both sides until the resolver substitutes the captured segment, then + # normalizes the concrete path (#927). Empty/non-string entries are skipped. + target_patterns = [ + str(paths_base / t) + for t in targets + if isinstance(t, str) and t + ] + if target_patterns: + aliases[alias] = target_patterns + + return aliases + +def _read_json_config(path: Path) -> "dict | None": + """Parse a tsconfig/jsconfig as JSON, falling back to JSONC (#2153). + + Mirrors the read/parse handling in `_read_tsconfig_aliases`; returns None on + any unreadable or unparseable file so a malformed config degrades to "no + baseUrl" instead of raising. + """ + try: + raw = _read_file_text(path, encoding="utf-8", errors="replace") + except OSError: + return None + for candidate in (raw, _strip_jsonc(raw)): + try: + data = json.loads(candidate) + except Exception: + continue + return data if isinstance(data, dict) else None + return None + +def _find_js_config(start_dir: Path) -> "tuple[Path, Path] | None": + """Nearest tsconfig.json/jsconfig.json walking up from start_dir. + + `jsconfig.json` is the plain-JS spelling of the same file (already indexed by + json_config.py) and was never probed here, so a Rails/webpacker project that + configures resolution in jsconfig.json got no aliases at all (#2153). + tsconfig.json wins when both sit in one directory, matching tsc and editors, + which consult jsconfig.json only when there is no tsconfig.json. + """ + current = _resolve_path(start_dir) + for candidate in [current, *current.parents]: + for name in ("tsconfig.json", "jsconfig.json"): + config = candidate / name + if _path_exists(config): + return config, candidate + return None + +def _load_tsconfig_aliases(start_dir: Path) -> dict[str, list[str]]: + """Walk up from start_dir to find tsconfig/jsconfig.json and return compilerOptions.paths aliases. + + Follows extends chains so SvelteKit/Nuxt/NestJS inherited aliases are included. + Returns a dict mapping alias patterns to ordered resolved target patterns; + wildcard tokens remain intact for substitution during resolution (#927). + Result is cached by config path string. + """ + found = _find_js_config(start_dir) + if found is None: + return {} + config, candidate = found + key = str(config) + if key not in _TSCONFIG_ALIAS_CACHE: + _TSCONFIG_ALIAS_CACHE[key] = _read_tsconfig_aliases(config, candidate, seen=set()) + return _TSCONFIG_ALIAS_CACHE[key] + +def _load_tsconfig_base_url(start_dir: Path) -> "Path | None": + """`compilerOptions.baseUrl` of the nearest config, as an absolute directory. + + baseUrl was only ever used as the base that `paths` targets resolve against, + so a config declaring baseUrl and NO paths yielded an empty alias map and + every non-relative import went unresolved (#2153). Exposed separately so it + can act as a resolution root of last resort, after all declared aliases miss. + Returns None when no config declares baseUrl. + """ + found = _find_js_config(start_dir) + if found is None: + return None + config, candidate = found + key = str(config) + if key not in _TSCONFIG_BASEURL_CACHE: + base_url = None + data = _read_json_config(config) + if data is not None: + raw_base = data.get("compilerOptions", {}).get("baseUrl") + if isinstance(raw_base, str) and raw_base: + base_url = Path(os.path.normpath(candidate / raw_base)) + _TSCONFIG_BASEURL_CACHE[key] = base_url + return _TSCONFIG_BASEURL_CACHE[key] + +def _match_tsconfig_alias(raw: str, pattern: str) -> "tuple[tuple[int, int], str, bool] | None": + """Return (specificity, captured text, is_wildcard) when pattern matches raw. + + Exact aliases win first. Wildcard aliases follow TypeScript's longest-prefix + rule. The final branch preserves Graphify's existing support for treating a + non-wildcard alias as a directory prefix, but only after real wildcard matches. + """ + if "*" in pattern: + if pattern.count("*") != 1: + return None + prefix, suffix = pattern.split("*", 1) + if not raw.startswith(prefix) or not raw.endswith(suffix): + return None + end = len(raw) - len(suffix) if suffix else len(raw) + if end < len(prefix): + return None + return (1, -len(prefix)), raw[len(prefix):end], True + + if raw == pattern: + return (0, -len(pattern)), "", False + + prefix = pattern.rstrip("/") + if prefix and raw.startswith(prefix + "/"): + return (2, -len(prefix)), raw[len(prefix):].lstrip("/"), False + return None + +def _resolve_tsconfig_alias(raw: str, aliases: dict[str, list[str]], + base_url: "Path | None" = None) -> "Path | None": + """Resolve `raw` against the most specific matching tsconfig alias pattern. + + Within that pattern, try targets in declared order and return the first whose + candidate resolves to a real file. If none exist, return the first candidate + so existing phantom/external-edge behavior stays unchanged. + + `base_url` is a resolution root of last resort, tried only when NO declared + alias matches (#2153). It must not participate in the specificity contest: + a bare `*` alias would score (1, 0) and so beat a declared non-wildcard + directory-prefix alias at (2, -len), silently shadowing it and regressing + #1269. Unlike the alias path this returns a candidate only when it is a real + file on disk, so a genuine external package (`import React from 'react'`) + still resolves to nothing instead of a fabricated /react edge. + """ + best: "tuple[tuple[int, int], str, bool, list[str]] | None" = None + for pattern, targets in aliases.items(): + match = _match_tsconfig_alias(raw, pattern) + if match is None: + continue + specificity, captured, is_wildcard = match + if best is None or specificity < best[0]: + best = specificity, captured, is_wildcard, targets + + if best is None: + if base_url is not None: + candidate = Path(os.path.normpath(base_url / raw)) + resolved = _resolve_js_import_path(candidate) + if _path_is_file(resolved): + return resolved + return None + + _, captured, is_wildcard, targets = best + first = None + for target in targets: + if is_wildcard: + # TypeScript substitutes only when the matched star is non-empty. + substituted = target.replace("*", captured, 1) if captured else target + cand = Path(os.path.normpath(substituted)) + else: + cand = Path(target) + if captured: + cand = Path(os.path.normpath(cand / captured)) + resolved = _resolve_js_import_path(cand) + if _path_is_file(resolved): + return resolved + if first is None: + first = cand + return first + +def _find_workspace_root(start_dir: Path) -> Path | None: + current = _resolve_path(start_dir) + for candidate in [current, *current.parents]: + if _path_exists(candidate / "pnpm-workspace.yaml"): + return candidate + package_json = candidate / "package.json" + if _path_is_file(package_json): + try: + data = json.loads(_read_file_text(package_json, encoding="utf-8")) + except Exception: + continue + if "workspaces" in data: + return candidate + return None + +def _pnpm_workspace_globs(workspace_file: Path) -> list[str]: + globs: list[str] = [] + in_packages = False + text = _read_file_text(workspace_file, encoding="utf-8", errors="replace") + for raw_line in text.splitlines(): + line = raw_line.strip() + if not line or line.startswith("#"): + continue + if line.startswith("packages:"): + in_packages = True + continue + if in_packages and line.startswith("-"): + value = line[1:].strip().strip("'\"") + if value and not value.startswith("!"): + globs.append(value) + continue + if in_packages and not raw_line.startswith((" ", "\t")): + break + return globs + +def _workspace_globs(root: Path) -> list[str]: + pnpm_workspace = root / "pnpm-workspace.yaml" + if _path_exists(pnpm_workspace): + return _pnpm_workspace_globs(pnpm_workspace) + + package_json = root / "package.json" + try: + data = json.loads(_read_file_text(package_json, encoding="utf-8")) + except Exception: + return [] + + workspaces = data.get("workspaces") + if isinstance(workspaces, list): + return [item for item in workspaces if isinstance(item, str) and not item.startswith("!")] + if isinstance(workspaces, dict): + packages = workspaces.get("packages") + if isinstance(packages, list): + return [item for item in packages if isinstance(item, str) and not item.startswith("!")] + return [] + +def _load_workspace_packages(start_dir: Path) -> dict[str, Path]: + root = _find_workspace_root(start_dir) + if root is None: + return {} + manifest_mtimes = tuple( + (name, _path_stat(root / name).st_mtime_ns) + for name in _WORKSPACE_MANIFEST_NAMES + if _path_is_file(root / name) + ) + key = str((root, manifest_mtimes)) + if key in _WORKSPACE_PACKAGE_CACHE: + return _WORKSPACE_PACKAGE_CACHE[key] + + packages: dict[str, Path] = {} + for pattern in _workspace_globs(root): + package_dirs: list[Path] = ( + [root] if pattern in (".", "./") else list(_glob_paths(root, pattern)) + ) + for package_dir in package_dirs: + manifest = package_dir / "package.json" + if not _path_is_file(manifest): + continue + try: + data = json.loads(_read_file_text(manifest, encoding="utf-8")) + except Exception: + continue + name = data.get("name") + if isinstance(name, str) and name: + packages[name] = package_dir + _WORKSPACE_PACKAGE_CACHE[key] = packages + return packages + +_EXPORT_CONDITION_PRIORITY = ( + "source", "import", "module", "svelte", "types", "require", "default", +) + +def _resolve_export_target(value: Any) -> str | None: + """Resolve an `exports` map value (string or condition object) to a + relative target string, honouring _EXPORT_CONDITION_PRIORITY for objects + and recursing into nested condition objects.""" + if isinstance(value, str): + return value + if isinstance(value, dict): + for cond in _EXPORT_CONDITION_PRIORITY: + v = value.get(cond) + if isinstance(v, str): + return v + if isinstance(v, dict): + nested = _resolve_export_target(v) + if nested: + return nested + return None + +def _contained_in_package(resolved: Path, package_dir: Path) -> bool: + """Guard against `exports` targets that escape the package directory + (e.g. "./evil": "../../../etc/passwd"). Only accept paths that stay + within package_dir after resolution.""" + try: + return _resolve_path(resolved).is_relative_to(_resolve_path(package_dir)) + except ValueError: + return False + +def _package_entry_candidates(package_dir: Path, subpath: str) -> list[Path]: + manifest = package_dir / "package.json" + manifest_data: dict[str, Any] = {} + try: + manifest_data = json.loads(_read_file_text(manifest, encoding="utf-8")) + except Exception: + pass + + if subpath: + # Consult the package's `exports` subpath map before the bare-path + # fallback (#1308): "./browser" -> conditions -> file, plus single + # wildcard "./*" patterns. Targets that escape the package dir are + # rejected; resolution then falls through to the bare path. + exports = manifest_data.get("exports") + if isinstance(exports, dict): + subpath_key = "./" + subpath + target = _resolve_export_target(exports.get(subpath_key)) + if target: + candidate = package_dir / target + if _contained_in_package(candidate, package_dir): + return [candidate] + else: + for pattern, pattern_value in exports.items(): + if "*" in pattern and pattern.count("*") == 1: + prefix, suffix = pattern.split("*", 1) + if (subpath_key.startswith(prefix) + and (not suffix or subpath_key.endswith(suffix))): + matched = subpath_key[len(prefix):len(subpath_key) - len(suffix) if suffix else None] + resolved = _resolve_export_target(pattern_value) + if resolved and "*" in resolved: + candidate = package_dir / resolved.replace("*", matched) + if _contained_in_package(candidate, package_dir): + return [candidate] + return [package_dir / subpath] + + exports = manifest_data.get("exports") + if isinstance(exports, str): + return [package_dir / exports] + if isinstance(exports, dict): + dot_target = _resolve_export_target(exports.get(".")) + if dot_target: + return [package_dir / dot_target] + + candidates: list[Path] = [] + for key in ("svelte", "module", "main", "types"): + value = manifest_data.get(key) + if isinstance(value, str): + candidates.append(package_dir / value) + candidates.append(package_dir / "src/index") + candidates.append(package_dir / "index") + return candidates + +def _resolve_workspace_import(raw: str, start_dir: Path) -> Path | None: + packages = _load_workspace_packages(start_dir) + for package_name, package_dir in packages.items(): + if raw == package_name: + subpath = "" + elif raw.startswith(package_name + "/"): + subpath = raw[len(package_name) + 1:] + else: + continue + for candidate in _package_entry_candidates(package_dir, subpath): + resolved = _resolve_js_import_path(candidate) + if _path_is_file(resolved): + return resolved + return None + +def _resolve_js_module_path(raw: str | Path, start_dir: Path | None = None) -> Path | None: + """Resolve a JS/TS module path or specifier to a local source file. + + With a Path argument this preserves the path-based helper API used by + import-extension tests. With a string plus start_dir it resolves JS/TS + module specifiers including relative paths, tsconfig aliases, and workspace + packages. + """ + if isinstance(raw, Path): + return _resolve_js_import_path(raw) + if start_dir is None: + return _resolve_js_import_path(Path(raw)) + if raw.startswith("."): + return _resolve_js_import_path(start_dir / raw) + + aliases = _load_tsconfig_aliases(start_dir) + hit = _resolve_tsconfig_alias(raw, aliases, + base_url=_load_tsconfig_base_url(start_dir)) + if hit is not None: + return _resolve_js_import_path(hit) + + return _resolve_workspace_import(raw, start_dir) + +def _resolve_js_import_target(raw: str, str_path: str) -> "tuple[str, Path | None] | None": + """Resolve a JS/TS import path string to (target_nid, resolved_path). + + Handles relative paths, tsconfig path aliases, workspace packages, and + bare/scoped imports. + Returns None if `raw` is empty. + """ + if not raw: + return None + resolved_path = _resolve_js_module_path(raw, Path(str_path).parent) + if resolved_path is not None: + return _make_id(str(resolved_path)), resolved_path + module_name = raw.split("/")[-1] + if not module_name: + return None + # Unresolved: relative/absolute, tsconfig-alias and workspace resolution have + # all run and failed, so this is an external package (or a dangling local + # path). Namespace the id with the "ref" prefix — the J-4 convention already + # used for tsconfig `extends`/`$ref` externals — so it can NEVER collapse to + # the same _make_id as a local file/symbol node. Without it, the bare + # last-segment id (e.g. "tailwindcss/colors" -> "colors") collides with any + # unrelated local file of that stem via build.py's pre-migration alias index, + # producing a confident (EXTRACTED) cross-language phantom imports_from edge + # (#1638). The ref-namespaced target has no node, so build drops it as an + # external reference — the correct outcome for a third-party import. + return _make_id("ref", raw), None + +def _resolve_c_include_path(raw: str, str_path: str) -> "Path | None": + """Resolve a quoted #include path to a real file on disk. + + Searches relative to the including file's directory. Returns None for + system headers (<...>) or paths that don't exist on disk. + """ + if not raw: + return None + candidate = _resolve_path(Path(str_path).parent / raw) + if _path_is_file(candidate): + return candidate + return None + +def _resolve_lua_import_target(raw_module: str, str_path: str) -> str: + """Resolve a Lua require() module name to a node id. + + Lua module names use dots as path separators: `require("pkg.b")` looks for + `pkg/b.lua` (or `pkg/b/init.lua`) relative to a package root. We probe the + importing file's directory and walk upward looking for a matching file on + disk; if found, the returned id matches the file node id `_extract_generic` + assigns to that file (`_make_id(str(path))`), so the edge lands on a real + node. When nothing matches, fall back to `_make_id` of the full dotted + module name so cross-file resolution can still complete via the symbol + resolution pass instead of dropping the edge entirely (#1075). + """ + if not raw_module: + return "" + rel = raw_module.replace(".", "/") + try: + start_dir = Path(str_path).parent + except Exception: + start_dir = None + if start_dir is not None: + probe = start_dir + # Walk up a few levels so requires from nested files still resolve when + # the package root is above the importing file. + for _ in range(6): + for suffix in (".lua", ".luau"): + cand = probe / f"{rel}{suffix}" + if _path_is_file(cand): + return _make_id(str(cand)) + for suffix in (".lua", ".luau"): + cand = probe / rel / f"init{suffix}" + if _path_is_file(cand): + return _make_id(str(cand)) + if probe.parent == probe: + break + probe = probe.parent + return _make_id(raw_module) + +_VUE_SCRIPT_RE = re.compile( + r"""("'])*>)([\s\S]*?)()""", + re.IGNORECASE, +) + +_VUE_SCRIPT_LANG_RE = re.compile( + r"""\blang\s*=\s*['"]?([A-Za-z]+)['"]?""", re.IGNORECASE +) + +def _vue_mask_non_script(src: str) -> tuple[str, str | None]: + """Blank everything outside `` close tag + pos = m.end() + if lang is None: + lang_m = _VUE_SCRIPT_LANG_RE.search(m.group(1)) + if lang_m: + lang = lang_m.group(1).lower() + out.append(_blank(src[pos:])) + return "".join(out), lang + +def _source_key(source_file: str, root: Path) -> str: + if not source_file: + return "" + source_path = Path(source_file) + try: + return str(_resolve_path(source_path).relative_to(_resolve_path(root))) + except Exception: + return str(source_path) + +def _node_disambiguation_source_key(node: dict, root: Path) -> str: + source_file = str(node.get("source_file", "")) + if source_file: + return _source_key(source_file, root) + return _source_key(str(node.get("origin_file", "")), root) + +def _disambiguate_colliding_node_ids( + nodes: list[dict], + edges: list[dict], + raw_calls: list[dict], + root: Path, +) -> None: + """Rewrite only colliding node IDs, using source path as the disambiguator. + + Module anchor nodes (#1327) are exempt: ``import CoreKit`` from three files + yields three ``type=module`` nodes with the same id but different + source_files. Those are the *same* module, not distinct same-named symbols, + so they must collapse to one shared node — disambiguating them by path would + scatter a single module across N file-qualified duplicates. + """ + by_id: dict[str, list[dict]] = {} + for node in nodes: + if node.get("type") in ("module", "namespace"): + continue + nid = node.get("id") + if isinstance(nid, str) and nid: + by_id.setdefault(nid, []).append(node) + + remap: dict[tuple[str, str], str] = {} + ambiguous_ids: set[str] = set() + for old_id, group in by_id.items(): + source_keys = {_node_disambiguation_source_key(node, root) for node in group} + if len(group) < 2 or len(source_keys) < 2: + continue + ambiguous_ids.add(old_id) + # Salt the colliding id with the *path* it came from. The naive salt is + # ``_make_id(source_key, old_id)`` — source_key is the raw repo-relative + # path. But _make_id collapses every separator, so two DISTINCT paths + # whose only difference is a separator-vs-inner-punctuation swap + # (``a/b/c.md`` vs ``a.b/c.md``, ``foo/bar_baz.md`` vs ``foo_bar/baz.md``) + # normalize to the SAME salted id and still collide (#1522 — the residual + # of #1504 the 0.9.0 full-path stem didn't reach). When that happens, + # append a short stable hash of the *raw* source_key, which IS injective + # over distinct paths, so the colliders separate. Computed in code from + # source_file (never trusted from the LLM), so AST↔semantic parity holds. + naive: dict[str, str] = {} # source_key -> _make_id(source_key, old_id) + for source_key in source_keys: + if source_key: + naive[source_key] = _make_id(source_key, old_id) + # source_keys that, after normalization, are not unique among themselves. + seen: dict[str, int] = {} + for nid in naive.values(): + seen[nid] = seen.get(nid, 0) + 1 + needs_hash = {sk for sk, nid in naive.items() if seen.get(nid, 0) > 1} + for node in group: + source_key = _node_disambiguation_source_key(node, root) + if not source_key: + continue + if source_key in needs_hash: + salt = hashlib.sha1(source_key.encode("utf-8")).hexdigest()[:6] + new_id = _make_id(source_key, old_id, salt) + else: + new_id = naive.get(source_key) or _make_id(source_key, old_id) + remap[(old_id, source_key)] = new_id + if new_id != old_id: + node["id"] = new_id + + if not remap: + # No colliding ids to salt apart, but the transient `target_file` hint an + # importer stamps on every resolved import (#1814) still has to be dropped + # here — this early exit skips the edge loop below, so without it a + # non-colliding import would carry its absolute path into graph.json. + for edge in edges: + edge.pop("target_file", None) + return + + unambiguous_remaps: dict[str, str] = {} + for old_id, group in by_id.items(): + if old_id in ambiguous_ids: + continue + candidates = { + node["id"] for node in group + if isinstance(node.get("id"), str) and node["id"] != old_id + } + if len(candidates) == 1: + unambiguous_remaps[old_id] = next(iter(candidates)) + + # A C/ObjC/C++ `#include "foo.h"` / `#import "foo.h"` resolves to the header's + # file node, but `foo.h` and its sibling `foo.c`/`foo.m`/`foo.cpp` collapse to + # the same `foo` file id, so disambiguation salts them apart by path. A + # cross-file import edge from a THIRD file carries neither salt's source_key, so + # the (target, edge_source_key) lookup misses and the edge dangles on the now + # dead `foo` id. Repoint those import edges to the HEADER variant (the include + # always targeted the header), keyed by the original colliding id (#1475). + _HEADER_SUFFIXES = (".h", ".hpp", ".hh", ".hxx") + header_remaps: dict[str, str] = {} + for old_id in ambiguous_ids: + for node in by_id.get(old_id, []): + sk = _node_disambiguation_source_key(node, root) + if sk and Path(sk).suffix.lower() in _HEADER_SUFFIXES: + new_id = remap.get((old_id, sk)) + if new_id: + header_remaps[old_id] = new_id + break + + for edge in edges: + edge_source_key = _source_key(str(edge.get("source_file", "")), root) + source_key = (edge.get("source", ""), edge_source_key) + # An import/re-export edge's target is a FILE node that can collapse with a + # same-basename cross-extension sibling (foo.ts vs foo.mjs, #1814). Keying + # its target salt by the IMPORTER's own source_file mis-points it back at the + # importer's variant (a self-loop). When the emitter stamped the resolved + # target file, key the target salt by THAT file so the salt lands on the + # correct sibling. Generalizes the #1475 C/ObjC header carve-out (below) to + # every language and to re_exports. `pop` it as we consume it: this is the + # hint's only reader, and its absolute path must not persist into graph.json. + target_file = edge.pop("target_file", None) + if target_file and edge.get("relation") in ("imports", "imports_from", "re_exports"): + target_edge_key = _source_key(str(target_file), root) + else: + target_edge_key = edge_source_key + target_key = (edge.get("target", ""), target_edge_key) + if source_key in remap: + edge["source"] = remap[source_key] + elif edge.get("source") in unambiguous_remaps: + edge["source"] = unambiguous_remaps[str(edge["source"])] + # imports/imports_from always target a header file, so they must resolve to + # the header variant BEFORE the same-source-file salt is considered. Keying + # the import target by the importer's own source file mis-points a `.m` + # importing its own `.h` back at itself (self-loop), and is wrong for any + # cross-file import whose importer shares the colliding id (#1475). + if (edge.get("relation") in ("imports", "imports_from") + and edge.get("target") in header_remaps): + edge["target"] = header_remaps[str(edge["target"])] + elif target_key in remap: + edge["target"] = remap[target_key] + elif edge.get("target") in unambiguous_remaps: + edge["target"] = unambiguous_remaps[str(edge["target"])] + + for raw_call in raw_calls: + call_source_key = _source_key(str(raw_call.get("source_file", "")), root) + caller_key = (raw_call.get("caller_nid", ""), call_source_key) + if caller_key in remap: + raw_call["caller_nid"] = remap[caller_key] + elif raw_call.get("caller_nid") in unambiguous_remaps: + raw_call["caller_nid"] = unambiguous_remaps[str(raw_call["caller_nid"])] + +def _is_type_like_definition(node: dict) -> bool: + if node.get("type") == "namespace": + return False + label = str(node.get("label", "")).strip() + if not label: + return False + if label.endswith(")") or label.startswith("."): + return False + if "." in label: + return False + return node.get("file_type") == "code" + +def _js_source_path(source_file: str, root: Path) -> Path | None: + if not source_file: + return None + path = Path(source_file) + if not path.is_absolute(): + path = root / path + try: + return _resolve_path(path) + except Exception: + return path + +def _apply_symbol_resolution_facts( + paths: list[Path], + nodes: list[dict], + edges: list[dict], + root: Path, + facts: _SymbolResolutionFacts, +) -> None: + """Apply language-provided import/export/use facts to graph edges.""" + if not ( + facts.declarations + or facts.imports + or facts.aliases + or facts.exports + or facts.star_exports + or facts.namespace_exports + or facts.uses + or facts.module_imports + ): + return + + path_by_resolved = {_resolve_path(path): path for path in paths} + source_file_id = {_resolve_path(path): _make_id(str(path)) for path in paths} + symbol_nodes: dict[tuple[Path, str], str] = {} + for node in nodes: + source_path = _js_source_path(str(node.get("source_file", "")), root) + if source_path is None: + continue + label = str(node.get("label", "")).strip().strip("()").lstrip(".") + if label and node.get("id"): + symbol_nodes[(source_path, label)] = str(node["id"]) + + def ensure_symbol_node(path: Path, name: str, line: int) -> str: + resolved_path = _resolve_path(path) + existing = symbol_nodes.get((resolved_path, name)) + if existing is not None: + return existing + node_id = _make_id(_file_stem(path), name) + symbol_nodes[(resolved_path, name)] = node_id + nodes.append({ + "id": node_id, + "label": name, + "file_type": "code", + "source_file": str(path), + "source_location": f"L{line}", + }) + return node_id + + existing_edges = { + ( + str(edge.get("source")), + str(edge.get("target")), + str(edge.get("relation")), + str(edge.get("context") or ""), + ) + for edge in edges + } + + def add_edge(source: str, target: str, relation: str, context: str, line: int, source_path: Path, target_file: str | None = None, local_alias: str | None = None) -> None: + key = (source, target, relation, context or "") + if key in existing_edges: + return + existing_edges.add(key) + edge = { + "source": source, + "target": target, + "relation": relation, + "context": context, + "confidence": "EXTRACTED", + "source_file": str(source_path), + "source_location": f"L{line}", + "weight": 1.0, + } + # A re-export edge's target is a FILE node that can collapse with a + # same-basename cross-extension sibling; stamp the resolved target file so + # the id-disambiguation salt is keyed by the TARGET, not the importer (#1814). + if target_file is not None: + edge["target_file"] = target_file + # The local name this import bound in the importing file, when it differs + # from the target's own name (`from pkg import mod as alias`) -- lets the + # cross-file member-call resolver match `alias.func()` (#2082). + if local_alias is not None: + edge["local_alias"] = local_alias + edges.append(edge) + + for declaration in facts.declarations: + ensure_symbol_node(declaration.file_path, declaration.name, declaration.line) + + local_aliases_by_file: dict[Path, dict[str, tuple[Path, str]]] = {} + for import_fact in facts.imports: + file_path = _resolve_path(import_fact.file_path) + local_aliases_by_file.setdefault(file_path, {})[import_fact.local_name] = ( + _resolve_path(import_fact.target_path), + import_fact.imported_name, + ) + + pending_aliases_by_file: dict[Path, list[_SymbolAliasFact]] = {} + for alias_fact in facts.aliases: + resolved_file = _resolve_path(alias_fact.file_path) + pending_aliases_by_file.setdefault(resolved_file, []).append(alias_fact) + + for file_path, aliases in pending_aliases_by_file.items(): + local_aliases = local_aliases_by_file.setdefault(file_path, {}) + changed = True + while changed: + changed = False + for alias_fact in aliases: + if alias_fact.alias in local_aliases: + continue + origin = local_aliases.get(alias_fact.target_name) + if origin is not None: + local_aliases[alias_fact.alias] = origin + changed = True + + named_exports_by_file: dict[Path, dict[str, tuple[Path, str]]] = {} + star_exports_by_file: dict[Path, list[Path]] = {} + + for star_fact in facts.star_exports: + source_path = _resolve_path(star_fact.file_path) + target_path = _resolve_path(star_fact.target_path) + star_exports_by_file.setdefault(source_path, []).append(target_path) + source_id = source_file_id.get(source_path) + if source_id is not None: + add_edge( + source_id, + _make_id(str(path_by_resolved.get(target_path, target_path))), + "re_exports", + "export", + star_fact.line, + star_fact.file_path, + target_file=str(path_by_resolved.get(target_path, target_path)), + ) + + for namespace_fact in facts.namespace_exports: + source_path = _resolve_path(namespace_fact.file_path) + target_path = _resolve_path(namespace_fact.target_path) + namespace_id = ensure_symbol_node( + namespace_fact.file_path, + namespace_fact.exported_name, + namespace_fact.line, + ) + named_exports_by_file.setdefault(source_path, {})[ + namespace_fact.exported_name + ] = (source_path, namespace_fact.exported_name) + source_id = source_file_id.get(source_path) + if source_id is not None: + add_edge( + source_id, + namespace_id, + "contains", + "namespace_export", + namespace_fact.line, + namespace_fact.file_path, + ) + add_edge( + source_id, + _make_id(str(path_by_resolved.get(target_path, target_path))), + "re_exports", + "export", + namespace_fact.line, + namespace_fact.file_path, + target_file=str(path_by_resolved.get(target_path, target_path)), + ) + + for export_fact in facts.exports: + file_path = _resolve_path(export_fact.file_path) + origin: tuple[Path, str] | None = None + if export_fact.target_path is not None and export_fact.target_name is not None: + origin = (_resolve_path(export_fact.target_path), export_fact.target_name) + elif export_fact.local_name is not None: + origin = local_aliases_by_file.get(file_path, {}).get(export_fact.local_name) + if origin is None and (file_path, export_fact.local_name) in symbol_nodes: + origin = (file_path, export_fact.local_name) + if origin is None: + continue + named_exports_by_file.setdefault(file_path, {})[export_fact.exported_name] = origin + if origin[0] != file_path: + source_id = source_file_id.get(file_path) + if source_id is not None: + add_edge( + source_id, + _make_id(str(path_by_resolved.get(origin[0], origin[0]))), + "re_exports", + "export", + export_fact.line, + export_fact.file_path, + target_file=str(path_by_resolved.get(origin[0], origin[0])), + ) + + def resolve_exported_origin(target_path: Path, imported_name: str, seen: set[tuple[Path, str]] | None = None) -> tuple[Path, str]: + target_path = _resolve_path(target_path) + key = (target_path, imported_name) + if seen is None: + seen = set() + if key in seen: + return key + seen.add(key) + origin = named_exports_by_file.get(target_path, {}).get(imported_name) + if origin is not None: + return resolve_exported_origin(origin[0], origin[1], seen) + for star_target in star_exports_by_file.get(target_path, []): + star_key = (star_target, imported_name) + if star_key in symbol_nodes: + return star_key + resolved = resolve_exported_origin(star_target, imported_name, seen) + if resolved in symbol_nodes: + return resolved + return key + + for import_fact in facts.imports: + source_id = source_file_id.get(_resolve_path(import_fact.file_path)) + if source_id is None: + continue + origin_path, origin_symbol = resolve_exported_origin( + import_fact.target_path, + import_fact.imported_name, + ) + target_id = symbol_nodes.get((origin_path, origin_symbol)) + if target_id is None: + continue + add_edge( + source_id, + target_id, + "imports", + "import", + import_fact.line, + import_fact.file_path, + ) + + # #1146: emit file-to-file imports_from edges for package-form submodule imports. + for from_path, to_path, line, local_name in facts.module_imports: + try: + from_rel = from_path.relative_to(root) + to_rel = to_path.relative_to(root) + except ValueError: + continue + source_id = _make_id(_file_stem(from_rel)) + target_id = _make_id(_file_stem(to_rel)) + add_edge( + source_id, target_id, "imports_from", "submodule_import", line, from_path, + local_alias=local_name if local_name != to_path.stem else None, + ) + + # #2262 producer guard: never emit a `calls` use-edge from a source id + # that owns no node. All node appends (ensure_symbol_node, declarations, + # namespace exports) happened above, so the owned set is complete here. + # A node-less caller id can never be canonicalized by the extract() + # remaps (they learn only from nodes), so an absolute-derived one would + # leak the machine/scan-path slug into the edge source. Reattribute the + # edge to the caller's FILE node — the true file-level dependency + # survives, and the file id is exactly what the #2231 remap + # canonicalizes — or drop it when no file node id is available. + owned = {str(n.get("id")) for n in nodes} + for use_fact in facts.uses: + file_path = _resolve_path(use_fact.file_path) + target_id = None + unresolved_origin = local_aliases_by_file.get(file_path, {}).get(use_fact.local_name) + if unresolved_origin is not None: + origin_path, origin_symbol = resolve_exported_origin(*unresolved_origin) + target_id = symbol_nodes.get((origin_path, origin_symbol)) + if target_id is None and use_fact.relation in ("inherits", "implements"): + # Same-file fallback for HERITAGE only: a base declared in the same + # file (`class X extends Y`, `interface A extends B`) has no import + # alias, so resolve it directly against the file's own symbol nodes. + # Scoped to heritage because same-file calls/uses already resolve via + # the dedicated call-graph pass; widening this would duplicate those + # edges. Import resolution still takes precedence (#1095). + target_id = symbol_nodes.get((file_path, use_fact.local_name)) + if target_id is None: + continue + source_id = use_fact.source_id + if use_fact.relation == "calls" and source_id not in owned: + source_id = source_file_id.get(file_path) + if source_id is None: + continue + add_edge( + source_id, + target_id, + use_fact.relation, + use_fact.context, + use_fact.line, + use_fact.file_path, + ) + +def _parse_js_tree(path: Path): + try: + from tree_sitter import Language, Parser + # .vue embeds the script in non-JS markup; mask it out and parse the + # close tag + pos = m.end() + if lang is None: + lang_m = _VUE_SCRIPT_LANG_RE.search(m.group(1)) + if lang_m: + lang = lang_m.group(1).lower() + out.append(_blank(src[pos:])) + return "".join(out), lang + +def _source_key(source_file: str, root: Path) -> str: + if not source_file: + return "" + source_path = Path(source_file) + try: + return str(_resolve_path(source_path).relative_to(_resolve_path(root))) + except Exception: + return str(source_path) -# ── Language configs ────────────────────────────────────────────────────────── +def _node_disambiguation_source_key(node: dict, root: Path) -> str: + source_file = str(node.get("source_file", "")) + if source_file: + return _source_key(source_file, root) + return _source_key(str(node.get("origin_file", "")), root) -_PYTHON_CONFIG = LanguageConfig( - ts_module="tree_sitter_python", - class_types=frozenset({"class_definition"}), - function_types=frozenset({"function_definition"}), - import_types=frozenset({"import_statement", "import_from_statement"}), - call_types=frozenset({"call"}), - call_function_field="function", - call_accessor_node_types=frozenset({"attribute"}), - call_accessor_field="attribute", - call_accessor_object_field="object", - function_boundary_types=frozenset({"function_definition"}), - import_handler=_import_python, -) +def _disambiguate_colliding_node_ids( + nodes: list[dict], + edges: list[dict], + raw_calls: list[dict], + root: Path, +) -> None: + """Rewrite only colliding node IDs, using source path as the disambiguator. -_JS_CONFIG = LanguageConfig( - ts_module="tree_sitter_javascript", - class_types=frozenset({"class_declaration"}), - function_types=frozenset({"function_declaration", "generator_function_declaration", "method_definition"}), - import_types=frozenset({"import_statement", "export_statement"}), - call_types=frozenset({"call_expression", "new_expression"}), - call_function_field="function", - call_accessor_node_types=frozenset({"member_expression"}), - call_accessor_field="property", - call_accessor_object_field="object", - function_boundary_types=frozenset({"function_declaration", "generator_function_declaration", "arrow_function", "method_definition"}), - import_handler=_import_js, -) + Module anchor nodes (#1327) are exempt: ``import CoreKit`` from three files + yields three ``type=module`` nodes with the same id but different + source_files. Those are the *same* module, not distinct same-named symbols, + so they must collapse to one shared node — disambiguating them by path would + scatter a single module across N file-qualified duplicates. + """ + by_id: dict[str, list[dict]] = {} + for node in nodes: + if node.get("type") in ("module", "namespace"): + continue + nid = node.get("id") + if isinstance(nid, str) and nid: + by_id.setdefault(nid, []).append(node) + + remap: dict[tuple[str, str], str] = {} + ambiguous_ids: set[str] = set() + for old_id, group in by_id.items(): + source_keys = {_node_disambiguation_source_key(node, root) for node in group} + if len(group) < 2 or len(source_keys) < 2: + continue + ambiguous_ids.add(old_id) + # Salt the colliding id with the *path* it came from. The naive salt is + # ``_make_id(source_key, old_id)`` — source_key is the raw repo-relative + # path. But _make_id collapses every separator, so two DISTINCT paths + # whose only difference is a separator-vs-inner-punctuation swap + # (``a/b/c.md`` vs ``a.b/c.md``, ``foo/bar_baz.md`` vs ``foo_bar/baz.md``) + # normalize to the SAME salted id and still collide (#1522 — the residual + # of #1504 the 0.9.0 full-path stem didn't reach). When that happens, + # append a short stable hash of the *raw* source_key, which IS injective + # over distinct paths, so the colliders separate. Computed in code from + # source_file (never trusted from the LLM), so AST↔semantic parity holds. + naive: dict[str, str] = {} # source_key -> _make_id(source_key, old_id) + for source_key in source_keys: + if source_key: + naive[source_key] = _make_id(source_key, old_id) + # source_keys that, after normalization, are not unique among themselves. + seen: dict[str, int] = {} + for nid in naive.values(): + seen[nid] = seen.get(nid, 0) + 1 + needs_hash = {sk for sk, nid in naive.items() if seen.get(nid, 0) > 1} + for node in group: + source_key = _node_disambiguation_source_key(node, root) + if not source_key: + continue + if source_key in needs_hash: + salt = hashlib.sha1(source_key.encode("utf-8")).hexdigest()[:6] + new_id = _make_id(source_key, old_id, salt) + else: + new_id = naive.get(source_key) or _make_id(source_key, old_id) + remap[(old_id, source_key)] = new_id + if new_id != old_id: + node["id"] = new_id -_TS_CONFIG = LanguageConfig( - ts_module="tree_sitter_typescript", - ts_language_fn="language_typescript", - class_types=frozenset({ - "class_declaration", - "abstract_class_declaration", # TS abstract class - "interface_declaration", # parity with Java/C# - "enum_declaration", # named enums - "type_alias_declaration", # named type aliases - }), - function_types=frozenset({"function_declaration", "generator_function_declaration", "method_definition", "method_signature"}), - import_types=frozenset({"import_statement", "export_statement"}), - call_types=frozenset({"call_expression", "new_expression"}), - call_function_field="function", - call_accessor_node_types=frozenset({"member_expression"}), - call_accessor_field="property", - call_accessor_object_field="object", - function_boundary_types=frozenset({"function_declaration", "generator_function_declaration", "arrow_function", "method_definition"}), - import_handler=_import_js, -) + if not remap: + # No colliding ids to salt apart, but the transient `target_file` hint an + # importer stamps on every resolved import (#1814) still has to be dropped + # here — this early exit skips the edge loop below, so without it a + # non-colliding import would carry its absolute path into graph.json. + for edge in edges: + edge.pop("target_file", None) + return -# .tsx files must use the TSX grammar (JSX-aware), not the plain TypeScript grammar. -# tree-sitter-typescript ships two languages: language_typescript (for .ts) and -# language_tsx (for .tsx). Parsing .tsx with language_typescript silently fails on -# JSX expressions, dropping any call_expression nested inside JSX (e.g. {fmtDate(x)}). -_TSX_CONFIG = LanguageConfig( - ts_module="tree_sitter_typescript", - ts_language_fn="language_tsx", - class_types=_TS_CONFIG.class_types, - function_types=_TS_CONFIG.function_types, - import_types=_TS_CONFIG.import_types, - call_types=_TS_CONFIG.call_types, - call_function_field=_TS_CONFIG.call_function_field, - call_accessor_node_types=_TS_CONFIG.call_accessor_node_types, - call_accessor_field=_TS_CONFIG.call_accessor_field, - call_accessor_object_field=_TS_CONFIG.call_accessor_object_field, - function_boundary_types=_TS_CONFIG.function_boundary_types, - import_handler=_TS_CONFIG.import_handler, -) + unambiguous_remaps: dict[str, str] = {} + for old_id, group in by_id.items(): + if old_id in ambiguous_ids: + continue + candidates = { + node["id"] for node in group + if isinstance(node.get("id"), str) and node["id"] != old_id + } + if len(candidates) == 1: + unambiguous_remaps[old_id] = next(iter(candidates)) + + # A C/ObjC/C++ `#include "foo.h"` / `#import "foo.h"` resolves to the header's + # file node, but `foo.h` and its sibling `foo.c`/`foo.m`/`foo.cpp` collapse to + # the same `foo` file id, so disambiguation salts them apart by path. A + # cross-file import edge from a THIRD file carries neither salt's source_key, so + # the (target, edge_source_key) lookup misses and the edge dangles on the now + # dead `foo` id. Repoint those import edges to the HEADER variant (the include + # always targeted the header), keyed by the original colliding id (#1475). + _HEADER_SUFFIXES = (".h", ".hpp", ".hh", ".hxx") + header_remaps: dict[str, str] = {} + for old_id in ambiguous_ids: + for node in by_id.get(old_id, []): + sk = _node_disambiguation_source_key(node, root) + if sk and Path(sk).suffix.lower() in _HEADER_SUFFIXES: + new_id = remap.get((old_id, sk)) + if new_id: + header_remaps[old_id] = new_id + break -_JAVA_CONFIG = LanguageConfig( - ts_module="tree_sitter_java", - # record_declaration shares class_declaration's name/body/interfaces fields, - # so it becomes a first-class type node instead of an isolated file (#1373). - # Enums and annotation declarations use the same name/body contract. - class_types=frozenset({ - "class_declaration", "interface_declaration", "record_declaration", - "enum_declaration", "annotation_type_declaration", - }), - function_types=frozenset({"method_declaration", "constructor_declaration"}), - import_types=frozenset({"import_declaration"}), - # object_creation_expression (`new Foo(...)`) is handled by a dedicated Java - # branch in walk_calls below — its callee is in the `type` field, not `name`. - call_types=frozenset({"method_invocation", "object_creation_expression"}), - call_function_field="name", - call_accessor_node_types=frozenset(), - function_boundary_types=frozenset({"method_declaration", "constructor_declaration"}), - import_handler=_import_java, -) + for edge in edges: + edge_source_key = _source_key(str(edge.get("source_file", "")), root) + source_key = (edge.get("source", ""), edge_source_key) + # An import/re-export edge's target is a FILE node that can collapse with a + # same-basename cross-extension sibling (foo.ts vs foo.mjs, #1814). Keying + # its target salt by the IMPORTER's own source_file mis-points it back at the + # importer's variant (a self-loop). When the emitter stamped the resolved + # target file, key the target salt by THAT file so the salt lands on the + # correct sibling. Generalizes the #1475 C/ObjC header carve-out (below) to + # every language and to re_exports. `pop` it as we consume it: this is the + # hint's only reader, and its absolute path must not persist into graph.json. + target_file = edge.pop("target_file", None) + if target_file and edge.get("relation") in ("imports", "imports_from", "re_exports"): + target_edge_key = _source_key(str(target_file), root) + else: + target_edge_key = edge_source_key + target_key = (edge.get("target", ""), target_edge_key) + if source_key in remap: + edge["source"] = remap[source_key] + elif edge.get("source") in unambiguous_remaps: + edge["source"] = unambiguous_remaps[str(edge["source"])] + # imports/imports_from always target a header file, so they must resolve to + # the header variant BEFORE the same-source-file salt is considered. Keying + # the import target by the importer's own source file mis-points a `.m` + # importing its own `.h` back at itself (self-loop), and is wrong for any + # cross-file import whose importer shares the colliding id (#1475). + if (edge.get("relation") in ("imports", "imports_from") + and edge.get("target") in header_remaps): + edge["target"] = header_remaps[str(edge["target"])] + elif target_key in remap: + edge["target"] = remap[target_key] + elif edge.get("target") in unambiguous_remaps: + edge["target"] = unambiguous_remaps[str(edge["target"])] + + for raw_call in raw_calls: + call_source_key = _source_key(str(raw_call.get("source_file", "")), root) + caller_key = (raw_call.get("caller_nid", ""), call_source_key) + if caller_key in remap: + raw_call["caller_nid"] = remap[caller_key] + elif raw_call.get("caller_nid") in unambiguous_remaps: + raw_call["caller_nid"] = unambiguous_remaps[str(raw_call["caller_nid"])] + +def _is_type_like_definition(node: dict) -> bool: + if node.get("type") == "namespace": + return False + label = str(node.get("label", "")).strip() + if not label: + return False + if label.endswith(")") or label.startswith("."): + return False + if "." in label: + return False + return node.get("file_type") == "code" -_GROOVY_CONFIG = LanguageConfig( - ts_module="tree_sitter_groovy", - class_types=frozenset({"class_declaration", "interface_declaration"}), - function_types=frozenset({"method_declaration", "constructor_declaration"}), - import_types=frozenset({"import_declaration"}), - call_types=frozenset({"method_invocation"}), - call_function_field="name", - call_accessor_node_types=frozenset(), - function_boundary_types=frozenset({"method_declaration", "constructor_declaration"}), - import_handler=_import_java, -) +def _js_source_path(source_file: str, root: Path) -> Path | None: + if not source_file: + return None + path = Path(source_file) + if not path.is_absolute(): + path = root / path + try: + return _resolve_path(path) + except Exception: + return path -_C_CONFIG = LanguageConfig( - ts_module="tree_sitter_c", - class_types=frozenset(), - function_types=frozenset({"function_definition"}), - import_types=frozenset({"preproc_include"}), - call_types=frozenset({"call_expression"}), - call_function_field="function", - call_accessor_node_types=frozenset({"field_expression"}), - call_accessor_field="field", - function_boundary_types=frozenset({"function_definition"}), - import_handler=_import_c, - resolve_function_name_fn=_get_c_func_name, -) +def _apply_symbol_resolution_facts( + paths: list[Path], + nodes: list[dict], + edges: list[dict], + root: Path, + facts: _SymbolResolutionFacts, +) -> None: + """Apply language-provided import/export/use facts to graph edges.""" + if not ( + facts.declarations + or facts.imports + or facts.aliases + or facts.exports + or facts.star_exports + or facts.namespace_exports + or facts.uses + or facts.module_imports + ): + return -_CPP_CONFIG = LanguageConfig( - ts_module="tree_sitter_cpp", - class_types=frozenset({"class_specifier", "struct_specifier"}), - function_types=frozenset({"function_definition"}), - import_types=frozenset({"preproc_include"}), - call_types=frozenset({"call_expression"}), - call_function_field="function", - call_accessor_node_types=frozenset({"field_expression", "qualified_identifier"}), - call_accessor_field="field", - function_boundary_types=frozenset({"function_definition"}), - import_handler=_import_c, - resolve_function_name_fn=_get_cpp_func_name, -) + path_by_resolved = {_resolve_path(path): path for path in paths} + source_file_id = {_resolve_path(path): _make_id(str(path)) for path in paths} + symbol_nodes: dict[tuple[Path, str], str] = {} + for node in nodes: + source_path = _js_source_path(str(node.get("source_file", "")), root) + if source_path is None: + continue + label = str(node.get("label", "")).strip().strip("()").lstrip(".") + if label and node.get("id"): + symbol_nodes[(source_path, label)] = str(node["id"]) + + def ensure_symbol_node(path: Path, name: str, line: int) -> str: + resolved_path = _resolve_path(path) + existing = symbol_nodes.get((resolved_path, name)) + if existing is not None: + return existing + node_id = _make_id(_file_stem(path), name) + symbol_nodes[(resolved_path, name)] = node_id + nodes.append({ + "id": node_id, + "label": name, + "file_type": "code", + "source_file": str(path), + "source_location": f"L{line}", + }) + return node_id + + existing_edges = { + ( + str(edge.get("source")), + str(edge.get("target")), + str(edge.get("relation")), + str(edge.get("context") or ""), + ) + for edge in edges + } -_RUBY_CONFIG = LanguageConfig( - ts_module="tree_sitter_ruby", - # `module Foo` is a container node just like `class Foo` in tree-sitter's - # Ruby grammar (name in a `constant` child, body in `body_statement`), so it - # gets a node and its methods attach via `method` (#1640). Without it, plain - # utility/`module_function` modules produced no node and their methods hung - # off the file via `contains` with dot-less labels. - class_types=frozenset({"class", "module"}), - function_types=frozenset({"method", "singleton_method"}), - import_types=frozenset(), - call_types=frozenset({"call"}), - call_function_field="method", - call_accessor_node_types=frozenset(), - name_fallback_child_types=("constant", "scope_resolution", "identifier"), - body_fallback_child_types=("body_statement",), - function_boundary_types=frozenset({"method", "singleton_method"}), -) + def add_edge(source: str, target: str, relation: str, context: str, line: int, source_path: Path, target_file: str | None = None, local_alias: str | None = None) -> None: + key = (source, target, relation, context or "") + if key in existing_edges: + return + existing_edges.add(key) + edge = { + "source": source, + "target": target, + "relation": relation, + "context": context, + "confidence": "EXTRACTED", + "source_file": str(source_path), + "source_location": f"L{line}", + "weight": 1.0, + } + # A re-export edge's target is a FILE node that can collapse with a + # same-basename cross-extension sibling; stamp the resolved target file so + # the id-disambiguation salt is keyed by the TARGET, not the importer (#1814). + if target_file is not None: + edge["target_file"] = target_file + # The local name this import bound in the importing file, when it differs + # from the target's own name (`from pkg import mod as alias`) -- lets the + # cross-file member-call resolver match `alias.func()` (#2082). + if local_alias is not None: + edge["local_alias"] = local_alias + edges.append(edge) -_CSHARP_CONFIG = LanguageConfig( - ts_module="tree_sitter_c_sharp", - class_types=frozenset({ - "class_declaration", - "interface_declaration", - "enum_declaration", - "struct_declaration", - "record_declaration", - }), - function_types=frozenset({"method_declaration"}), - import_types=frozenset({"using_directive"}), - call_types=frozenset({"invocation_expression"}), - call_function_field="function", - call_accessor_node_types=frozenset({"member_access_expression"}), - call_accessor_field="name", - body_fallback_child_types=("declaration_list",), - function_boundary_types=frozenset({"method_declaration"}), - import_handler=_import_csharp, -) + for declaration in facts.declarations: + ensure_symbol_node(declaration.file_path, declaration.name, declaration.line) -_KOTLIN_CONFIG = LanguageConfig( - ts_module="tree_sitter_kotlin", - class_types=frozenset({"class_declaration", "object_declaration"}), - function_types=frozenset({"function_declaration"}), - # Grammar 1.1.0 (PyPI tree_sitter_kotlin) names the import node `import`; - # older forks use `import_header`. Accept both (#2526). - import_types=frozenset({"import_header", "import"}), - call_types=frozenset({"call_expression"}), - call_function_field="", - call_accessor_node_types=frozenset({"navigation_expression"}), - call_accessor_field="", - # Different tree-sitter-kotlin grammar versions name plain identifier - # nodes differently: PyPI's `tree_sitter_kotlin` uses `identifier`, - # older forks use `simple_identifier`. Accept both so the extractor - # works across grammar generations. - name_fallback_child_types=("simple_identifier", "identifier"), - body_fallback_child_types=("function_body", "class_body", "enum_class_body"), - function_boundary_types=frozenset({"function_declaration"}), - import_handler=_import_kotlin, -) + local_aliases_by_file: dict[Path, dict[str, tuple[Path, str]]] = {} + for import_fact in facts.imports: + file_path = _resolve_path(import_fact.file_path) + local_aliases_by_file.setdefault(file_path, {})[import_fact.local_name] = ( + _resolve_path(import_fact.target_path), + import_fact.imported_name, + ) -_SCALA_CONFIG = LanguageConfig( - ts_module="tree_sitter_scala", - class_types=frozenset({"class_definition", "object_definition"}), - function_types=frozenset({"function_definition"}), - import_types=frozenset({"import_declaration"}), - call_types=frozenset({"call_expression"}), - call_function_field="", - call_accessor_node_types=frozenset({"field_expression"}), - call_accessor_field="field", - name_fallback_child_types=("identifier",), - body_fallback_child_types=("template_body",), - function_boundary_types=frozenset({"function_definition"}), - import_handler=_import_scala, -) + pending_aliases_by_file: dict[Path, list[_SymbolAliasFact]] = {} + for alias_fact in facts.aliases: + resolved_file = _resolve_path(alias_fact.file_path) + pending_aliases_by_file.setdefault(resolved_file, []).append(alias_fact) + + for file_path, aliases in pending_aliases_by_file.items(): + local_aliases = local_aliases_by_file.setdefault(file_path, {}) + changed = True + while changed: + changed = False + for alias_fact in aliases: + if alias_fact.alias in local_aliases: + continue + origin = local_aliases.get(alias_fact.target_name) + if origin is not None: + local_aliases[alias_fact.alias] = origin + changed = True + + named_exports_by_file: dict[Path, dict[str, tuple[Path, str]]] = {} + star_exports_by_file: dict[Path, list[Path]] = {} + + for star_fact in facts.star_exports: + source_path = _resolve_path(star_fact.file_path) + target_path = _resolve_path(star_fact.target_path) + star_exports_by_file.setdefault(source_path, []).append(target_path) + source_id = source_file_id.get(source_path) + if source_id is not None: + add_edge( + source_id, + _make_id(str(path_by_resolved.get(target_path, target_path))), + "re_exports", + "export", + star_fact.line, + star_fact.file_path, + target_file=str(path_by_resolved.get(target_path, target_path)), + ) -_PHP_CONFIG = LanguageConfig( - ts_module="tree_sitter_php", - ts_language_fn="language_php", - class_types=frozenset({"class_declaration"}), - function_types=frozenset({"function_definition", "method_declaration"}), - import_types=frozenset({"namespace_use_clause"}), - call_types=frozenset({"function_call_expression", "member_call_expression", "scoped_call_expression", "class_constant_access_expression"}), - static_prop_types=frozenset({"scoped_property_access_expression"}), - helper_fn_names=frozenset({"config"}), - container_bind_methods=frozenset({"bind", "singleton", "scoped", "instance"}), - event_listener_properties=frozenset({"listen", "subscribe"}), - call_function_field="function", - call_accessor_node_types=frozenset({"member_call_expression"}), - call_accessor_field="name", - name_fallback_child_types=("name",), - body_fallback_child_types=("declaration_list", "compound_statement"), - function_boundary_types=frozenset({"function_definition", "method_declaration"}), - import_handler=_import_php, -) + for namespace_fact in facts.namespace_exports: + source_path = _resolve_path(namespace_fact.file_path) + target_path = _resolve_path(namespace_fact.target_path) + namespace_id = ensure_symbol_node( + namespace_fact.file_path, + namespace_fact.exported_name, + namespace_fact.line, + ) + named_exports_by_file.setdefault(source_path, {})[ + namespace_fact.exported_name + ] = (source_path, namespace_fact.exported_name) + source_id = source_file_id.get(source_path) + if source_id is not None: + add_edge( + source_id, + namespace_id, + "contains", + "namespace_export", + namespace_fact.line, + namespace_fact.file_path, + ) + add_edge( + source_id, + _make_id(str(path_by_resolved.get(target_path, target_path))), + "re_exports", + "export", + namespace_fact.line, + namespace_fact.file_path, + target_file=str(path_by_resolved.get(target_path, target_path)), + ) + for export_fact in facts.exports: + file_path = _resolve_path(export_fact.file_path) + origin: tuple[Path, str] | None = None + if export_fact.target_path is not None and export_fact.target_name is not None: + origin = (_resolve_path(export_fact.target_path), export_fact.target_name) + elif export_fact.local_name is not None: + origin = local_aliases_by_file.get(file_path, {}).get(export_fact.local_name) + if origin is None and (file_path, export_fact.local_name) in symbol_nodes: + origin = (file_path, export_fact.local_name) + if origin is None: + continue + named_exports_by_file.setdefault(file_path, {})[export_fact.exported_name] = origin + if origin[0] != file_path: + source_id = source_file_id.get(file_path) + if source_id is not None: + add_edge( + source_id, + _make_id(str(path_by_resolved.get(origin[0], origin[0]))), + "re_exports", + "export", + export_fact.line, + export_fact.file_path, + target_file=str(path_by_resolved.get(origin[0], origin[0])), + ) -def _import_lua(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str, scope_stack: list[str] | None = None) -> None: - """Extract require('module') from Lua variable_declaration nodes.""" - text = _read_text(node, source) - import re - m = re.search(r"""require\s*[\('"]\s*['"]?([^'")\s]+)""", text) - if m: - raw_module = m.group(1) - if raw_module: - tgt_nid = _resolve_lua_import_target(raw_module, str_path) - if tgt_nid: - edges.append({ - "source": file_nid, - "target": tgt_nid, - "relation": "imports", - "context": "import", - "confidence": "EXTRACTED", - "confidence_score": 1.0, - "source_file": str_path, - "source_location": str(node.start_point[0] + 1), - "weight": 1.0, - }) - - -_LUA_CONFIG = LanguageConfig( - ts_module="tree_sitter_lua", - ts_language_fn="language", - class_types=frozenset(), - function_types=frozenset({"function_declaration"}), - import_types=frozenset({"variable_declaration"}), - call_types=frozenset({"function_call"}), - call_function_field="name", - call_accessor_node_types=frozenset({"method_index_expression"}), - call_accessor_field="name", - name_fallback_child_types=("identifier", "method_index_expression"), - body_fallback_child_types=("block",), - function_boundary_types=frozenset({"function_declaration"}), - import_handler=_import_lua, -) + def resolve_exported_origin(target_path: Path, imported_name: str, seen: set[tuple[Path, str]] | None = None) -> tuple[Path, str]: + target_path = _resolve_path(target_path) + key = (target_path, imported_name) + if seen is None: + seen = set() + if key in seen: + return key + seen.add(key) + origin = named_exports_by_file.get(target_path, {}).get(imported_name) + if origin is not None: + return resolve_exported_origin(origin[0], origin[1], seen) + for star_target in star_exports_by_file.get(target_path, []): + star_key = (star_target, imported_name) + if star_key in symbol_nodes: + return star_key + resolved = resolve_exported_origin(star_target, imported_name, seen) + if resolved in symbol_nodes: + return resolved + return key + for import_fact in facts.imports: + source_id = source_file_id.get(_resolve_path(import_fact.file_path)) + if source_id is None: + continue + origin_path, origin_symbol = resolve_exported_origin( + import_fact.target_path, + import_fact.imported_name, + ) + target_id = symbol_nodes.get((origin_path, origin_symbol)) + if target_id is None: + continue + add_edge( + source_id, + target_id, + "imports", + "import", + import_fact.line, + import_fact.file_path, + ) -def _import_swift(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str, scope_stack: list[str] | None = None) -> list[tuple[str, str]]: - """Emit module-level ``imports`` edges and report the imported modules. - - A Swift ``import CoreKit`` names a module, not a file path, so — unlike the - file-resolving JS/TS handlers — there is no existing node for the edge to - point at. The returned ``(id, label)`` pairs let the extractor materialize a - ``type=module`` anchor node so the edge survives; without it ``build_from_json`` - prunes every Swift import edge as a dangling/external reference (#1327). - """ - modules: list[tuple[str, str]] = [] - for child in node.children: - if child.type == "identifier": - raw = _read_text(child, source) - tgt_nid = _make_id(raw) - edges.append({ - "source": file_nid, - "target": tgt_nid, - "relation": "imports", - "context": "import", - "confidence": "EXTRACTED", - "source_file": str_path, - "source_location": f"L{node.start_point[0] + 1}", - "weight": 1.0, - }) - modules.append((tgt_nid, raw)) - break - return modules - - -_SWIFT_CONFIG = LanguageConfig( - ts_module="tree_sitter_swift", - class_types=frozenset({"class_declaration", "protocol_declaration"}), - function_types=frozenset({"function_declaration", "init_declaration", "deinit_declaration", "subscript_declaration"}), - import_types=frozenset({"import_declaration"}), - call_types=frozenset({"call_expression"}), - call_function_field="", - call_accessor_node_types=frozenset({"navigation_expression"}), - call_accessor_field="", - name_fallback_child_types=("simple_identifier", "type_identifier", "user_type"), - body_fallback_child_types=("class_body", "protocol_body", "function_body", "enum_class_body"), - function_boundary_types=frozenset({"function_declaration", "init_declaration", "deinit_declaration", "subscript_declaration"}), - import_handler=_import_swift, -) - -# ── Ruby local type inference (for member-call resolution) ───────────────────── - - -# `Const = (...)` shapes that define a lightweight class named after the -# constant. tree-sitter parses each as an `assignment`, not a `class`, so the -# generic class branch never saw them (#1640). - - -# ── Generic extractor ───────────────────────────────────────────────────────── - - -# ── Python rationale extraction ─────────────────────────────────────────────── - -_RATIONALE_PREFIXES = ("# NOTE:", "# IMPORTANT:", "# HACK:", "# WHY:", "# RATIONALE:", "# TODO:", "# FIXME:") - - -def _shorten_rationale_label(text: str, width: int = 80) -> str: - """Collapse whitespace and truncate ``text`` to ``width`` chars for a - rationale node label, cutting on a word boundary rather than mid-word. - Shared by the Python and JS/TS rationale extractors (#2206). - - ``textwrap.shorten`` collapses to just the placeholder when the first - "word" alone exceeds ``width`` (e.g. a docstring/comment that opens with - an unbroken URL) -- that would emit a content-free label, so fall back to - a plain character truncation of the normalized text in that case. - """ - label = textwrap.shorten(text, width=width, placeholder="…") - if label in ("", "…"): - flat = " ".join(text.split()) - label = flat if len(flat) <= width else flat[: width - 1] + "…" - return label - + # #1146: emit file-to-file imports_from edges for package-form submodule imports. + for from_path, to_path, line, local_name in facts.module_imports: + try: + from_rel = from_path.relative_to(root) + to_rel = to_path.relative_to(root) + except ValueError: + continue + source_id = _make_id(_file_stem(from_rel)) + target_id = _make_id(_file_stem(to_rel)) + add_edge( + source_id, target_id, "imports_from", "submodule_import", line, from_path, + local_alias=local_name if local_name != to_path.stem else None, + ) -def _is_autogenerated_python(source: bytes) -> bool: - """Return True if this Python file is auto-generated and its module docstring is noise. + # #2262 producer guard: never emit a `calls` use-edge from a source id + # that owns no node. All node appends (ensure_symbol_node, declarations, + # namespace exports) happened above, so the owned set is complete here. + # A node-less caller id can never be canonicalized by the extract() + # remaps (they learn only from nodes), so an absolute-derived one would + # leak the machine/scan-path slug into the edge source. Reattribute the + # edge to the caller's FILE node — the true file-level dependency + # survives, and the file id is exactly what the #2231 remap + # canonicalizes — or drop it when no file node id is available. + owned = {str(n.get("id")) for n in nodes} + for use_fact in facts.uses: + file_path = _resolve_path(use_fact.file_path) + target_id = None + unresolved_origin = local_aliases_by_file.get(file_path, {}).get(use_fact.local_name) + if unresolved_origin is not None: + origin_path, origin_symbol = resolve_exported_origin(*unresolved_origin) + target_id = symbol_nodes.get((origin_path, origin_symbol)) + if target_id is None and use_fact.relation in ("inherits", "implements"): + # Same-file fallback for HERITAGE only: a base declared in the same + # file (`class X extends Y`, `interface A extends B`) has no import + # alias, so resolve it directly against the file's own symbol nodes. + # Scoped to heritage because same-file calls/uses already resolve via + # the dedicated call-graph pass; widening this would duplicate those + # edges. Import resolution still takes precedence (#1095). + target_id = symbol_nodes.get((file_path, use_fact.local_name)) + if target_id is None: + continue + source_id = use_fact.source_id + if use_fact.relation == "calls" and source_id not in owned: + source_id = source_file_id.get(file_path) + if source_id is None: + continue + add_edge( + source_id, + target_id, + use_fact.relation, + use_fact.context, + use_fact.line, + use_fact.file_path, + ) - Covers: Alembic/Flask-Migrate revisions, Django migrations, protobuf/gRPC/OpenAPI stubs. - Module docstrings in these files are change annotations or boilerplate, not rationale. - """ - head = source[:2048].decode("utf-8", errors="replace") - # Generic generated-file markers (protobuf, gRPC, OpenAPI codegen, etc.) - if any(m in head for m in ("DO NOT EDIT", "@generated", "Generated by the protocol buffer")): - return True - # Alembic / Flask-Migrate revision files - if (re.search(r"^revision\s*[:=]", head, re.MULTILINE) - and "def upgrade(" in head - and "down_revision" in head): - return True - # Django migrations - if "class Migration(migrations.Migration)" in head and "operations" in head: - return True - return False - - -def _extract_python_rationale(path: Path, result: dict) -> None: - """Post-pass: extract docstrings and rationale comments from Python source. - Mutates result in-place by appending to result['nodes'] and result['edges']. - """ +def _parse_js_tree(path: Path): try: - import tree_sitter_python as tspython from tree_sitter import Language, Parser - language = Language(tspython.language()) + # .vue embeds the script in non-JS markup; mask it out and parse the + #