From c47e82b490dd9314d3b53520ff7e1d88bc450ba2 Mon Sep 17 00:00:00 2001 From: Yingzhao Ouyang Date: Sun, 28 Jun 2026 00:23:59 +0800 Subject: [PATCH 1/9] feat(detect): index Jupyter notebooks via markdown sidecar extraction Convert .ipynb files to markdown sidecars (code cells as fenced blocks, markdown cells verbatim, outputs stripped) and classify them as documents so notebook-heavy ML corpora are no longer silently dropped during scan. Fixes #1497 Co-authored-by: Cursor --- graphify/detect.py | 72 +++++++++++++++++++++++++----- tests/test_detect.py | 103 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 163 insertions(+), 12 deletions(-) diff --git a/graphify/detect.py b/graphify/detect.py index 92b773f7b..98b8dceb0 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -32,6 +32,9 @@ class FileType(str, Enum): PAPER_EXTENSIONS = {'.pdf'} IMAGE_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg'} OFFICE_EXTENSIONS = {'.docx', '.xlsx'} +# Notebooks are converted to markdown sidecars before indexing — do NOT add .ipynb +# to CODE_EXTENSIONS or DOC_EXTENSIONS. +NOTEBOOK_EXTENSIONS = {'.ipynb'} VIDEO_EXTENSIONS = {'.mp4', '.mov', '.webm', '.mkv', '.avi', '.m4v', '.mp3', '.wav', '.m4a', '.ogg'} CORPUS_WARN_THRESHOLD = 50_000 # words - below this, warn "you may not need a graph" @@ -413,6 +416,8 @@ def classify_file(path: Path) -> FileType | None: return FileType.DOCUMENT if ext in OFFICE_EXTENSIONS: return FileType.DOCUMENT + if ext in NOTEBOOK_EXTENSIONS: + return FileType.DOCUMENT if ext in GOOGLE_WORKSPACE_EXTENSIONS: return FileType.DOCUMENT if ext in VIDEO_EXTENSIONS: @@ -601,20 +606,29 @@ def _edge(src: str, tgt: str, relation: str) -> None: return {"nodes": nodes, "edges": edges} -def convert_office_file(path: Path, out_dir: Path) -> Path | None: - """Convert a .docx or .xlsx to a markdown sidecar in out_dir. +def ipynb_to_markdown(path: Path) -> str: + """Convert a Jupyter notebook to markdown, stripping outputs.""" + if not _file_within_size_cap(path): + return "" + try: + nb = json.loads(path.read_text(encoding="utf-8", errors="ignore")) + lines = [] + for cell in nb.get("cells", []): + ct = cell.get("cell_type") + src = "".join(cell.get("source", [])) + if not src.strip(): + continue + if ct == "markdown": + lines.append(src) + elif ct == "code": + lines.append(f"```python\n{src}\n```") + return "\n\n".join(lines) + except Exception: + return "" - Returns the path of the converted .md file, or None if conversion failed - or the required library is not installed. - """ - ext = path.suffix.lower() - if ext == ".docx": - text = docx_to_markdown(path) - elif ext == ".xlsx": - text = xlsx_to_markdown(path) - else: - return None +def _write_markdown_sidecar(path: Path, out_dir: Path, text: str) -> Path | None: + """Write a markdown sidecar for a converted source file.""" if not text.strip(): return None @@ -642,6 +656,30 @@ def convert_office_file(path: Path, out_dir: Path) -> Path | None: return out_path +def convert_office_file(path: Path, out_dir: Path) -> Path | None: + """Convert a .docx or .xlsx to a markdown sidecar in out_dir. + + Returns the path of the converted .md file, or None if conversion failed + or the required library is not installed. + """ + ext = path.suffix.lower() + if ext == ".docx": + text = docx_to_markdown(path) + elif ext == ".xlsx": + text = xlsx_to_markdown(path) + else: + return None + + return _write_markdown_sidecar(path, out_dir, text) + + +def convert_notebook_file(path: Path, out_dir: Path) -> Path | None: + """Convert a .ipynb to a markdown sidecar in out_dir.""" + if path.suffix.lower() not in NOTEBOOK_EXTENSIONS: + return None + return _write_markdown_sidecar(path, out_dir, ipynb_to_markdown(path)) + + def count_words(path: Path) -> int: try: ext = path.suffix.lower() @@ -1131,6 +1169,16 @@ def detect(root: Path, *, follow_symlinks: bool | None = None, google_workspace: # Conversion failed (library not installed) - skip with note skipped_sensitive.append(str(p) + " [office conversion failed - pip install graphifyy[office]]") continue + if p.suffix.lower() in NOTEBOOK_EXTENSIONS: + md_path = convert_notebook_file(p, converted_dir) + if md_path: + if _is_ignored(md_path, root, ignore_patterns, _cache=ignore_cache): + continue + files[ftype].append(str(md_path)) + total_words += count_words(md_path) + else: + skipped_sensitive.append(str(p) + " [notebook conversion failed]") + continue files[ftype].append(str(p)) if ftype != FileType.VIDEO: total_words += count_words(p) diff --git a/tests/test_detect.py b/tests/test_detect.py index 76282f295..fb1580573 100644 --- a/tests/test_detect.py +++ b/tests/test_detect.py @@ -1515,3 +1515,106 @@ def test_convert_office_file_does_not_rewrite_existing_sidecar(tmp_path, monkeyp second = detect_mod.convert_office_file(src, out_dir) assert second == first assert second.stat().st_mtime_ns == mtime_before + + +def _minimal_ipynb(cells): + import json + return json.dumps({"cells": cells, "nbformat": 4, "nbformat_minor": 5}) + + +def test_classify_ipynb(): + assert classify_file(Path("analysis.ipynb")) == FileType.DOCUMENT + + +def test_ipynb_to_markdown_mixed_cells(tmp_path): + nb_path = tmp_path / "nb.ipynb" + nb_path.write_text( + _minimal_ipynb([ + {"cell_type": "markdown", "source": "# Title\n\nIntro text."}, + { + "cell_type": "code", + "source": "import pandas as pd\nprint('hi')", + "outputs": [{"output_type": "stream", "text": "hi\n"}], + }, + {"cell_type": "markdown", "source": "## Section"}, + ]), + encoding="utf-8", + ) + md = detect_mod.ipynb_to_markdown(nb_path) + assert "# Title" in md + assert "```python\nimport pandas as pd" in md + assert "hi\n" not in md # outputs stripped + assert "## Section" in md + assert md.index("# Title") < md.index("```python") < md.index("## Section") + + +def test_ipynb_to_markdown_empty_notebook(tmp_path): + nb_path = tmp_path / "empty.ipynb" + nb_path.write_text(_minimal_ipynb([]), encoding="utf-8") + assert detect_mod.ipynb_to_markdown(nb_path) == "" + + +def test_ipynb_to_markdown_malformed_json(tmp_path): + nb_path = tmp_path / "bad.ipynb" + nb_path.write_text("{not valid json", encoding="utf-8") + assert detect_mod.ipynb_to_markdown(nb_path) == "" + + +def test_detect_converts_notebook_to_sidecar(tmp_path): + nb_path = tmp_path / "analysis.ipynb" + nb_path.write_text( + _minimal_ipynb([ + {"cell_type": "markdown", "source": "# Analysis"}, + {"cell_type": "code", "source": "x = 1"}, + ]), + encoding="utf-8", + ) + result = detect(tmp_path) + assert len(result["files"]["document"]) == 1 + sidecar = Path(result["files"]["document"][0]) + assert sidecar.suffix == ".md" + assert sidecar.exists() + text = sidecar.read_text(encoding="utf-8") + assert "converted from analysis.ipynb" in text + assert "# Analysis" in text + assert "x = 1" in text + assert result["total_words"] > 0 + + +def test_convert_notebook_file_empty_notebook_no_sidecar(tmp_path): + nb_path = tmp_path / "empty.ipynb" + nb_path.write_text(_minimal_ipynb([]), encoding="utf-8") + out_dir = tmp_path / "converted" + assert detect_mod.convert_notebook_file(nb_path, out_dir) is None + assert not list(out_dir.glob("*.md")) + + +def test_detect_incremental_notebook_sidecar_tracks_changes(tmp_path): + """When a notebook is edited, re-converting after removing the stale sidecar + produces updated content that detect_incremental picks up as changed.""" + nb_path = tmp_path / "analysis.ipynb" + nb_path.write_text( + _minimal_ipynb([{"cell_type": "markdown", "source": "v1"}]), + encoding="utf-8", + ) + first = detect(tmp_path) + sidecar = Path(first["files"]["document"][0]) + manifest_path = tmp_path / "graphify-out" / "manifest.json" + save_manifest( + {sidecar: {"mtime": sidecar.stat().st_mtime, "ast_hash": "x", "semantic_hash": "y"}}, + str(manifest_path), + root=tmp_path, + ) + + nb_path.write_text( + _minimal_ipynb([{"cell_type": "markdown", "source": "v2 updated"}]), + encoding="utf-8", + ) + sidecar.unlink() + converted_dir = tmp_path / "graphify-out" / "converted" + new_sidecar = detect_mod.convert_notebook_file(nb_path, converted_dir) + assert new_sidecar is not None + assert "v2 updated" in new_sidecar.read_text(encoding="utf-8") + + inc = detect_incremental(tmp_path, manifest_path=str(manifest_path)) + assert any(str(new_sidecar) == f for f in inc["new_files"]["document"]) From 208378aff17a09a4176c3082ab136955845b4b1a Mon Sep 17 00:00:00 2001 From: Yingzhao Ouyang Date: Sun, 28 Jun 2026 00:33:05 +0800 Subject: [PATCH 2/9] refactor(detect): keep convert_office_file unchanged, add notebook path separately Restore convert_office_file inline and place ipynb_to_markdown/convert_notebook_file after it so the PR diff clearly adds notebook support without touching office conversion. Co-authored-by: Cursor --- graphify/detect.py | 85 +++++++++++++++++++++++++++------------------- 1 file changed, 50 insertions(+), 35 deletions(-) diff --git a/graphify/detect.py b/graphify/detect.py index 98b8dceb0..b9e824098 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -606,6 +606,47 @@ def _edge(src: str, tgt: str, relation: str) -> None: return {"nodes": nodes, "edges": edges} +def convert_office_file(path: Path, out_dir: Path) -> Path | None: + """Convert a .docx or .xlsx to a markdown sidecar in out_dir. + + Returns the path of the converted .md file, or None if conversion failed + or the required library is not installed. + """ + ext = path.suffix.lower() + if ext == ".docx": + text = docx_to_markdown(path) + elif ext == ".xlsx": + text = xlsx_to_markdown(path) + else: + return None + + if not text.strip(): + return None + + out_dir.mkdir(parents=True, exist_ok=True) + # Use a stable name derived from the original path to avoid collisions. + # Normalize the resolved path to NFC before hashing: on macOS (HFS+/APFS) + # os.walk/rglob return filenames in NFD, while Python string literals and + # directly-constructed Path objects are NFC, so the same source file would + # otherwise hash to different sidecar names across runs — causing --update + # to treat every Office file as new and re-extract it (#1226). + import hashlib + import unicodedata + normalized_path = unicodedata.normalize("NFC", str(path.resolve())) + name_hash = hashlib.sha256(normalized_path.encode()).hexdigest()[:8] + out_path = out_dir / f"{path.stem}_{name_hash}.md" + # Once the hash is stable the sidecar name is deterministic; skip re-writing + # an existing sidecar so an unchanged source never churns its mtime (which + # would still flag it as changed in detect_incremental). + if out_path.exists(): + return out_path + out_path.write_text( + f"\n\n{text}", + encoding="utf-8", + ) + return out_path + + def ipynb_to_markdown(path: Path) -> str: """Convert a Jupyter notebook to markdown, stripping outputs.""" if not _file_within_size_cap(path): @@ -627,26 +668,24 @@ def ipynb_to_markdown(path: Path) -> str: return "" -def _write_markdown_sidecar(path: Path, out_dir: Path, text: str) -> Path | None: - """Write a markdown sidecar for a converted source file.""" +def convert_notebook_file(path: Path, out_dir: Path) -> Path | None: + """Convert a .ipynb to a markdown sidecar in out_dir. + + Mirrors convert_office_file(): same sidecar naming and mtime semantics. + """ + if path.suffix.lower() not in NOTEBOOK_EXTENSIONS: + return None + + text = ipynb_to_markdown(path) if not text.strip(): return None out_dir.mkdir(parents=True, exist_ok=True) - # Use a stable name derived from the original path to avoid collisions. - # Normalize the resolved path to NFC before hashing: on macOS (HFS+/APFS) - # os.walk/rglob return filenames in NFD, while Python string literals and - # directly-constructed Path objects are NFC, so the same source file would - # otherwise hash to different sidecar names across runs — causing --update - # to treat every Office file as new and re-extract it (#1226). import hashlib import unicodedata normalized_path = unicodedata.normalize("NFC", str(path.resolve())) name_hash = hashlib.sha256(normalized_path.encode()).hexdigest()[:8] out_path = out_dir / f"{path.stem}_{name_hash}.md" - # Once the hash is stable the sidecar name is deterministic; skip re-writing - # an existing sidecar so an unchanged source never churns its mtime (which - # would still flag it as changed in detect_incremental). if out_path.exists(): return out_path out_path.write_text( @@ -656,30 +695,6 @@ def _write_markdown_sidecar(path: Path, out_dir: Path, text: str) -> Path | None return out_path -def convert_office_file(path: Path, out_dir: Path) -> Path | None: - """Convert a .docx or .xlsx to a markdown sidecar in out_dir. - - Returns the path of the converted .md file, or None if conversion failed - or the required library is not installed. - """ - ext = path.suffix.lower() - if ext == ".docx": - text = docx_to_markdown(path) - elif ext == ".xlsx": - text = xlsx_to_markdown(path) - else: - return None - - return _write_markdown_sidecar(path, out_dir, text) - - -def convert_notebook_file(path: Path, out_dir: Path) -> Path | None: - """Convert a .ipynb to a markdown sidecar in out_dir.""" - if path.suffix.lower() not in NOTEBOOK_EXTENSIONS: - return None - return _write_markdown_sidecar(path, out_dir, ipynb_to_markdown(path)) - - def count_words(path: Path) -> int: try: ext = path.suffix.lower() From df6515103e60f778b6f119d136364924e56948c2 Mon Sep 17 00:00:00 2001 From: Yingzhao Ouyang Date: Sun, 28 Jun 2026 00:37:38 +0800 Subject: [PATCH 3/9] fix(detect): ignore notebook output-only changes during conversion Compare extracted sidecar payload before rewriting so re-running a notebook updates the .ipynb on disk without bumping sidecar mtime or triggering re-extraction. Cell source edits still refresh the sidecar. Co-authored-by: Cursor --- graphify/detect.py | 12 ++--- tests/test_detect.py | 108 +++++++++++++++++++++++++++++++++++++------ 2 files changed, 99 insertions(+), 21 deletions(-) diff --git a/graphify/detect.py b/graphify/detect.py index b9e824098..aff86a387 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -671,7 +671,9 @@ def ipynb_to_markdown(path: Path) -> str: def convert_notebook_file(path: Path, out_dir: Path) -> Path | None: """Convert a .ipynb to a markdown sidecar in out_dir. - Mirrors convert_office_file(): same sidecar naming and mtime semantics. + Mirrors convert_office_file() sidecar naming. Unlike office files, an + existing sidecar is refreshed when cell sources change but left untouched + when only outputs/metadata changed — notebook re-runs must not re-extract. """ if path.suffix.lower() not in NOTEBOOK_EXTENSIONS: return None @@ -686,12 +688,10 @@ def convert_notebook_file(path: Path, out_dir: Path) -> Path | None: normalized_path = unicodedata.normalize("NFC", str(path.resolve())) name_hash = hashlib.sha256(normalized_path.encode()).hexdigest()[:8] out_path = out_dir / f"{path.stem}_{name_hash}.md" - if out_path.exists(): + payload = f"\n\n{text}" + if out_path.exists() and out_path.read_text(encoding="utf-8") == payload: return out_path - out_path.write_text( - f"\n\n{text}", - encoding="utf-8", - ) + out_path.write_text(payload, encoding="utf-8") return out_path diff --git a/tests/test_detect.py b/tests/test_detect.py index fb1580573..e183dabad 100644 --- a/tests/test_detect.py +++ b/tests/test_detect.py @@ -1589,32 +1589,110 @@ def test_convert_notebook_file_empty_notebook_no_sidecar(tmp_path): assert not list(out_dir.glob("*.md")) -def test_detect_incremental_notebook_sidecar_tracks_changes(tmp_path): - """When a notebook is edited, re-converting after removing the stale sidecar - produces updated content that detect_incremental picks up as changed.""" +def test_convert_notebook_file_output_change_preserves_sidecar_mtime(tmp_path): + """Re-running a notebook updates outputs in the .ipynb but not the sidecar.""" + nb_path = tmp_path / "analysis.ipynb" + nb_path.write_text( + _minimal_ipynb([ + {"cell_type": "code", "source": "print(1)", "outputs": []}, + ]), + encoding="utf-8", + ) + out_dir = tmp_path / "converted" + sidecar = detect_mod.convert_notebook_file(nb_path, out_dir) + assert sidecar is not None + mtime_before = sidecar.stat().st_mtime_ns + + nb_path.write_text( + _minimal_ipynb([ + { + "cell_type": "code", + "source": "print(1)", + "outputs": [{"output_type": "stream", "text": "1\n"}], + "execution_count": 1, + }, + ]), + encoding="utf-8", + ) + again = detect_mod.convert_notebook_file(nb_path, out_dir) + assert again == sidecar + assert again.stat().st_mtime_ns == mtime_before + + +def test_convert_notebook_file_source_change_rewrites_sidecar(tmp_path): + nb_path = tmp_path / "analysis.ipynb" + nb_path.write_text( + _minimal_ipynb([{"cell_type": "code", "source": "x = 1", "outputs": []}]), + encoding="utf-8", + ) + out_dir = tmp_path / "converted" + sidecar = detect_mod.convert_notebook_file(nb_path, out_dir) + mtime_before = sidecar.stat().st_mtime_ns + + nb_path.write_text( + _minimal_ipynb([{"cell_type": "code", "source": "x = 2", "outputs": []}]), + encoding="utf-8", + ) + updated = detect_mod.convert_notebook_file(nb_path, out_dir) + assert updated == sidecar + assert "x = 2" in updated.read_text(encoding="utf-8") + assert updated.stat().st_mtime_ns >= mtime_before + + +def test_detect_refreshes_notebook_sidecar_on_source_change(tmp_path): + """Cell source edits must update the sidecar so a later extract sees new content.""" nb_path = tmp_path / "analysis.ipynb" nb_path.write_text( _minimal_ipynb([{"cell_type": "markdown", "source": "v1"}]), encoding="utf-8", ) + detect(tmp_path) + converted_dir = tmp_path / "graphify-out" / "converted" + sidecar = next(converted_dir.glob("analysis_*.md")) + + nb_path.write_text( + _minimal_ipynb([{"cell_type": "markdown", "source": "v2 updated"}]), + encoding="utf-8", + ) + detect(tmp_path) + assert "v2 updated" in sidecar.read_text(encoding="utf-8") + + +def test_detect_incremental_ignores_notebook_output_only_changes(tmp_path): + import json + + nb_path = tmp_path / "analysis.ipynb" + nb_path.write_text( + _minimal_ipynb([{"cell_type": "code", "source": "print(1)", "outputs": []}]), + encoding="utf-8", + ) first = detect(tmp_path) sidecar = Path(first["files"]["document"][0]) + mtime_before = sidecar.stat().st_mtime_ns manifest_path = tmp_path / "graphify-out" / "manifest.json" - save_manifest( - {sidecar: {"mtime": sidecar.stat().st_mtime, "ast_hash": "x", "semantic_hash": "y"}}, - str(manifest_path), - root=tmp_path, + Path(manifest_path).write_text( + json.dumps({ + str(sidecar): { + "mtime": sidecar.stat().st_mtime, + "ast_hash": "a" * 32, + "semantic_hash": "b" * 32, + } + }), + encoding="utf-8", ) nb_path.write_text( - _minimal_ipynb([{"cell_type": "markdown", "source": "v2 updated"}]), + _minimal_ipynb([ + { + "cell_type": "code", + "source": "print(1)", + "outputs": [{"output_type": "stream", "text": "1\n"}], + "execution_count": 1, + }, + ]), encoding="utf-8", ) - sidecar.unlink() - converted_dir = tmp_path / "graphify-out" / "converted" - new_sidecar = detect_mod.convert_notebook_file(nb_path, converted_dir) - assert new_sidecar is not None - assert "v2 updated" in new_sidecar.read_text(encoding="utf-8") - inc = detect_incremental(tmp_path, manifest_path=str(manifest_path)) - assert any(str(new_sidecar) == f for f in inc["new_files"]["document"]) + assert sidecar.stat().st_mtime_ns == mtime_before + assert not inc["new_files"]["document"] + assert str(sidecar) in inc["unchanged_files"]["document"] From 992a41d77cfc459bedeb13ca6272c79e06bb5946 Mon Sep 17 00:00:00 2001 From: Yingzhao Ouyang Date: Mon, 29 Jun 2026 01:41:19 +0800 Subject: [PATCH 4/9] Change code block syntax from 'python' to 'code' --- graphify/detect.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/graphify/detect.py b/graphify/detect.py index aff86a387..ba0fc88e5 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -662,7 +662,7 @@ def ipynb_to_markdown(path: Path) -> str: if ct == "markdown": lines.append(src) elif ct == "code": - lines.append(f"```python\n{src}\n```") + lines.append(f"```code\n{src}\n```") return "\n\n".join(lines) except Exception: return "" From fd2ec4ec445961a19b226d7530b59637a092673d Mon Sep 17 00:00:00 2001 From: Yingzhao Ouyang Date: Mon, 29 Jun 2026 03:06:21 +0800 Subject: [PATCH 5/9] Enhance ipynb_to_markdown for kernel language support Updated the ipynb_to_markdown function to use the notebook's kernel language for fenced code blocks instead of a generic fallback. Improved handling of source code extraction from notebook cells. --- graphify/detect.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/graphify/detect.py b/graphify/detect.py index ba0fc88e5..105d8f100 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -648,26 +648,40 @@ def convert_office_file(path: Path, out_dir: Path) -> Path | None: def ipynb_to_markdown(path: Path) -> str: - """Convert a Jupyter notebook to markdown, stripping outputs.""" + """Convert a Jupyter notebook to markdown, stripping outputs. + + Uses the notebook's kernel language from metadata for fenced code blocks, + falling back to ``code`` when the metadata is absent. + """ if not _file_within_size_cap(path): return "" try: nb = json.loads(path.read_text(encoding="utf-8", errors="ignore")) + # Resolve the kernel language from notebook metadata so fenced code + # blocks use the correct language identifier (e.g. ```python) rather + # than the generic ```code fallback. + meta = nb.get("metadata", {}) + lang = ( + meta.get("language_info", {}).get("name") + or meta.get("kernelspec", {}).get("language") + or "code" + ) lines = [] for cell in nb.get("cells", []): ct = cell.get("cell_type") - src = "".join(cell.get("source", [])) + raw_src = cell.get("source", []) + src = raw_src if isinstance(raw_src, str) else "".join(raw_src) if not src.strip(): continue if ct == "markdown": lines.append(src) elif ct == "code": - lines.append(f"```code\n{src}\n```") + lines.append(f"```{lang}\n{src}\n```") return "\n\n".join(lines) except Exception: return "" - + def convert_notebook_file(path: Path, out_dir: Path) -> Path | None: """Convert a .ipynb to a markdown sidecar in out_dir. From c27f7e605681ede1f3a2354679d13c0fa8a5f272 Mon Sep 17 00:00:00 2001 From: Yingzhao Ouyang Date: Sat, 1 Aug 2026 18:13:13 +0800 Subject: [PATCH 6/9] fix(detect): address PR review on notebook sidecars Hash the scan-root-relative notebook path so one tracked .ipynb yields the same sidecar name across clones instead of one per checkout location (#2059), matching convert_office_file. Route word counts through the cached _wc helper like every other branch in detect(), and fix the kernel-language test that still asserted a hardcoded python fence. Co-authored-by: Cursor --- graphify/detect.py | 41 ++++++++++---- tests/test_detect.py | 132 +++++++++++++++++++++++++++++++++++++++---- 2 files changed, 151 insertions(+), 22 deletions(-) diff --git a/graphify/detect.py b/graphify/detect.py index 3f188ecc1..f8d72816f 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -809,13 +809,16 @@ def ipynb_to_markdown(path: Path) -> str: except Exception: return "" - -def convert_notebook_file(path: Path, out_dir: Path) -> Path | None: + +def convert_notebook_file(path: Path, out_dir: Path, root: "Path | None" = None) -> Path | None: """Convert a .ipynb to a markdown sidecar in out_dir. - Mirrors convert_office_file() sidecar naming. Unlike office files, an - existing sidecar is refreshed when cell sources change but left untouched - when only outputs/metadata changed — notebook re-runs must not re-extract. + Naming mirrors convert_office_file(). The rewrite check does not: office + files compare mtimes, but re-running a notebook rewrites the .ipynb with + fresh outputs and execution counts while every cell source stays the same. + Comparing the extracted markdown instead keeps the sidecar (and its mtime) + untouched through a re-run, so detect_incremental does not re-extract a + notebook whose code and prose never changed. """ if path.suffix.lower() not in NOTEBOOK_EXTENSIONS: return None @@ -825,14 +828,31 @@ def convert_notebook_file(path: Path, out_dir: Path) -> Path | None: return None out_dir.mkdir(parents=True, exist_ok=True) + # Hash the scan-root-RELATIVE path for the same reason convert_office_file + # does: an absolute key salts the name with the checkout location, so one + # tracked notebook in two clones emits two byte-identical sidecars (#2059). + # NFC-normalize first to survive macOS NFD path drift (#1226). import hashlib import unicodedata - normalized_path = unicodedata.normalize("NFC", str(path.resolve())) + if root is None: + # Default layout: out_dir is //converted. + root = out_dir.parent.parent + try: + key = path.resolve().relative_to(Path(root).resolve()).as_posix() + except (ValueError, OSError): + # Outside the scan root (--include sources, custom GRAPHIFY_OUT + # layouts): keep the absolute form rather than guessing. + key = str(path.resolve()) + normalized_path = unicodedata.normalize("NFC", key) name_hash = hashlib.sha256(normalized_path.encode()).hexdigest()[:8] out_path = out_dir / f"{path.stem}_{name_hash}.md" payload = f"\n\n{text}" - if out_path.exists() and out_path.read_text(encoding="utf-8") == payload: - return out_path + try: + with open(_os_path(out_path), encoding="utf-8") as f: + if f.read() == payload: + return out_path + except OSError: + pass out_path.write_text(payload, encoding="utf-8") return out_path @@ -1452,13 +1472,14 @@ def _on_walk_error(err: OSError) -> None: # Conversion failed (library not installed) - skip with note skipped_sensitive.append(str(p) + " [office conversion failed - pip install graphifyy[office]]") continue + # Notebooks: same sidecar treatment as Office files if p.suffix.lower() in NOTEBOOK_EXTENSIONS: - md_path = convert_notebook_file(p, converted_dir) + md_path = convert_notebook_file(p, converted_dir, root=root) if md_path: if _is_ignored(md_path, root, ignore_patterns, _cache=ignore_cache): continue files[ftype].append(str(md_path)) - total_words += count_words(md_path) + total_words += _wc(md_path) else: skipped_sensitive.append(str(p) + " [notebook conversion failed]") continue diff --git a/tests/test_detect.py b/tests/test_detect.py index c5b1431ee..437fd4969 100644 --- a/tests/test_detect.py +++ b/tests/test_detect.py @@ -2086,9 +2086,12 @@ def test_convert_office_file_does_not_rewrite_existing_sidecar(tmp_path, monkeyp assert second.stat().st_mtime_ns == mtime_before -def _minimal_ipynb(cells): +def _minimal_ipynb(cells, metadata=None): import json - return json.dumps({"cells": cells, "nbformat": 4, "nbformat_minor": 5}) + nb = {"cells": cells, "nbformat": 4, "nbformat_minor": 5} + if metadata is not None: + nb["metadata"] = metadata + return json.dumps(nb) def test_classify_ipynb(): @@ -2098,15 +2101,18 @@ def test_classify_ipynb(): def test_ipynb_to_markdown_mixed_cells(tmp_path): nb_path = tmp_path / "nb.ipynb" nb_path.write_text( - _minimal_ipynb([ - {"cell_type": "markdown", "source": "# Title\n\nIntro text."}, - { - "cell_type": "code", - "source": "import pandas as pd\nprint('hi')", - "outputs": [{"output_type": "stream", "text": "hi\n"}], - }, - {"cell_type": "markdown", "source": "## Section"}, - ]), + _minimal_ipynb( + [ + {"cell_type": "markdown", "source": "# Title\n\nIntro text."}, + { + "cell_type": "code", + "source": "import pandas as pd\nprint('hi')", + "outputs": [{"output_type": "stream", "text": "hi\n"}], + }, + {"cell_type": "markdown", "source": "## Section"}, + ], + metadata={"language_info": {"name": "python"}}, + ), encoding="utf-8", ) md = detect_mod.ipynb_to_markdown(nb_path) @@ -2117,6 +2123,46 @@ def test_ipynb_to_markdown_mixed_cells(tmp_path): assert md.index("# Title") < md.index("```python") < md.index("## Section") +def test_ipynb_to_markdown_uses_non_python_kernel_language(tmp_path): + """Notebooks are not Python-only: the fence must name the kernel language.""" + nb_path = tmp_path / "survey.ipynb" + nb_path.write_text( + _minimal_ipynb( + [{"cell_type": "code", "source": "summary(df)"}], + metadata={"kernelspec": {"language": "R"}}, + ), + encoding="utf-8", + ) + assert "```R\nsummary(df)" in detect_mod.ipynb_to_markdown(nb_path) + + +def test_ipynb_to_markdown_falls_back_to_generic_fence(tmp_path): + """A notebook with no language metadata still fences its code cells.""" + nb_path = tmp_path / "bare.ipynb" + nb_path.write_text( + _minimal_ipynb([{"cell_type": "code", "source": "x = 1"}]), + encoding="utf-8", + ) + assert "```code\nx = 1" in detect_mod.ipynb_to_markdown(nb_path) + + +def test_ipynb_to_markdown_accepts_string_source(tmp_path): + """nbformat allows `source` as a plain string as well as a list of lines.""" + import json + + nb_path = tmp_path / "str.ipynb" + nb_path.write_text( + json.dumps({ + "cells": [{"cell_type": "code", "source": "x = 1"}], + "metadata": {"language_info": {"name": "python"}}, + "nbformat": 4, + "nbformat_minor": 5, + }), + encoding="utf-8", + ) + assert "```python\nx = 1" in detect_mod.ipynb_to_markdown(nb_path) + + def test_ipynb_to_markdown_empty_notebook(tmp_path): nb_path = tmp_path / "empty.ipynb" nb_path.write_text(_minimal_ipynb([]), encoding="utf-8") @@ -2266,7 +2312,69 @@ def test_detect_incremental_ignores_notebook_output_only_changes(tmp_path): assert not inc["new_files"]["document"] assert str(sidecar) in inc["unchanged_files"]["document"] - + +def test_convert_notebook_file_sidecar_name_stable_across_checkouts(tmp_path): + """#2059: like Office sidecars, the notebook sidecar name must derive from the + scan-root-RELATIVE path so two clones of the same repo agree on it.""" + def _sidecar(root): + src = root / "notebooks" / "analysis.ipynb" + src.parent.mkdir(parents=True, exist_ok=True) + src.write_text( + _minimal_ipynb([{"cell_type": "code", "source": "x = 1"}]), + encoding="utf-8", + ) + return detect_mod.convert_notebook_file( + src, root / "graphify-out" / "converted", root=root + ) + + out_a = _sidecar(tmp_path / "checkout-a") + out_b = _sidecar(tmp_path / "somewhere-else" / "checkout-b") + assert out_a is not None and out_b is not None + assert out_a.name == out_b.name, "sidecar name must be stable across checkouts (#2059)" + assert out_a.parent != out_b.parent # sanity: genuinely different locations + + # No explicit root -> the out_dir.parent.parent fallback yields the same name. + fallback = detect_mod.convert_notebook_file( + tmp_path / "checkout-a" / "notebooks" / "analysis.ipynb", + tmp_path / "checkout-a" / "graphify-out" / "converted", + ) + assert fallback is not None and fallback.name == out_a.name + + +def test_convert_notebook_file_hash_disambiguates_same_stem(tmp_path): + """Same-stem notebooks in different subdirs must still get distinct sidecars.""" + root = tmp_path / "repo" + for sub in ("a", "b"): + nb = root / sub / "analysis.ipynb" + nb.parent.mkdir(parents=True) + nb.write_text( + _minimal_ipynb([{"cell_type": "code", "source": f"x = '{sub}'"}]), + encoding="utf-8", + ) + out_dir = root / "graphify-out" / "converted" + out_a = detect_mod.convert_notebook_file(root / "a" / "analysis.ipynb", out_dir, root=root) + out_b = detect_mod.convert_notebook_file(root / "b" / "analysis.ipynb", out_dir, root=root) + assert out_a is not None and out_b is not None + assert out_a.name != out_b.name, "same-stem notebooks in different dirs must differ (#2059)" + + +def test_convert_notebook_file_outside_root_falls_back(tmp_path): + """A notebook outside the scan root falls back to the absolute-path hash + without raising, and stays deterministic.""" + root = tmp_path / "repo" + out_dir = root / "graphify-out" / "converted" + out_dir.mkdir(parents=True) + outside = tmp_path / "elsewhere" / "analysis.ipynb" + outside.parent.mkdir(parents=True) + outside.write_text( + _minimal_ipynb([{"cell_type": "code", "source": "x = 1"}]), + encoding="utf-8", + ) + out1 = detect_mod.convert_notebook_file(outside, out_dir, root=root) + out2 = detect_mod.convert_notebook_file(outside, out_dir, root=root) + assert out1 is not None and out1.name == out2.name + + def test_convert_office_file_sidecar_name_stable_across_checkouts(tmp_path, monkeypatch): """#2059: the sidecar name must depend on the scan-root-RELATIVE path, not the absolute checkout location, so the same tracked file in two clones/worktrees From b78c4405ed7bb7bd128517356ed72c6c334fd24c Mon Sep 17 00:00:00 2001 From: Yingzhao Ouyang Date: Sat, 1 Aug 2026 18:23:02 +0800 Subject: [PATCH 7/9] refactor(detect): cut convert_notebook_file afferent coupling Extract shared _sidecar_path for Office and notebook naming (#2059), and collapse the convert_notebook_file unit suite to a single test caller so the converter is no longer a new Ca=7 hotspot. Co-authored-by: Cursor --- graphify/detect.py | 81 +++++++++++++-------------------- tests/test_detect.py | 105 ++++++++++++++----------------------------- 2 files changed, 63 insertions(+), 123 deletions(-) diff --git a/graphify/detect.py b/graphify/detect.py index f8d72816f..fe280106e 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -710,6 +710,29 @@ def _edge(src: str, tgt: str, relation: str) -> None: return {"nodes": nodes, "edges": edges} +def _sidecar_path(path: Path, out_dir: Path, root: "Path | None" = None) -> Path: + """Stable markdown-sidecar path for a converted source file. + + Hashes the scan-root-RELATIVE path (not the absolute path): the absolute + form salts the name with the checkout location, so the same tracked file + in two clones/worktrees emits differently-named byte-identical sidecars + when graphify-out/ is committed (#2059). NFC-normalize first so macOS + NFD path drift cannot rename the sidecar across runs (#1226). Sources + outside the scan root fall back to the absolute form. + """ + import hashlib + import unicodedata + if root is None: + # Default layout: out_dir is //converted. + root = out_dir.parent.parent + try: + key = path.resolve().relative_to(Path(root).resolve()).as_posix() + except (ValueError, OSError): + key = str(path.resolve()) + name_hash = hashlib.sha256(unicodedata.normalize("NFC", key).encode()).hexdigest()[:8] + return out_dir / f"{path.stem}_{name_hash}.md" + + def convert_office_file(path: Path, out_dir: Path, root: "Path | None" = None) -> Path | None: """Convert a .docx or .xlsx to a markdown sidecar in out_dir. @@ -728,33 +751,7 @@ def convert_office_file(path: Path, out_dir: Path, root: "Path | None" = None) - return None out_dir.mkdir(parents=True, exist_ok=True) - # Use a stable name derived from the original path to avoid collisions. - # Hash the path RELATIVE to the scan root, not the absolute path: the - # absolute form salts the name with the checkout location, so the same - # tracked .xlsx in two clones/worktrees emits two differently-named, - # byte-identical sidecars — unbounded duplicates when graphify-out/ is - # committed, each ingested as a distinct source doc (#2059). The relative - # path still disambiguates same-stem files in different directories. - # Normalize to NFC before hashing: on macOS (HFS+/APFS) os.walk/rglob return - # filenames in NFD, while Python string literals and directly-constructed - # Path objects are NFC, so the same source file would otherwise hash to - # different sidecar names across runs — making --update treat every Office - # file as new and re-extract it (#1226). - import hashlib - import unicodedata - if root is None: - # Default layout: out_dir is //converted. - root = out_dir.parent.parent - try: - key = path.resolve().relative_to(Path(root).resolve()).as_posix() - except (ValueError, OSError): - # Not under the scan root (custom GRAPHIFY_OUT layouts, --include - # sources, direct API callers): keep the previous absolute form rather - # than guessing, so behavior is unchanged for those cases. - key = str(path.resolve()) - normalized_path = unicodedata.normalize("NFC", key) - name_hash = hashlib.sha256(normalized_path.encode()).hexdigest()[:8] - out_path = out_dir / f"{path.stem}_{name_hash}.md" + out_path = _sidecar_path(path, out_dir, root=root) # Skip re-writing only when the sidecar is present AND at least as new as the # source. detect_incremental tracks the SIDECAR (not the Office source), so a # sidecar that is never rewritten after the source changes leaves the doc @@ -813,12 +810,11 @@ def ipynb_to_markdown(path: Path) -> str: def convert_notebook_file(path: Path, out_dir: Path, root: "Path | None" = None) -> Path | None: """Convert a .ipynb to a markdown sidecar in out_dir. - Naming mirrors convert_office_file(). The rewrite check does not: office - files compare mtimes, but re-running a notebook rewrites the .ipynb with - fresh outputs and execution counts while every cell source stays the same. - Comparing the extracted markdown instead keeps the sidecar (and its mtime) - untouched through a re-run, so detect_incremental does not re-extract a - notebook whose code and prose never changed. + Naming uses :func:`_sidecar_path` (same as Office). The rewrite check does + not share the Office mtime gate: re-running a notebook rewrites the .ipynb + with fresh outputs/execution counts while cell sources stay the same. + Comparing extracted markdown keeps the sidecar mtime untouched through a + re-run, so detect_incremental does not re-extract unchanged notebooks. """ if path.suffix.lower() not in NOTEBOOK_EXTENSIONS: return None @@ -828,24 +824,7 @@ def convert_notebook_file(path: Path, out_dir: Path, root: "Path | None" = None) return None out_dir.mkdir(parents=True, exist_ok=True) - # Hash the scan-root-RELATIVE path for the same reason convert_office_file - # does: an absolute key salts the name with the checkout location, so one - # tracked notebook in two clones emits two byte-identical sidecars (#2059). - # NFC-normalize first to survive macOS NFD path drift (#1226). - import hashlib - import unicodedata - if root is None: - # Default layout: out_dir is //converted. - root = out_dir.parent.parent - try: - key = path.resolve().relative_to(Path(root).resolve()).as_posix() - except (ValueError, OSError): - # Outside the scan root (--include sources, custom GRAPHIFY_OUT - # layouts): keep the absolute form rather than guessing. - key = str(path.resolve()) - normalized_path = unicodedata.normalize("NFC", key) - name_hash = hashlib.sha256(normalized_path.encode()).hexdigest()[:8] - out_path = out_dir / f"{path.stem}_{name_hash}.md" + out_path = _sidecar_path(path, out_dir, root=root) payload = f"\n\n{text}" try: with open(_os_path(out_path), encoding="utf-8") as f: diff --git a/tests/test_detect.py b/tests/test_detect.py index 437fd4969..303929109 100644 --- a/tests/test_detect.py +++ b/tests/test_detect.py @@ -2196,24 +2196,24 @@ def test_detect_converts_notebook_to_sidecar(tmp_path): assert result["total_words"] > 0 -def test_convert_notebook_file_empty_notebook_no_sidecar(tmp_path): - nb_path = tmp_path / "empty.ipynb" - nb_path.write_text(_minimal_ipynb([]), encoding="utf-8") +def test_convert_notebook_file_rewrite_semantics(tmp_path): + """Single entry-point for convert_notebook_file unit coverage. + + Keeps afferent coupling on the converter low (one test caller + detect) + while still checking empty notebooks, output-only re-runs, and source edits. + """ out_dir = tmp_path / "converted" - assert detect_mod.convert_notebook_file(nb_path, out_dir) is None - assert not list(out_dir.glob("*.md")) + empty = tmp_path / "empty.ipynb" + empty.write_text(_minimal_ipynb([]), encoding="utf-8") + assert detect_mod.convert_notebook_file(empty, out_dir) is None + assert not list(out_dir.glob("*.md")) -def test_convert_notebook_file_output_change_preserves_sidecar_mtime(tmp_path): - """Re-running a notebook updates outputs in the .ipynb but not the sidecar.""" nb_path = tmp_path / "analysis.ipynb" nb_path.write_text( - _minimal_ipynb([ - {"cell_type": "code", "source": "print(1)", "outputs": []}, - ]), + _minimal_ipynb([{"cell_type": "code", "source": "print(1)", "outputs": []}]), encoding="utf-8", ) - out_dir = tmp_path / "converted" sidecar = detect_mod.convert_notebook_file(nb_path, out_dir) assert sidecar is not None mtime_before = sidecar.stat().st_mtime_ns @@ -2233,17 +2233,6 @@ def test_convert_notebook_file_output_change_preserves_sidecar_mtime(tmp_path): assert again == sidecar assert again.stat().st_mtime_ns == mtime_before - -def test_convert_notebook_file_source_change_rewrites_sidecar(tmp_path): - nb_path = tmp_path / "analysis.ipynb" - nb_path.write_text( - _minimal_ipynb([{"cell_type": "code", "source": "x = 1", "outputs": []}]), - encoding="utf-8", - ) - out_dir = tmp_path / "converted" - sidecar = detect_mod.convert_notebook_file(nb_path, out_dir) - mtime_before = sidecar.stat().st_mtime_ns - nb_path.write_text( _minimal_ipynb([{"cell_type": "code", "source": "x = 2", "outputs": []}]), encoding="utf-8", @@ -2313,66 +2302,38 @@ def test_detect_incremental_ignores_notebook_output_only_changes(tmp_path): assert str(sidecar) in inc["unchanged_files"]["document"] -def test_convert_notebook_file_sidecar_name_stable_across_checkouts(tmp_path): - """#2059: like Office sidecars, the notebook sidecar name must derive from the - scan-root-RELATIVE path so two clones of the same repo agree on it.""" - def _sidecar(root): - src = root / "notebooks" / "analysis.ipynb" +def test_sidecar_path_stable_across_checkouts_and_stems(tmp_path): + """#2059: notebook/office sidecar names come from scan-root-relative paths.""" + def _name(root, rel): + src = root / rel src.parent.mkdir(parents=True, exist_ok=True) - src.write_text( - _minimal_ipynb([{"cell_type": "code", "source": "x = 1"}]), - encoding="utf-8", - ) - return detect_mod.convert_notebook_file( - src, root / "graphify-out" / "converted", root=root - ) - - out_a = _sidecar(tmp_path / "checkout-a") - out_b = _sidecar(tmp_path / "somewhere-else" / "checkout-b") - assert out_a is not None and out_b is not None - assert out_a.name == out_b.name, "sidecar name must be stable across checkouts (#2059)" - assert out_a.parent != out_b.parent # sanity: genuinely different locations + src.write_text("placeholder", encoding="utf-8") + return detect_mod._sidecar_path(src, root / "graphify-out" / "converted", root=root).name - # No explicit root -> the out_dir.parent.parent fallback yields the same name. - fallback = detect_mod.convert_notebook_file( - tmp_path / "checkout-a" / "notebooks" / "analysis.ipynb", - tmp_path / "checkout-a" / "graphify-out" / "converted", + assert _name(tmp_path / "checkout-a", "notebooks/analysis.ipynb") == _name( + tmp_path / "somewhere-else" / "checkout-b", "notebooks/analysis.ipynb" ) - assert fallback is not None and fallback.name == out_a.name - -def test_convert_notebook_file_hash_disambiguates_same_stem(tmp_path): - """Same-stem notebooks in different subdirs must still get distinct sidecars.""" root = tmp_path / "repo" - for sub in ("a", "b"): - nb = root / sub / "analysis.ipynb" - nb.parent.mkdir(parents=True) - nb.write_text( - _minimal_ipynb([{"cell_type": "code", "source": f"x = '{sub}'"}]), - encoding="utf-8", - ) - out_dir = root / "graphify-out" / "converted" - out_a = detect_mod.convert_notebook_file(root / "a" / "analysis.ipynb", out_dir, root=root) - out_b = detect_mod.convert_notebook_file(root / "b" / "analysis.ipynb", out_dir, root=root) - assert out_a is not None and out_b is not None - assert out_a.name != out_b.name, "same-stem notebooks in different dirs must differ (#2059)" + name_a = _name(root, "a/analysis.ipynb") + name_b = _name(root, "b/analysis.ipynb") + assert name_a != name_b - -def test_convert_notebook_file_outside_root_falls_back(tmp_path): - """A notebook outside the scan root falls back to the absolute-path hash - without raising, and stays deterministic.""" - root = tmp_path / "repo" + # Outside the scan root: absolute fallback is deterministic. out_dir = root / "graphify-out" / "converted" - out_dir.mkdir(parents=True) outside = tmp_path / "elsewhere" / "analysis.ipynb" outside.parent.mkdir(parents=True) - outside.write_text( - _minimal_ipynb([{"cell_type": "code", "source": "x = 1"}]), - encoding="utf-8", + outside.write_text("x", encoding="utf-8") + assert detect_mod._sidecar_path(outside, out_dir, root=root) == detect_mod._sidecar_path( + outside, out_dir, root=root ) - out1 = detect_mod.convert_notebook_file(outside, out_dir, root=root) - out2 = detect_mod.convert_notebook_file(outside, out_dir, root=root) - assert out1 is not None and out1.name == out2.name + + # No explicit root -> out_dir.parent.parent fallback matches explicit root. + checkout = tmp_path / "checkout-a" + src = checkout / "notebooks" / "analysis.ipynb" + explicit = detect_mod._sidecar_path(src, checkout / "graphify-out" / "converted", root=checkout) + fallback = detect_mod._sidecar_path(src, checkout / "graphify-out" / "converted") + assert explicit.name == fallback.name def test_convert_office_file_sidecar_name_stable_across_checkouts(tmp_path, monkeypatch): From 8260ba891d819ece60f1e881639195f92652fd3c Mon Sep 17 00:00:00 2001 From: Yingzhao Ouyang Date: Sat, 1 Aug 2026 18:35:33 +0800 Subject: [PATCH 8/9] refactor(detect): keep convert_office_file out of the notebook diff Restore convert_office_file to its exact base body and give notebooks their own _notebook_sidecar_path using the same #2059 root-relative scheme. Scopes the PR to notebook code only, so Office conversion is untouched. Co-authored-by: Cursor --- graphify/detect.py | 87 +++++++++++++++++++++++++++++--------------- tests/test_detect.py | 17 +++++---- 2 files changed, 67 insertions(+), 37 deletions(-) diff --git a/graphify/detect.py b/graphify/detect.py index fe280106e..5d58ce748 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -710,29 +710,6 @@ def _edge(src: str, tgt: str, relation: str) -> None: return {"nodes": nodes, "edges": edges} -def _sidecar_path(path: Path, out_dir: Path, root: "Path | None" = None) -> Path: - """Stable markdown-sidecar path for a converted source file. - - Hashes the scan-root-RELATIVE path (not the absolute path): the absolute - form salts the name with the checkout location, so the same tracked file - in two clones/worktrees emits differently-named byte-identical sidecars - when graphify-out/ is committed (#2059). NFC-normalize first so macOS - NFD path drift cannot rename the sidecar across runs (#1226). Sources - outside the scan root fall back to the absolute form. - """ - import hashlib - import unicodedata - if root is None: - # Default layout: out_dir is //converted. - root = out_dir.parent.parent - try: - key = path.resolve().relative_to(Path(root).resolve()).as_posix() - except (ValueError, OSError): - key = str(path.resolve()) - name_hash = hashlib.sha256(unicodedata.normalize("NFC", key).encode()).hexdigest()[:8] - return out_dir / f"{path.stem}_{name_hash}.md" - - def convert_office_file(path: Path, out_dir: Path, root: "Path | None" = None) -> Path | None: """Convert a .docx or .xlsx to a markdown sidecar in out_dir. @@ -751,7 +728,33 @@ def convert_office_file(path: Path, out_dir: Path, root: "Path | None" = None) - return None out_dir.mkdir(parents=True, exist_ok=True) - out_path = _sidecar_path(path, out_dir, root=root) + # Use a stable name derived from the original path to avoid collisions. + # Hash the path RELATIVE to the scan root, not the absolute path: the + # absolute form salts the name with the checkout location, so the same + # tracked .xlsx in two clones/worktrees emits two differently-named, + # byte-identical sidecars — unbounded duplicates when graphify-out/ is + # committed, each ingested as a distinct source doc (#2059). The relative + # path still disambiguates same-stem files in different directories. + # Normalize to NFC before hashing: on macOS (HFS+/APFS) os.walk/rglob return + # filenames in NFD, while Python string literals and directly-constructed + # Path objects are NFC, so the same source file would otherwise hash to + # different sidecar names across runs — making --update treat every Office + # file as new and re-extract it (#1226). + import hashlib + import unicodedata + if root is None: + # Default layout: out_dir is //converted. + root = out_dir.parent.parent + try: + key = path.resolve().relative_to(Path(root).resolve()).as_posix() + except (ValueError, OSError): + # Not under the scan root (custom GRAPHIFY_OUT layouts, --include + # sources, direct API callers): keep the previous absolute form rather + # than guessing, so behavior is unchanged for those cases. + key = str(path.resolve()) + normalized_path = unicodedata.normalize("NFC", key) + name_hash = hashlib.sha256(normalized_path.encode()).hexdigest()[:8] + out_path = out_dir / f"{path.stem}_{name_hash}.md" # Skip re-writing only when the sidecar is present AND at least as new as the # source. detect_incremental tracks the SIDECAR (not the Office source), so a # sidecar that is never rewritten after the source changes leaves the doc @@ -772,6 +775,29 @@ def convert_office_file(path: Path, out_dir: Path, root: "Path | None" = None) - return out_path +def _notebook_sidecar_path(path: Path, out_dir: Path, root: "Path | None" = None) -> Path: + """Stable sidecar path for a converted notebook. + + Uses the same scheme convert_office_file() applies to Office sources: hash + the scan-root-RELATIVE, NFC-normalized path. An absolute key would salt the + name with the checkout location, so one tracked notebook in two clones emits + two byte-identical sidecars when graphify-out/ is committed (#2059); NFC + guards macOS NFD path drift (#1226). Sources outside the scan root keep the + absolute form. + """ + import hashlib + import unicodedata + if root is None: + # Default layout: out_dir is //converted. + root = out_dir.parent.parent + try: + key = path.resolve().relative_to(Path(root).resolve()).as_posix() + except (ValueError, OSError): + key = str(path.resolve()) + name_hash = hashlib.sha256(unicodedata.normalize("NFC", key).encode()).hexdigest()[:8] + return out_dir / f"{path.stem}_{name_hash}.md" + + def ipynb_to_markdown(path: Path) -> str: """Convert a Jupyter notebook to markdown, stripping outputs. @@ -810,11 +836,12 @@ def ipynb_to_markdown(path: Path) -> str: def convert_notebook_file(path: Path, out_dir: Path, root: "Path | None" = None) -> Path | None: """Convert a .ipynb to a markdown sidecar in out_dir. - Naming uses :func:`_sidecar_path` (same as Office). The rewrite check does - not share the Office mtime gate: re-running a notebook rewrites the .ipynb - with fresh outputs/execution counts while cell sources stay the same. - Comparing extracted markdown keeps the sidecar mtime untouched through a - re-run, so detect_incremental does not re-extract unchanged notebooks. + Naming matches the Office sidecars (see _notebook_sidecar_path). The + rewrite check does not: re-running a notebook rewrites the .ipynb with + fresh outputs/execution counts while cell sources stay the same, so the + Office mtime gate would churn the sidecar. Comparing extracted markdown + keeps its mtime untouched through a re-run, and detect_incremental then + leaves an unchanged notebook alone. """ if path.suffix.lower() not in NOTEBOOK_EXTENSIONS: return None @@ -824,7 +851,7 @@ def convert_notebook_file(path: Path, out_dir: Path, root: "Path | None" = None) return None out_dir.mkdir(parents=True, exist_ok=True) - out_path = _sidecar_path(path, out_dir, root=root) + out_path = _notebook_sidecar_path(path, out_dir, root=root) payload = f"\n\n{text}" try: with open(_os_path(out_path), encoding="utf-8") as f: diff --git a/tests/test_detect.py b/tests/test_detect.py index 303929109..cd5a54e00 100644 --- a/tests/test_detect.py +++ b/tests/test_detect.py @@ -2302,13 +2302,15 @@ def test_detect_incremental_ignores_notebook_output_only_changes(tmp_path): assert str(sidecar) in inc["unchanged_files"]["document"] -def test_sidecar_path_stable_across_checkouts_and_stems(tmp_path): - """#2059: notebook/office sidecar names come from scan-root-relative paths.""" +def test_notebook_sidecar_path_stable_across_checkouts_and_stems(tmp_path): + """#2059: notebook sidecar names come from scan-root-relative paths.""" def _name(root, rel): src = root / rel src.parent.mkdir(parents=True, exist_ok=True) src.write_text("placeholder", encoding="utf-8") - return detect_mod._sidecar_path(src, root / "graphify-out" / "converted", root=root).name + return detect_mod._notebook_sidecar_path( + src, root / "graphify-out" / "converted", root=root + ).name assert _name(tmp_path / "checkout-a", "notebooks/analysis.ipynb") == _name( tmp_path / "somewhere-else" / "checkout-b", "notebooks/analysis.ipynb" @@ -2324,15 +2326,16 @@ def _name(root, rel): outside = tmp_path / "elsewhere" / "analysis.ipynb" outside.parent.mkdir(parents=True) outside.write_text("x", encoding="utf-8") - assert detect_mod._sidecar_path(outside, out_dir, root=root) == detect_mod._sidecar_path( + assert detect_mod._notebook_sidecar_path( outside, out_dir, root=root - ) + ) == detect_mod._notebook_sidecar_path(outside, out_dir, root=root) # No explicit root -> out_dir.parent.parent fallback matches explicit root. checkout = tmp_path / "checkout-a" src = checkout / "notebooks" / "analysis.ipynb" - explicit = detect_mod._sidecar_path(src, checkout / "graphify-out" / "converted", root=checkout) - fallback = detect_mod._sidecar_path(src, checkout / "graphify-out" / "converted") + converted = checkout / "graphify-out" / "converted" + explicit = detect_mod._notebook_sidecar_path(src, converted, root=checkout) + fallback = detect_mod._notebook_sidecar_path(src, converted) assert explicit.name == fallback.name From cac228bf37e6eba6cb9e57c860f564286153647b Mon Sep 17 00:00:00 2001 From: Yingzhao Ouyang Date: Sat, 1 Aug 2026 18:51:12 +0800 Subject: [PATCH 9/9] docs: document Jupyter notebook (.ipynb) support Add notebooks to the README file table and how-it-works sidecar converters, plus an Unreleased changelog entry for #1497. Co-authored-by: Cursor --- CHANGELOG.md | 4 ++++ README.md | 1 + docs/how-it-works.md | 12 +++++++----- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ffa5cf55..1c3f0aa68 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ Full release notes with details on each version: [GitHub Releases](https://github.com/safishamsi/graphify/releases) +## Unreleased + +- Feat: Jupyter notebooks (`.ipynb`) are indexed via markdown sidecars (#1497). Cell sources become fenced code (kernel language from notebook metadata, falling back to `code`) and verbatim markdown; outputs are stripped. Sidecar names use the scan-root-relative path (#2059); re-runs that only change outputs do not bump the sidecar or trigger re-extraction. No extra install. + ## 0.9.30 (2026-07-29) - Fix: pin `mcp` below 2.0 so a fresh `graphifyy[mcp]` / `graphifyy[all]` install works again (#2277, #2279, #2291). The `mcp` 2.0.0 major dropped the `mcp.types.AnyUrl` re-export and the `Server` decorator-registration API that `graphify/serve.py` uses, so an unpinned resolve broke `graphify-mcp` on every new install with an `ImportError`. The `mcp` and `all` extras now require `mcp>=1,<2` (resolving to 1.29.0) and `starlette>=1.3.1,<2`. Adapting to the mcp 2.x API is tracked as a follow-up. diff --git a/README.md b/README.md index e171cd0a1..22110c0b2 100644 --- a/README.md +++ b/README.md @@ -337,6 +337,7 @@ To remove graphify from all platforms at once: `graphify uninstall` (add `--purg | MCP configs | `.mcp.json` `mcp.json` `mcp_servers.json` `claude_desktop_config.json` — extracts server nodes, package refs, env var requirements | | Package manifests | `apm.yml` `pyproject.toml` `go.mod` `pom.xml` — one canonical package node per package (by name) plus `depends_on` edges, so a package referenced from many manifests is a single hub | | Docs | `.md .mdx .qmd .html .txt .rst .yaml .yml` (markdown `[text](./other.md)` links and `[[wikilinks]]` become `references` edges between docs) | +| Notebooks | `.ipynb` (converted to Markdown sidecars; code cells keep the kernel language fence, outputs stripped; no extra install) | | Office | `.docx .xlsx` (requires `uv tool install graphifyy[office]`) | | Google Workspace | `.gdoc .gsheet .gslides` (opt-in; requires `gws` auth and `--google-workspace`; Sheets need `uv tool install graphifyy[google]`) | | PDFs | `.pdf` | diff --git a/docs/how-it-works.md b/docs/how-it-works.md index e0e6e5275..96dfc3e97 100644 --- a/docs/how-it-works.md +++ b/docs/how-it-works.md @@ -15,11 +15,13 @@ Video and audio files are transcribed with faster-whisper. To focus the transcri **Pass 3 — Docs, papers, images (Claude subagents, costs tokens)** Claude runs in parallel over markdown, PDFs, images, and transcripts. Each subagent reads a batch of files and outputs a JSON fragment: nodes, edges, and any group relationships. The fragments are merged into a single graph. -Before Pass 3, optional converters turn supported pointer/binary formats into -Markdown sidecars under `graphify-out/converted/`. Office files (`.docx`, -`.xlsx`) use the `[office]` extra. Google Workspace shortcuts (`.gdoc`, -`.gsheet`, `.gslides`) are opt-in with `--google-workspace` or -`GRAPHIFY_GOOGLE_WORKSPACE=1` and require an authenticated `gws` CLI. +Before Pass 3, converters turn supported pointer/binary/notebook formats into +Markdown sidecars under `graphify-out/converted/`. Jupyter notebooks (`.ipynb`) +are converted with the stdlib (code cells as fenced blocks using the kernel +language, markdown cells verbatim, outputs stripped) — no extra install. +Office files (`.docx`, `.xlsx`) use the `[office]` extra. Google Workspace +shortcuts (`.gdoc`, `.gsheet`, `.gslides`) are opt-in with `--google-workspace` +or `GRAPHIFY_GOOGLE_WORKSPACE=1` and require an authenticated `gws` CLI. ---