feat(detect): index Jupyter notebooks via markdown sidecars

graphify/detect.py

@@ -33,6 +33,9 @@ class FileType(str, Enum):
 PAPER_EXTENSIONS = {'.pdf'}
 IMAGE_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg'}
 OFFICE_EXTENSIONS = {'.docx', '.xlsx'}
+# Notebooks are converted to markdown sidecars before indexing — do NOT add .ipynb
+# to CODE_EXTENSIONS or DOC_EXTENSIONS.
+NOTEBOOK_EXTENSIONS = {'.ipynb'}
 VIDEO_EXTENSIONS = {'.mp4', '.mov', '.webm', '.mkv', '.avi', '.m4v', '.mp3', '.wav', '.m4a', '.ogg'}
 
 CORPUS_WARN_THRESHOLD = 50_000    # words - below this, warn "you may not need a graph"
@@ -517,6 +520,8 @@ def classify_file(path: Path) -> FileType | None:
         return FileType.DOCUMENT
     if ext in OFFICE_EXTENSIONS:
         return FileType.DOCUMENT
+    if ext in NOTEBOOK_EXTENSIONS:
+        return FileType.DOCUMENT
     if ext in GOOGLE_WORKSPACE_EXTENSIONS:
         return FileType.DOCUMENT
     if ext in VIDEO_EXTENSIONS:
@@ -705,6 +710,29 @@ def _edge(src: str, tgt: str, relation: str) -> None:
     return {"nodes": nodes, "edges": edges}
 
 
+def _sidecar_path(path: Path, out_dir: Path, root: "Path | None" = None) -> Path:
+    """Stable markdown-sidecar path for a converted source file.
+
+    Hashes the scan-root-RELATIVE path (not the absolute path): the absolute
+    form salts the name with the checkout location, so the same tracked file
+    in two clones/worktrees emits differently-named byte-identical sidecars
+    when graphify-out/ is committed (#2059). NFC-normalize first so macOS
+    NFD path drift cannot rename the sidecar across runs (#1226). Sources
+    outside the scan root fall back to the absolute form.
+    """
+    import hashlib
+    import unicodedata
+    if root is None:
+        # Default layout: out_dir is <root>/<graphify-out>/converted.
+        root = out_dir.parent.parent
+    try:
+        key = path.resolve().relative_to(Path(root).resolve()).as_posix()
+    except (ValueError, OSError):
+        key = str(path.resolve())
+    name_hash = hashlib.sha256(unicodedata.normalize("NFC", key).encode()).hexdigest()[:8]
+    return out_dir / f"{path.stem}_{name_hash}.md"
+
+
 def convert_office_file(path: Path, out_dir: Path, root: "Path | None" = None) -> Path | None:
     """Convert a .docx or .xlsx to a markdown sidecar in out_dir.
 
@@ -723,33 +751,7 @@ def convert_office_file(path: Path, out_dir: Path, root: "Path | None" = None) -
         return None
 
     out_dir.mkdir(parents=True, exist_ok=True)
-    # Use a stable name derived from the original path to avoid collisions.
-    # Hash the path RELATIVE to the scan root, not the absolute path: the
-    # absolute form salts the name with the checkout location, so the same
-    # tracked .xlsx in two clones/worktrees emits two differently-named,
-    # byte-identical sidecars — unbounded duplicates when graphify-out/ is
-    # committed, each ingested as a distinct source doc (#2059). The relative
-    # path still disambiguates same-stem files in different directories.
-    # Normalize to NFC before hashing: on macOS (HFS+/APFS) os.walk/rglob return
-    # filenames in NFD, while Python string literals and directly-constructed
-    # Path objects are NFC, so the same source file would otherwise hash to
-    # different sidecar names across runs — making --update treat every Office
-    # file as new and re-extract it (#1226).
-    import hashlib
-    import unicodedata
-    if root is None:
-        # Default layout: out_dir is <root>/<graphify-out>/converted.
-        root = out_dir.parent.parent
-    try:
-        key = path.resolve().relative_to(Path(root).resolve()).as_posix()
-    except (ValueError, OSError):
-        # Not under the scan root (custom GRAPHIFY_OUT layouts, --include
-        # sources, direct API callers): keep the previous absolute form rather
-        # than guessing, so behavior is unchanged for those cases.
-        key = str(path.resolve())
-    normalized_path = unicodedata.normalize("NFC", key)
-    name_hash = hashlib.sha256(normalized_path.encode()).hexdigest()[:8]
-    out_path = out_dir / f"{path.stem}_{name_hash}.md"
+    out_path = _sidecar_path(path, out_dir, root=root)
     # Skip re-writing only when the sidecar is present AND at least as new as the
     # source. detect_incremental tracks the SIDECAR (not the Office source), so a
     # sidecar that is never rewritten after the source changes leaves the doc
@@ -770,6 +772,70 @@ def convert_office_file(path: Path, out_dir: Path, root: "Path | None" = None) -
     return out_path
 
 
+def ipynb_to_markdown(path: Path) -> str:
+    """Convert a Jupyter notebook to markdown, stripping outputs.
+
+    Uses the notebook's kernel language from metadata for fenced code blocks,
+    falling back to ``code`` when the metadata is absent.
+    """
+    if not _file_within_size_cap(path):
+        return ""
+    try:
+        nb = json.loads(path.read_text(encoding="utf-8", errors="ignore"))
+        # Resolve the kernel language from notebook metadata so fenced code
+        # blocks use the correct language identifier (e.g. ```python) rather
+        # than the generic ```code fallback.
+        meta = nb.get("metadata", {})
+        lang = (
+            meta.get("language_info", {}).get("name")
+            or meta.get("kernelspec", {}).get("language")
+            or "code"
+        )
+        lines = []
+        for cell in nb.get("cells", []):
+            ct = cell.get("cell_type")
+            raw_src = cell.get("source", [])
+            src = raw_src if isinstance(raw_src, str) else "".join(raw_src)
+            if not src.strip():
+                continue
+            if ct == "markdown":
+                lines.append(src)
+            elif ct == "code":
+                lines.append(f"```{lang}\n{src}\n```")
+        return "\n\n".join(lines)
+    except Exception:
+        return ""
+
+
+def convert_notebook_file(path: Path, out_dir: Path, root: "Path | None" = None) -> Path | None:
+    """Convert a .ipynb to a markdown sidecar in out_dir.
+
+    Naming uses :func:`_sidecar_path` (same as Office). The rewrite check does
+    not share the Office mtime gate: re-running a notebook rewrites the .ipynb
+    with fresh outputs/execution counts while cell sources stay the same.
+    Comparing extracted markdown keeps the sidecar mtime untouched through a
+    re-run, so detect_incremental does not re-extract unchanged notebooks.
+    """
+    if path.suffix.lower() not in NOTEBOOK_EXTENSIONS:
+        return None
+
+    text = ipynb_to_markdown(path)
+    if not text.strip():
+        return None
+
+    out_dir.mkdir(parents=True, exist_ok=True)
+    out_path = _sidecar_path(path, out_dir, root=root)
+    payload = f"<!-- converted from {path.name} -->\n\n{text}"
+    try:
+        with open(_os_path(out_path), encoding="utf-8") as f:
+            if f.read() == payload:
+                return out_path
+    except OSError:
+        pass
+    out_path.write_text(payload, encoding="utf-8")
+    return out_path
+
+
 def count_words(path: Path) -> int:
     try:
         ext = path.suffix.lower()
@@ -1385,6 +1451,17 @@ def _on_walk_error(err: OSError) -> None:
                     # Conversion failed (library not installed) - skip with note
                     skipped_sensitive.append(str(p) + " [office conversion failed - pip install graphifyy[office]]")
                 continue
+            # Notebooks: same sidecar treatment as Office files
+            if p.suffix.lower() in NOTEBOOK_EXTENSIONS:
+                md_path = convert_notebook_file(p, converted_dir, root=root)
+                if md_path:
+                    if _is_ignored(md_path, root, ignore_patterns, _cache=ignore_cache):
+                        continue
+                    files[ftype].append(str(md_path))
+                    total_words += _wc(md_path)
+                else:
+                    skipped_sensitive.append(str(p) + " [notebook conversion failed]")
+                continue
             files[ftype].append(str(p))
             if ftype != FileType.VIDEO:
                 total_words += _wc(p)