diff --git a/README.md b/README.md index b7ca37251..fabc6309a 100644 --- a/README.md +++ b/README.md @@ -175,6 +175,7 @@ Install only what you need: | `sql` | SQL schema extraction | `uv tool install "graphifyy[sql]"` | | `postgres` | Live PostgreSQL introspection (`--postgres DSN`) | `uv tool install "graphifyy[postgres]"` | | `dm` | BYOND DreamMaker `.dm`/`.dme` AST extraction (may need a C compiler + `python3-dev` if no wheel matches your platform) | `uv tool install "graphifyy[dm]"` | +| `commonlisp` | Common Lisp `.lisp`/`.cl`/`.lsp`/`.asd` AST extraction | `uv tool install "graphifyy[commonlisp]"` | | `terraform` | Terraform / HCL `.tf`/`.tfvars`/`.hcl` AST extraction | `uv tool install "graphifyy[terraform]"` | | `chinese` | Chinese query segmentation (jieba) | `uv tool install "graphifyy[chinese]"` | | `all` | Everything above | `uv tool install "graphifyy[all]"` | @@ -237,7 +238,7 @@ To remove graphify from all platforms at once: `graphify uninstall` (add `--purg | Type | Extensions | |------|-----------| -| Code (28 tree-sitter grammars) | `.py .ts .js .jsx .tsx .mjs .go .rs .java .c .cpp .h .hpp .rb .cs .kt .scala .php .swift .lua .luau .zig .ps1 .ex .exs .m .mm .jl .vue .svelte .astro .groovy .gradle .dart .v .sv .svh .sql .f .f90 .f95 .f03 .f08 .pas .pp .dpr .dpk .lpr .inc .dfm .lfm .lpk .sh .bash .json .dm .dme .dmi .dmm .dmf .sln .csproj .fsproj .vbproj .razor .cshtml` (`.dm`/`.dme` requires `uv tool install graphifyy[dm]`) | +| Code (29 tree-sitter grammars) | `.py .ts .js .jsx .tsx .mjs .go .rs .java .c .cpp .h .hpp .rb .cs .kt .scala .php .swift .lua .luau .zig .ps1 .ex .exs .m .mm .jl .vue .svelte .astro .groovy .gradle .dart .v .sv .svh .sql .f .f90 .f95 .f03 .f08 .pas .pp .dpr .dpk .lpr .inc .dfm .lfm .lpk .sh .bash .json .dm .dme .dmi .dmm .dmf .sln .csproj .fsproj .vbproj .razor .cshtml .lisp .cl .lsp .asd` (`.dm`/`.dme` requires `uv tool install graphifyy[dm]`; `.lisp`/`.cl`/`.lsp`/`.asd` requires `uv tool install graphifyy[commonlisp]`) | | Salesforce Apex | `.cls .trigger` (regex-based; classes, interfaces, enums, methods, triggers, SOQL/DML edges) | | Terraform / HCL | `.tf .tfvars .hcl` (requires `uv tool install graphifyy[terraform]`) | | MCP configs | `.mcp.json` `mcp.json` `mcp_servers.json` `claude_desktop_config.json` — extracts server nodes, package refs, env var requirements | diff --git a/graphify/detect.py b/graphify/detect.py index 2eff3f80c..5cb8b3926 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -25,7 +25,7 @@ class FileType(str, Enum): _MANIFEST_PATH = "graphify-out/manifest.json" -CODE_EXTENSIONS = {'.py', '.ts', '.tsx', '.js', '.jsx', '.mjs', '.ejs', '.ets', '.go', '.rs', '.java', '.groovy', '.gradle', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.rb', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.luau', '.toc', '.zig', '.ps1', '.ex', '.exs', '.m', '.mm', '.jl', '.vue', '.svelte', '.astro', '.dart', '.v', '.sv', '.svh', '.sql', '.r', '.f', '.F', '.f90', '.F90', '.f95', '.F95', '.f03', '.F03', '.f08', '.F08', '.pas', '.pp', '.dpr', '.dpk', '.lpr', '.inc', '.dfm', '.lfm', '.lpk', '.sh', '.bash', '.json', '.tf', '.tfvars', '.hcl', '.dm', '.dme', '.dmi', '.dmm', '.dmf', '.sln', '.csproj', '.fsproj', '.vbproj', '.razor', '.cshtml', '.cls', '.trigger'} +CODE_EXTENSIONS = {'.py', '.ts', '.tsx', '.js', '.jsx', '.mjs', '.ejs', '.ets', '.go', '.rs', '.java', '.groovy', '.gradle', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.rb', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.luau', '.toc', '.zig', '.ps1', '.ex', '.exs', '.m', '.mm', '.jl', '.lisp', '.cl', '.lsp', '.asd', '.vue', '.svelte', '.astro', '.dart', '.v', '.sv', '.svh', '.sql', '.r', '.f', '.F', '.f90', '.F90', '.f95', '.F95', '.f03', '.F03', '.f08', '.F08', '.pas', '.pp', '.dpr', '.dpk', '.lpr', '.inc', '.dfm', '.lfm', '.lpk', '.sh', '.bash', '.json', '.tf', '.tfvars', '.hcl', '.dm', '.dme', '.dmi', '.dmm', '.dmf', '.sln', '.csproj', '.fsproj', '.vbproj', '.razor', '.cshtml', '.cls', '.trigger'} DOC_EXTENSIONS = {'.md', '.mdx', '.qmd', '.txt', '.rst', '.html', '.yaml', '.yml'} PAPER_EXTENSIONS = {'.pdf'} IMAGE_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg'} diff --git a/graphify/extract.py b/graphify/extract.py index a40304b07..2d9fd2999 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -7,6 +7,7 @@ import re import sys import unicodedata +import warnings from dataclasses import dataclass, field from pathlib import Path from typing import Any, Callable @@ -10947,6 +10948,516 @@ def _body_of(block): return {"nodes": nodes, "edges": edges} +# ── Common Lisp ────────────────────────────────────────────────────────────── + +# Standard CL definer forms that introduce data/type/variable bindings +# (not callable, no body to walk for calls) +_CL_DATA_DEFINERS = frozenset({ + "defstruct", "deftype", "define-condition", + "defvar", "defparameter", "defconstant", + "define-symbol-macro", +}) + +# Subset of data definers that take a VALUE as their second argument, which +# may itself be a string literal. For these, we must not mistake the value +# for a docstring. +_CL_VALUE_DEFINERS = frozenset({"defvar", "defparameter", "defconstant"}) + +# Standard CL macro-style definers +_CL_MACRO_DEFINERS = frozenset({ + "define-modify-macro", "define-compiler-macro", + "define-setf-expander", "defsetf", +}) + +# Symbols that start with "def" but are NOT definitions (denylist for the +# def-prefix heuristic that catches custom definers like definline-maybe) +_CL_NOT_DEFINERS = frozenset({ + "default", "default-value", "defaults", "define", + "defer", "deferred", "deflate", "deftest", +}) + +# CL special forms / macros that should not count as "calls" in the graph +_CL_SPECIAL_FORMS = frozenset({ + "let", "let*", "flet", "labels", "macrolet", + "if", "when", "unless", "cond", "case", "ecase", "typecase", "etypecase", + "progn", "prog1", "prog2", "block", "return-from", "tagbody", "go", + "lambda", "function", "funcall", "apply", + "setf", "setq", "psetf", "psetq", "incf", "decf", "push", "pop", + "loop", "do", "do*", "dolist", "dotimes", "map", "mapcar", "mapc", + "format", "print", "princ", "prin1", "write", "terpri", + "and", "or", "not", "null", + "values", "multiple-value-bind", "multiple-value-setq", + "declare", "the", "locally", "eval-when", + "quote", "backquote", + "error", "signal", "warn", "assert", "check-type", + "handler-case", "handler-bind", "restart-case", "ignore-errors", + "unwind-protect", "catch", "throw", + "with-open-file", "with-output-to-string", "with-input-from-string", + "with-slots", "with-accessors", + "defvar", "defparameter", "defconstant", + "in-package", "require", "provide", "use-package", + "t", "nil", +}) + + +def extract_commonlisp(path: Path) -> dict: + """Extract packages, classes, functions, methods, macros, and calls from a Common Lisp file.""" + try: + import tree_sitter_commonlisp as tscl + from tree_sitter import Language, Parser + except ImportError: + return {"nodes": [], "edges": [], "error": "tree-sitter-commonlisp not installed"} + + try: + # tree-sitter-commonlisp 0.4.1 (latest) ships the old binding that returns + # an int pointer from language(); tree_sitter.Language only accepts that + # int via a deprecated path. Silence that one warning at the call site + # until the grammar ships a PyCapsule binding. + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", + message="int argument support is deprecated", + category=DeprecationWarning, + ) + language = Language(tscl.language()) + parser = Parser(language) + source = path.read_bytes() + tree = parser.parse(source) + root = tree.root_node + except Exception as e: + return {"nodes": [], "edges": [], "error": str(e)} + + stem = path.stem + str_path = str(path) + nodes: list[dict] = [] + edges: list[dict] = [] + seen_ids: set[str] = set() + function_bodies: list[tuple[str, object]] = [] # (nid, body_nodes) + current_package: str | None = None + + def _text(node) -> str: + return source[node.start_byte:node.end_byte].decode("utf-8", errors="replace") + + # CL symbols can contain operator chars like = < > ? ! + * / that the + # generic _make_id strips. Map them to readable suffixes so upi=, upi<, + # upi> become distinct ids. + _CL_CHAR_MAP = { + '=': '_eq', '<': '_lt', '>': '_gt', '?': '_p', '!': '_bang', + '+': '_plus', '*': '_star', '/': '_slash', '%': '_pct', '&': '_amp', + } + + def _cl_id(*parts: str) -> str: + normalized = [] + for p in parts: + normalized.append(''.join(_CL_CHAR_MAP.get(c, c) for c in p)) + return _make_id(*normalized) + + def add_node(nid: str, label: str, line: int, file_type: str = "code") -> None: + if nid not in seen_ids: + seen_ids.add(nid) + nodes.append({ + "id": nid, + "label": label, + "file_type": file_type, + "source_file": str_path, + "source_location": f"L{line}", + }) + + def add_edge(src: str, tgt: str, relation: str, line: int, + confidence: str = "EXTRACTED", weight: float = 1.0) -> None: + edges.append({ + "source": src, + "target": tgt, + "relation": relation, + "confidence": confidence, + "source_file": str_path, + "source_location": f"L{line}", + "weight": weight, + }) + + file_nid = _cl_id(stem) + add_node(file_nid, path.name, 1) + + def _first_sym(node) -> str | None: + """Get the first sym_lit text from a list_lit's children.""" + for child in node.children: + if child.type == "sym_lit": + return _text(child) + return None + + def _kwd_text(node) -> str: + """Extract the symbol name from a kwd_lit, stripping the leading colon.""" + t = _text(node) + return t.lstrip(":").lstrip("#:") + + def _handle_defpackage(node) -> None: + nonlocal current_package + children = node.children + pkg_name = None + for child in children: + if child.type == "kwd_lit" and pkg_name is None: + pkg_name = _kwd_text(child) + break + if child.type == "sym_lit" and _text(child) != "defpackage": + pkg_name = _text(child) + break + if not pkg_name: + return + current_package = pkg_name + pkg_nid = _cl_id(stem, pkg_name) + line = node.start_point[0] + 1 + add_node(pkg_nid, pkg_name, line) + add_edge(file_nid, pkg_nid, "contains", line) + + # Extract :use clauses as imports + for child in children: + if child.type == "list_lit": + first = _first_sym(child) + if first is None: + # Could be (:use ...) with kwd_lit + for gc in child.children: + if gc.type == "kwd_lit" and _kwd_text(gc) == "use": + for uc in child.children: + if uc.type == "kwd_lit" and uc != gc: + mod_name = _kwd_text(uc) + if mod_name != "use": + tgt_nid = _cl_id(mod_name) + add_edge(pkg_nid, tgt_nid, "imports", + child.start_point[0] + 1) + break + + def _handle_defclass(node) -> None: + children = node.children + # Find class name: second sym_lit after "defclass" + sym_lits = [c for c in children if c.type == "sym_lit"] + if len(sym_lits) < 2: + return + class_name = _text(sym_lits[1]) + line = node.start_point[0] + 1 + class_nid = _cl_id(stem, class_name) + add_node(class_nid, class_name, line) + + parent_nid = file_nid + if current_package: + pkg_nid = _cl_id(stem, current_package) + if pkg_nid in seen_ids: + parent_nid = pkg_nid + add_edge(parent_nid, class_nid, "contains", line) + + # Superclasses (third element, a list) + list_lits = [c for c in children if c.type == "list_lit"] + if list_lits: + superclass_list = list_lits[0] + for sc in superclass_list.children: + if sc.type == "sym_lit": + sc_name = _text(sc) + sc_nid = _cl_id(stem, sc_name) + add_edge(class_nid, sc_nid, "inherits", + superclass_list.start_point[0] + 1) + + def _handle_defun_node(node) -> None: + """Handle a defun AST node (covers defun, defmethod, defgeneric, defmacro).""" + header = None + for child in node.children: + if child.type == "defun_header": + header = child + break + if not header: + return + + # Determine keyword type + keyword_type = "defun" + func_name = None + for child in header.children: + if child.type == "defun_keyword": + for kc in child.children: + if kc.type in ("defun", "defmethod", "defgeneric", "defmacro"): + keyword_type = kc.type + break + if child.type == "sym_lit" and func_name is None: + func_name = _text(child) + + if not func_name: + return + + line = node.start_point[0] + 1 + func_nid = _cl_id(stem, func_name) + + if keyword_type == "defmethod": + label = f".{func_name}()" + elif keyword_type == "defmacro": + label = f"{func_name} (macro)" + elif keyword_type == "defgeneric": + label = f"{func_name} (generic)" + else: + label = f"{func_name}()" + + add_node(func_nid, label, line) + + parent_nid = file_nid + if current_package: + pkg_nid = _cl_id(stem, current_package) + if pkg_nid in seen_ids: + parent_nid = pkg_nid + if keyword_type == "defmethod": + add_edge(parent_nid, func_nid, "method", line) + else: + add_edge(parent_nid, func_nid, "contains", line) + + # Method specializers: (defmethod name ((param class) ...)) + if keyword_type == "defmethod": + for child in header.children: + if child.type == "list_lit": + # This is the parameter list + for param in child.children: + if param.type == "list_lit": + syms = [c for c in param.children if c.type == "sym_lit"] + if len(syms) >= 2: + specializer_name = _text(syms[1]) + spec_nid = _cl_id(stem, specializer_name) + add_edge(func_nid, spec_nid, "specializes", + param.start_point[0] + 1) + break + + # Docstring + for child in node.children: + if child.type == "str_lit": + doc_text = _text(child).strip('"') + if doc_text: + doc_nid = _cl_id(func_nid, "rationale") + add_node(doc_nid, doc_text[:120], child.start_point[0] + 1, file_type="rationale") + add_edge(doc_nid, func_nid, "rationale_for", + child.start_point[0] + 1) + break + + # Collect body for call extraction + body_nodes = [c for c in node.children + if c.type not in ("(", ")", "defun_header", "str_lit")] + if body_nodes: + function_bodies.append((func_nid, body_nodes)) + + def _extract_def_name(node, def_keyword: str) -> str | None: + """Get the name being defined. May be a sym_lit or a list_lit whose + first sym_lit is the name (e.g. (defstruct (foo :conc-name "BAR-") ...)).""" + seen_keyword = False + for child in node.children: + if not seen_keyword: + if child.type == "sym_lit" and _text(child).lower() == def_keyword: + seen_keyword = True + continue + if child.type == "sym_lit": + return _text(child) + if child.type == "list_lit": + for gc in child.children: + if gc.type == "sym_lit": + return _text(gc) + return None + return None + + def _collect_def_body(node, def_keyword: str) -> list: + """Collect body forms from (DEFKEYWORD name (params) [doc] body...). + Skips the def keyword, the name, the params list (if present), and + a leading docstring str_lit.""" + children = [c for c in node.children if c.type not in ("(", ")")] + idx = 0 + # Skip def keyword + if idx < len(children) and children[idx].type == "sym_lit" \ + and _text(children[idx]).lower() == def_keyword: + idx += 1 + # Skip name (sym_lit or list_lit) + if idx < len(children) and children[idx].type in ("sym_lit", "list_lit"): + idx += 1 + # Skip params list (the next list_lit, if any) + if idx < len(children) and children[idx].type == "list_lit": + idx += 1 + # Skip leading docstring + if idx < len(children) and children[idx].type == "str_lit": + idx += 1 + return children[idx:] + + def _handle_def_form(node, def_keyword: str) -> None: + """Handle a generic (DEFKEYWORD name ...) form. Covers standard CL + definers (defstruct, deftype, defvar, define-condition, etc.) and + custom def-prefixed macros (definline, definline-maybe, etc.).""" + name = _extract_def_name(node, def_keyword) + if not name: + return + line = node.start_point[0] + 1 + nid = _cl_id(stem, name) + + kw = def_keyword.lower() + if kw in _CL_DATA_DEFINERS: + label = name + is_callable = False + elif kw in _CL_MACRO_DEFINERS or "macro" in kw: + label = f"{name} (macro)" + is_callable = True + else: + # Custom def-prefixed: assume function-like + label = f"{name}()" + is_callable = True + + add_node(nid, label, line) + + parent_nid = file_nid + if current_package: + pkg_nid = _cl_id(stem, current_package) + if pkg_nid in seen_ids: + parent_nid = pkg_nid + add_edge(parent_nid, nid, "contains", line) + + # Docstring. Skip for defvar/defparameter/defconstant — their second + # argument is a value (possibly a string literal) which would be + # wrongly captured as a docstring. + if kw not in _CL_VALUE_DEFINERS: + for child in node.children: + if child.type == "str_lit": + doc_text = _text(child).strip('"') + if doc_text: + doc_nid = _cl_id(nid, "rationale") + add_node(doc_nid, doc_text[:120], child.start_point[0] + 1, file_type="rationale") + add_edge(doc_nid, nid, "rationale_for", + child.start_point[0] + 1) + break + + if is_callable: + body_nodes = _collect_def_body(node, def_keyword) + if body_nodes: + function_bodies.append((nid, body_nodes)) + + def _is_def_prefixed(sym: str) -> bool: + """Heuristic: does this symbol look like a definition macro?""" + if sym in _CL_NOT_DEFINERS: + return False + return sym.startswith("def") or sym.startswith("define-") + + def _process_form(top) -> bool: + """Dispatch on a single list_lit form. Returns True if it was + recognized as a definition or package directive (caller must NOT + recurse into it). Returns False if unrecognized, so the caller can + recurse to find defs nested inside wrapper macros like + (optimizing ...), (eval-when ...), (progn ...), etc.""" + nonlocal current_package + + # Check for defun node type inside list_lit + for child in top.children: + if child.type == "defun": + _handle_defun_node(child) + return True + + first = _first_sym(top) + if not first: + return False + first_lower = first.lower() + + if first_lower == "defpackage": + _handle_defpackage(top) + return True + if first_lower == "in-package": + for child in top.children: + if child.type == "kwd_lit": + current_package = _kwd_text(child) + break + if child.type == "sym_lit" and _text(child) != "in-package": + current_package = _text(child) + break + return True + if first_lower == "defclass": + _handle_defclass(top) + return True + if first_lower in ("require", "ql:quickload"): + for child in top.children: + if child.type in ("kwd_lit", "str_lit"): + mod_name = _kwd_text(child) if child.type == "kwd_lit" else _text(child).strip('"') + if mod_name: + tgt_nid = _cl_id(mod_name) + add_edge(file_nid, tgt_nid, "imports", + top.start_point[0] + 1) + break + return True + if first_lower in _CL_DATA_DEFINERS or first_lower in _CL_MACRO_DEFINERS: + _handle_def_form(top, first_lower) + return True + if _is_def_prefixed(first_lower): + # Custom definer (definline, definline-maybe, defcomponent, etc.) + _handle_def_form(top, first_lower) + return True + + return False + + def _walk_forms(parent) -> None: + """Walk list_lit children of `parent`, processing each. If a form + isn't recognized, recurse into it — many CL codebases wrap + definitions in macros like (optimizing ...), (eval-when ...), or + (progn ...) that aren't themselves definers but contain defs. + Also descends into reader conditionals (#+feature / #-feature), + which wrap their guarded form in an include_reader_macro node.""" + for top in parent.children: + if top.type == "list_lit": + if not _process_form(top): + _walk_forms(top) + elif top.type == "include_reader_macro": + _walk_forms(top) + + _walk_forms(root) + + # Call extraction pass + label_to_nid: dict[str, str] = {} + for n in nodes: + raw = n["label"] + normalised = raw.replace(" (macro)", "").replace(" (generic)", "").strip("()").lstrip(".") + label_to_nid[normalised.lower()] = n["id"] + + seen_call_pairs: set[tuple[str, str]] = set() + + def walk_calls(node, caller_nid: str) -> None: + if node.type == "defun": + return + if node.type == "list_lit": + callee = _first_sym(node) + if callee and callee.lower() not in _CL_SPECIAL_FORMS: + tgt_nid = label_to_nid.get(callee.lower()) + if tgt_nid and tgt_nid != caller_nid: + pair = (caller_nid, tgt_nid) + if pair not in seen_call_pairs: + seen_call_pairs.add(pair) + add_edge(caller_nid, tgt_nid, "calls", + node.start_point[0] + 1, confidence="EXTRACTED", weight=1.0) + for child in node.children: + walk_calls(child, caller_nid) + + for caller_nid, body_nodes in function_bodies: + for body_node in body_nodes: + walk_calls(body_node, caller_nid) + + clean_edges = [e for e in edges if e["source"] in seen_ids and + (e["target"] in seen_ids or e["relation"] in ("imports", "imports_from"))] + return {"nodes": nodes, "edges": clean_edges} + + +# ── Main extract and collect_files ──────────────────────────────────────────── + + +def _check_tree_sitter_version() -> None: + """Raise a clear error if tree-sitter is too old for the new Language API.""" + try: + from tree_sitter import LANGUAGE_VERSION + except ImportError: + raise ImportError( + "tree-sitter is not installed. Run: pip install 'tree-sitter>=0.23.0'" + ) + # Language API v2 starts at LANGUAGE_VERSION 14 + if LANGUAGE_VERSION < 14: + import tree_sitter as _ts + raise RuntimeError( + f"tree-sitter {getattr(_ts, '__version__', 'unknown')} is too old. " + f"graphify requires tree-sitter >= 0.23.0 (Language API v2). " + f"Run: pip install --upgrade tree-sitter" + ) + + + + _DISPATCH: dict[str, Any] = { ".py": extract_python, ".js": extract_js, @@ -11000,6 +11511,10 @@ def _body_of(block): ".sv": extract_verilog, ".svh": extract_verilog, ".sql": extract_sql, + ".lisp": extract_commonlisp, + ".cl": extract_commonlisp, + ".lsp": extract_commonlisp, + ".asd": extract_commonlisp, ".md": extract_markdown, ".mdx": extract_markdown, ".qmd": extract_markdown, diff --git a/pyproject.toml b/pyproject.toml index 85baa4843..fb11f47ab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,12 +66,13 @@ gemini = ["openai", "tiktoken"] openai = ["openai", "tiktoken"] chinese = ["jieba"] sql = ["tree-sitter-sql"] +commonlisp = ["tree-sitter-commonlisp"] # tree-sitter-dm (BYOND DreamMaker) ships only a Windows wheel, so on Linux/Mac it # must compile from source (needs a C toolchain + python3-dev). Keeping it optional # avoids breaking the default `uv tool install graphifyy` for everyone (#1104). dm = ["tree-sitter-dm"] terraform = ["tree-sitter-hcl"] -all = ["mcp", "neo4j", "pypdf", "markdownify", "watchdog", "graspologic; python_version < '3.13'", "python-docx", "openpyxl", "faster-whisper; python_version >= '3.11'", "yt-dlp", "matplotlib", "numpy>=2.0; python_version >= '3.13'", "openai", "tiktoken", "boto3", "anthropic", "tree-sitter-sql", "jieba", "tree-sitter-dm", "tree-sitter-hcl"] +all = ["mcp", "neo4j", "pypdf", "markdownify", "watchdog", "graspologic; python_version < '3.13'", "python-docx", "openpyxl", "faster-whisper; python_version >= '3.11'", "yt-dlp", "matplotlib", "numpy>=2.0; python_version >= '3.13'", "openai", "tiktoken", "boto3", "anthropic", "tree-sitter-sql", "jieba", "tree-sitter-dm", "tree-sitter-hcl", "tree-sitter-commonlisp"] [project.scripts] graphify = "graphify.__main__:main" diff --git a/tests/fixtures/sample.lisp b/tests/fixtures/sample.lisp new file mode 100644 index 000000000..4dc2dc417 --- /dev/null +++ b/tests/fixtures/sample.lisp @@ -0,0 +1,70 @@ +(defpackage :http-server + (:use :cl :alexandria) + (:export #:make-server #:start #:stop)) + +(in-package :http-server) + +(deftype port-number () '(integer 1 65535)) + +(defstruct request-stats + bytes-in + bytes-out + duration-ms) + +(defstruct (connection (:conc-name conn-)) + id + socket + state) + +(defvar *active-connections* nil) +(defparameter *default-port* 8080) +(defconstant +max-headers+ 100) + +(define-condition server-error (error) + ((reason :initarg :reason :reader error-reason))) + +(defclass server () + ((host :initarg :host :accessor server-host) + (port :initarg :port :accessor server-port) + (handler :initarg :handler :accessor server-handler))) + +(defclass ssl-server (server) + ((cert-path :initarg :cert-path :accessor ssl-cert-path))) + +(defgeneric process-request (server request)) + +(defmethod process-request ((srv server) (req string)) + "Process an incoming HTTP request." + (let ((parsed (parse-headers req))) + (funcall (server-handler srv) parsed))) + +;; Custom definer (Franz-style) — should be picked up by the def-prefix heuristic +(definline-maybe header= (a b) + "Fast header equality." + (string-equal a b)) + +(definline header< (a b) + (string< a b)) + +(defun make-server (host port handler) + "Create a new server instance." + (make-instance 'server :host host :port port :handler handler)) + +(defun start (server) + "Start the server listening on its configured port." + (format t "Starting server on ~a:~a~%" (server-host server) (server-port server)) + (process-request server "GET / HTTP/1.1")) + +(defun stop (server) + (format t "Stopping server~%")) + +(defun compare-headers (h1 h2) + "Compare two headers using the custom definers." + (or (header= h1 h2) (header< h1 h2))) + +(defmacro with-server ((var host port handler) &body body) + "Execute body with a running server bound to var." + `(let ((,var (make-server ,host ,port ,handler))) + (unwind-protect + (progn (start ,var) ,@body) + (stop ,var)))) diff --git a/tests/test_languages.py b/tests/test_languages.py index c8e2acca7..fd676b7f7 100644 --- a/tests/test_languages.py +++ b/tests/test_languages.py @@ -8,7 +8,7 @@ extract_swift, extract_go, extract_julia, extract_js, extract_fortran, extract_groovy, extract_sln, extract_csproj, extract_razor, extract_dm, extract_dmi, extract_dmm, extract_dmf, - extract_powershell, extract_apex, + extract_powershell, extract_apex, extract_commonlisp, ) FIXTURES = Path(__file__).parent / "fixtures" @@ -1628,3 +1628,215 @@ def test_apex_no_dangling_edges(): for e in r["edges"]: assert e["source"] in node_ids, f"dangling source in {fixture}: {e}" assert e["target"] in node_ids, f"dangling target in {fixture}: {e}" + + +# ── Common Lisp ────────────────────────────────────────────────────────────── + +def test_cl_finds_package(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + assert "error" not in r + assert "http-server" in _labels(r) + +def test_cl_finds_class(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + assert "server" in _labels(r) + assert "ssl-server" in _labels(r) + +def test_cl_finds_defun(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + labels = _labels(r) + assert any("make-server" in l for l in labels) + assert any("start" in l for l in labels) + assert any("stop" in l for l in labels) + +def test_cl_finds_generic(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + assert any("process-request" in l for l in _labels(r)) + +def test_cl_finds_macro(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + assert any("with-server" in l and "macro" in l for l in _labels(r)) + +def test_cl_emits_calls(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + calls = _calls(r) + # start() calls process-request + assert any("start" in src and "process-request" in tgt for src, tgt in calls) + # with-server macro calls make-server and start + assert any("with-server" in src and "make-server" in tgt for src, tgt in calls) + assert any("with-server" in src and "start" in tgt for src, tgt in calls) + +def test_cl_calls_are_extracted(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + for e in r["edges"]: + if e["relation"] == "calls": + assert e["confidence"] == "EXTRACTED" + +def test_cl_no_dangling_edges(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + node_ids = {n["id"] for n in r["nodes"]} + for e in r["edges"]: + if e["relation"] in ("contains", "method", "calls"): + assert e["source"] in node_ids + +def test_cl_docstrings(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + rationale_edges = [e for e in r["edges"] if e["relation"] == "rationale_for"] + assert len(rationale_edges) >= 3 # make-server, start, process-request have docstrings + labels = _labels(r) + assert any("Process an incoming" in l for l in labels) + +def test_cl_method_specializers(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + spec_edges = [e for e in r["edges"] if e["relation"] == "specializes"] + assert len(spec_edges) >= 1 + # process-request specializes on server + assert any("process_request" in e["source"] and "server" in e["target"] for e in spec_edges) + +def test_cl_inherits(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + inherit_edges = [e for e in r["edges"] if e["relation"] == "inherits"] + assert len(inherit_edges) >= 1 + assert any("ssl_server" in e["source"] and "server" in e["target"] for e in inherit_edges) + +def test_cl_imports(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + import_edges = [e for e in r["edges"] if e["relation"] == "imports"] + targets = {e["target"] for e in import_edges} + assert "cl" in targets + assert "alexandria" in targets + +def test_cl_finds_deftype(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + assert "port-number" in _labels(r) + +def test_cl_finds_defstruct(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + labels = _labels(r) + assert "request-stats" in labels + # defstruct with options form: (defstruct (name ...) ...) + assert "connection" in labels + +def test_cl_finds_defvar_defparameter_defconstant(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + labels = _labels(r) + assert "*active-connections*" in labels + assert "*default-port*" in labels + assert "+max-headers+" in labels + +def test_cl_finds_define_condition(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + assert "server-error" in _labels(r) + +def test_cl_finds_custom_definer(): + """The def-prefix heuristic should catch definline / definline-maybe.""" + r = extract_commonlisp(FIXTURES / "sample.lisp") + labels = _labels(r) + # definline-maybe and definline should produce function-style nodes + assert "header=()" in labels + assert "header<()" in labels + +def test_cl_custom_definer_in_call_graph(): + """Functions defined via custom definers should appear in the call graph.""" + r = extract_commonlisp(FIXTURES / "sample.lisp") + calls = _calls(r) + # compare-headers calls header= and header< (defined via definline-maybe / definline) + assert any("compare-headers" in src and "header=" in tgt for src, tgt in calls) + assert any("compare-headers" in src and "header<" in tgt for src, tgt in calls) + +def test_cl_operator_names_disambiguated(): + """upi=, upi<, upi> must produce distinct ids (operator chars matter).""" + r = extract_commonlisp(FIXTURES / "sample.lisp") + # header= and header< must have different ids + eq_ids = [n["id"] for n in r["nodes"] if n["label"] == "header=()"] + lt_ids = [n["id"] for n in r["nodes"] if n["label"] == "header<()"] + assert len(eq_ids) == 1 + assert len(lt_ids) == 1 + assert eq_ids[0] != lt_ids[0] + +def test_cl_default_value_not_treated_as_definition(): + """The def-prefix heuristic must not match denylisted symbols.""" + import tempfile + code = "(in-package :cl-user)\n(default-value foo)\n" + with tempfile.NamedTemporaryFile(mode='w', suffix='.lisp', delete=False) as f: + f.write(code) + path = Path(f.name) + try: + r = extract_commonlisp(path) + labels = _labels(r) + # default-value is denylisted, shouldn't create a "foo" node + assert "foo" not in labels + assert "foo()" not in labels + finally: + path.unlink() + +def test_cl_defs_inside_wrapper_macro(): + """Definitions nested inside wrapper macros like (optimizing ...) or + (eval-when ...) must be extracted. Many CL codebases wrap hot-path + inline functions in application-specific macros.""" + import tempfile + code = """(in-package :cl-user) +(optimizing + (definline-maybe packet-type (p) (aref p 0)) + (definline-maybe set-packet-type (p v) (setf (aref p 0) v))) +(eval-when (:compile-toplevel :load-toplevel :execute) + (defun helper () 42)) +(progn + (defun progn-def () 'ok)) +""" + with tempfile.NamedTemporaryFile(mode='w', suffix='.lisp', delete=False) as f: + f.write(code) + path = Path(f.name) + try: + r = extract_commonlisp(path) + labels = _labels(r) + assert "packet-type()" in labels + assert "set-packet-type()" in labels + assert "helper()" in labels + assert "progn-def()" in labels + finally: + path.unlink() + +def test_cl_defs_inside_reader_conditional(): + """#+feature / #-feature reader conditionals wrap their guarded form + in an include_reader_macro AST node, which the walker must descend + into to find the nested definition.""" + import tempfile + code = """(in-package :cl-user) +#+little-endian +(definline-maybe byte-hash= (a b) (eq a b)) +#-sbcl +(defun only-on-non-sbcl () 1) +""" + with tempfile.NamedTemporaryFile(mode='w', suffix='.lisp', delete=False) as f: + f.write(code) + path = Path(f.name) + try: + r = extract_commonlisp(path) + labels = _labels(r) + assert "byte-hash=()" in labels + assert "only-on-non-sbcl()" in labels + finally: + path.unlink() + +def test_cl_defparameter_string_value_not_docstring(): + """For defvar/defparameter/defconstant, a string literal in the VALUE + position must not be wrongly captured as a docstring node.""" + import tempfile + code = '''(in-package :cl-user) +(defparameter *config-path* "/etc/app/config") +(defvar *greeting* "hello world") +''' + with tempfile.NamedTemporaryFile(mode='w', suffix='.lisp', delete=False) as f: + f.write(code) + path = Path(f.name) + try: + r = extract_commonlisp(path) + labels = _labels(r) + assert "*config-path*" in labels + assert "*greeting*" in labels + # The string VALUES must not show up as rationale nodes + assert not any("/etc/app/config" in l for l in labels) + assert not any("hello world" in l for l in labels) + finally: + path.unlink()