From ee6dc0c9944262c1be371e7de5c96d2eed722688 Mon Sep 17 00:00:00 2001 From: Mohammed Hafiz <65387738+Hafiz408@users.noreply.github.com> Date: Fri, 26 Jun 2026 19:08:04 +0530 Subject: [PATCH 01/17] feat(crossrepo): skill scaffold + repo selection + output dir Co-Authored-By: Claude Sonnet 4.6 --- .../skills/understand-crossrepo/SKILL.md | 206 ++++++++++++++++++ 1 file changed, 206 insertions(+) create mode 100644 understand-anything-plugin/skills/understand-crossrepo/SKILL.md diff --git a/understand-anything-plugin/skills/understand-crossrepo/SKILL.md b/understand-anything-plugin/skills/understand-crossrepo/SKILL.md new file mode 100644 index 000000000..9c8acc1d8 --- /dev/null +++ b/understand-anything-plugin/skills/understand-crossrepo/SKILL.md @@ -0,0 +1,206 @@ +--- +name: understand-crossrepo +description: Analyze multiple interlinked microservice repos and build one combined knowledge graph — each repo a layer, with typed cross-repo edges — explorable in the existing dashboard +argument-hint: "[repoA repoB ...] [--out ]" +--- + +# /understand-crossrepo + +Analyze two or more related repositories together and produce a single `crossrepo-knowledge-graph.json` in a shared output directory. Each repo becomes a named layer; cross-repo edges capture the real runtime dependencies between services that single-repo graphs cannot express. + +## Options + +- `$ARGUMENTS` may contain: + - One or more repo paths (non-flag tokens) — each treated as a repo to include. Paths may be absolute or relative to the current working directory. + - `--out ` — write the combined graph to this directory instead of the default. + +--- + +## Progress Reporting + +Use the same conventions as `/understand`: + +- **Phase transitions:** `[Phase N/M] ...` +- **Phase completion:** `Phase N complete. .` + +--- + +## Phase 0 — Pre-flight: plugin root, repo selection, output dir + +### Step 0.1 — Resolve PLUGIN_ROOT and ensure core is built + +Do **not** assume the plugin root is simply two directories above the skill path string. In many installations `~/.agents/skills/understand-crossrepo` is a symlink into the real plugin checkout. Prefer runtime-provided plugin roots first, then fall back to universal symlinks, skill symlink resolution, and common clone-based install paths. + +```bash +SKILL_REAL=$(realpath ~/.agents/skills/understand-crossrepo 2>/dev/null || readlink -f ~/.agents/skills/understand-crossrepo 2>/dev/null || echo "") +SELF_RELATIVE=$([ -n "$SKILL_REAL" ] && cd "$SKILL_REAL/../.." 2>/dev/null && pwd || echo "") +COPILOT_SKILL_REAL=$(realpath ~/.copilot/skills/understand-crossrepo 2>/dev/null || readlink -f ~/.copilot/skills/understand-crossrepo 2>/dev/null || echo "") +COPILOT_SELF_RELATIVE=$([ -n "$COPILOT_SKILL_REAL" ] && cd "$COPILOT_SKILL_REAL/../.." 2>/dev/null && pwd || echo "") + +PLUGIN_ROOT="" +for candidate in \ + "${CLAUDE_PLUGIN_ROOT}" \ + "$HOME/.understand-anything-plugin" \ + "$SELF_RELATIVE" \ + "$COPILOT_SELF_RELATIVE" \ + "$HOME/.codex/understand-anything/understand-anything-plugin" \ + "$HOME/.opencode/understand-anything/understand-anything-plugin" \ + "$HOME/.pi/understand-anything/understand-anything-plugin" \ + "$HOME/understand-anything/understand-anything-plugin"; do + if [ -n "$candidate" ] && [ -f "$candidate/package.json" ] && [ -f "$candidate/pnpm-workspace.yaml" ]; then + PLUGIN_ROOT="$candidate" + break + fi +done + +if [ -z "$PLUGIN_ROOT" ]; then + echo "Error: Cannot find the understand-anything plugin root." + echo "Checked:" + echo " - ${CLAUDE_PLUGIN_ROOT:-}" + echo " - $HOME/.understand-anything-plugin" + echo " - ${SELF_RELATIVE:-}" + echo " - ${COPILOT_SELF_RELATIVE:-}" + echo " - $HOME/.codex/understand-anything/understand-anything-plugin" + echo " - $HOME/.opencode/understand-anything/understand-anything-plugin" + echo " - $HOME/.pi/understand-anything/understand-anything-plugin" + echo " - $HOME/understand-anything/understand-anything-plugin" + echo "Make sure the plugin is installed correctly." + exit 1 +fi + +if [ ! -f "$PLUGIN_ROOT/packages/core/dist/index.js" ]; then + cd "$PLUGIN_ROOT" && (pnpm install --frozen-lockfile 2>/dev/null || pnpm install) && pnpm --filter @understand-anything/core build +fi +``` + +If `pnpm` is missing, report to the user: "Install Node.js ≥ 22 and pnpm ≥ 10, then re-run `/understand-crossrepo`." + +### Step 0.2 — Repo selection + +Parse `$ARGUMENTS` and collect every non-flag token (anything that does not start with `--`) as a candidate repo path. Also strip `--out ` and its value from the token list before collecting (so `--out` values are never mistaken for repo paths). + +**Path provided:** If one or more non-flag tokens are found: + +1. For each token, resolve it to an absolute path: if the token is relative, resolve it against the current working directory. +2. Verify with `test -d `. If any path does not exist or is not a directory, report an error naming the bad path and **STOP**. +3. Set `$REPO_PATHS` to the list of resolved absolute paths. + +**No paths provided:** Ask the user for a parent directory that contains the repos: + +> "Which parent directory holds the repos you want to analyze together? (e.g. `/workspace/myproject`)" + +Once the user supplies a path (resolve it to absolute if relative): + +```bash +find -maxdepth 1 -type d \( \ + -name ".git" -prune -o \ + -exec sh -c 'test -d "$1/.git" || test -f "$1/package.json" || test -f "$1/pyproject.toml" || test -f "$1/requirements.txt" || test -d "$1/.understand-anything"' _ {} \; -print \ +\) 2>/dev/null | sort +``` + +Present the list to the user as a numbered menu, for example: + +``` +Found these candidate repos under : + 1) savo_gemba_service + 2) savo_gemba_ui + 3) savo_pricing_service + 4) savo_pricing_ui + ... +Enter the numbers to include (e.g. 1 3 4), or "all": +``` + +Wait for the user's reply. Resolve their selection back to absolute paths and set `$REPO_PATHS`. + +**Minimum-repo guard:** If `$REPO_PATHS` contains fewer than 2 paths after resolution, report: + +> "Error: /understand-crossrepo requires at least 2 repos. Please provide 2 or more valid repo paths." + +Then **STOP**. + +### Step 0.3 — Namespace assignment + +Each repo's namespace is its directory basename (e.g. `/workspace/savo_gemba_service` → `savo_gemba_service`). Namespaces are used as the `` segment in node IDs: `:/[:member]`. + +**Collision detection:** If two selected repos share the same basename, disambiguate: + +1. For each colliding repo, compute a 6-character prefix of its SHA-256 (or `md5`) hash: + ```bash + echo -n "" | sha256sum | cut -c1-6 + ``` +2. Append `_` to each colliding basename to form the namespace, e.g. `shared_lib_a1b2c3` and `shared_lib_d4e5f6`. +3. Warn the user: + > "Warning: repos '' and '' share the basename ''. Disambiguated namespaces: '' and ''." + +Store the final `namespace → absolute-path` mapping as `$REPO_NAMESPACES`. + +### Step 0.4 — Output directory setup + +Parse `$ARGUMENTS` for `--out `. If found, resolve it to an absolute path (relative → absolute against cwd) and set `$OUT_DIR` to that value. + +Otherwise, compute the common parent of all paths in `$REPO_PATHS`: + +```bash +# Find longest common directory prefix across all repo paths. +# If repos are siblings (most common case), this is simply their shared parent. +# If they span different trees, use the filesystem root as a fallback. +COMMON_PARENT=$(printf '%s\n' "${REPO_PATHS[@]}" | \ + awk 'BEGIN{FS=OFS="/"} NR==1{n=split($0,a); for(i=1;i<=n;i++) p[i]=a[i]; pn=n} \ + NR>1{n=split($0,a); for(i=1;i<=pn;i++) if(a[i]!=p[i]){pn=i-1; break}} \ + END{for(i=1;i<=pn;i++) printf "%s%s",(i>1?OFS:""),p[i]; print ""}') +[ -z "$COMMON_PARENT" ] && COMMON_PARENT="/" +OUT_DIR="${COMMON_PARENT}/.understand-anything-crossrepo" +``` + +Create the required subdirectories: + +```bash +mkdir -p "$OUT_DIR/.understand-anything/intermediate" +mkdir -p "$OUT_DIR/.understand-anything/tmp" +``` + +If `mkdir` fails (e.g. permission denied), report the error and **STOP**. + +Report to the user: + +> "Output directory: `$OUT_DIR` +> Repos selected (namespace → path):" +> - `` → `` +> - ... + +--- + +## Phases 1–7 (added in later tasks) + +The following phases are scaffolded here for continuity. Their full logic is authored in Tasks 2–6. + +| Phase | Name | Task | +|-------|------|------| +| 1 | Per-repo scan | Task 2 | +| 2 | Per-repo file analysis (parallel) | Task 2 | +| 3 | Cross-repo linker — detect and emit cross-repo edges | Task 3 | +| 4 | Merge into unified graph — one node set, typed cross-repo edges | Task 4 | +| 5 | Architecture + tour over the combined graph | Task 5 | +| 6 | Review + validate the combined graph | Task 5 | +| 7 | Save `crossrepo-knowledge-graph.json` + launch dashboard | Task 6 | + +--- + +## Node ID Convention (reference) + +All nodes use a namespaced ID: `:/[:member]` + +Examples: +- `file:savo_gemba_service/app/models.py` +- `function:savo_gemba_ui/src/api/client.ts:fetchEmployee` +- `endpoint:savo_pricing_service/routes/pricing.py:POST /price` + +Cross-repo edges use standard edge types from the single-repo schema (e.g. `calls`, `depends_on`, `reads_from`) with `source` and `target` IDs from different namespaces. + +--- + +## Error Handling + +- Report all errors to the user immediately. Never silently continue after a STOP condition. +- STOP conditions in Phase 0: missing repo path, fewer than 2 repos, `mkdir` failure. +- Non-STOP warnings (namespace collisions, individual-repo scan failures in later phases) are collected in `$PHASE_WARNINGS` and included in the final report. From c6e348c62dc8ef0c2749d1a152b865102a5be487 Mon Sep 17 00:00:00 2001 From: Mohammed Hafiz <65387738+Hafiz408@users.noreply.github.com> Date: Fri, 26 Jun 2026 19:17:10 +0530 Subject: [PATCH 02/17] feat(crossrepo): deterministic cross-repo signal extractor + tests --- .../__tests__/extract-signals.test.mjs | 212 ++++++ .../extract-crossrepo-signals.mjs | 628 ++++++++++++++++++ 2 files changed, 840 insertions(+) create mode 100644 understand-anything-plugin/skills/understand-crossrepo/__tests__/extract-signals.test.mjs create mode 100644 understand-anything-plugin/skills/understand-crossrepo/extract-crossrepo-signals.mjs diff --git a/understand-anything-plugin/skills/understand-crossrepo/__tests__/extract-signals.test.mjs b/understand-anything-plugin/skills/understand-crossrepo/__tests__/extract-signals.test.mjs new file mode 100644 index 000000000..252007830 --- /dev/null +++ b/understand-anything-plugin/skills/understand-crossrepo/__tests__/extract-signals.test.mjs @@ -0,0 +1,212 @@ +/** + * TDD test for extract-crossrepo-signals.mjs + * Uses node:test + node:assert (stdlib only, no vitest/jest). + * Gate: node --test skills/understand-crossrepo/__tests__/extract-signals.test.mjs + */ + +import { test, after } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, dirname, resolve } from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const SCRIPT = resolve(__dirname, '..', 'extract-crossrepo-signals.mjs'); + +// Track temp dirs for cleanup +const tempDirs = []; + +after(() => { + for (const d of tempDirs) { + try { rmSync(d, { recursive: true, force: true }); } catch {} + } +}); + +/** Build a fixture repo and run the extractor. Returns parsed JSON output. */ +function runExtractor(files, namespace = 'test_service') { + const repoDir = mkdtempSync(join(tmpdir(), 'ua-crossrepo-test-')); + tempDirs.push(repoDir); + + for (const [relPath, contents] of Object.entries(files)) { + const abs = join(repoDir, relPath); + mkdirSync(dirname(abs), { recursive: true }); + writeFileSync(abs, contents, 'utf-8'); + } + + const outDir = mkdtempSync(join(tmpdir(), 'ua-crossrepo-out-')); + tempDirs.push(outDir); + const outPath = join(outDir, 'signals.json'); + + const result = spawnSync('node', [SCRIPT, repoDir, namespace, outPath], { + encoding: 'utf-8', + }); + + if (result.status !== 0) { + throw new Error(`Extractor exited ${result.status}:\nstdout: ${result.stdout}\nstderr: ${result.stderr}`); + } + + return JSON.parse(readFileSync(outPath, 'utf-8')); +} + +// --------------------------------------------------------------------------- +// Fixture: a fake pricing service repo +// --------------------------------------------------------------------------- +const FIXTURE_FILES = { + // Env file with outbound API signal + '.env': [ + 'BRIDGE_API_URL=https://bridge.example/api', + 'DB_HOST=my-db.internal', + 'SOME_VAR=not_a_url', + ].join('\n'), + + // Keycloak / auth config + 'config/keycloak.json': JSON.stringify({ + realm: 'savo', + 'auth-server-url': 'https://auth.example/auth', + 'ssl-required': 'external', + resource: 'savo_pricing', + 'public-client': true, + }), + + // Python route file with FastAPI decorator + 'routes/pricing.py': [ + 'from fastapi import APIRouter', + 'router = APIRouter(prefix="/api/pricing")', + '', + '@router.get("/items")', + 'async def list_items():', + ' pass', + '', + '@router.post("/submit")', + 'async def submit_price():', + ' pass', + ].join('\n'), + + // pyproject.toml for service name + 'pyproject.toml': [ + '[tool.poetry]', + 'name = "savo-pricing-service"', + ].join('\n'), +}; + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +test('output has required top-level fields', () => { + const out = runExtractor(FIXTURE_FILES, 'savo_pricing_service'); + assert.ok(typeof out.repo === 'string', 'repo field missing'); + assert.ok(typeof out.namespace === 'string', 'namespace field missing'); + assert.ok(typeof out.identity === 'object', 'identity field missing'); + assert.ok(Array.isArray(out.outbound), 'outbound must be array'); +}); + +test('namespace is echoed verbatim', () => { + const out = runExtractor(FIXTURE_FILES, 'savo_pricing_service'); + assert.equal(out.namespace, 'savo_pricing_service'); +}); + +test('repo is the basename of repoPath', () => { + const out = runExtractor(FIXTURE_FILES, 'savo_pricing_service'); + // The temp dir basename starts with "ua-crossrepo-test-" — we just check it's a non-empty string + assert.ok(out.repo.length > 0, 'repo should be non-empty basename'); +}); + +test('BRIDGE_API_URL env var produces outbound api signal', () => { + const out = runExtractor(FIXTURE_FILES, 'savo_pricing_service'); + const apiSignals = out.outbound.filter(s => s.kind === 'api'); + assert.ok(apiSignals.length > 0, 'expected at least one outbound api signal'); + + // Find the BRIDGE signal specifically + const bridgeSignal = apiSignals.find(s => s.value.includes('BRIDGE_API_URL')); + assert.ok(bridgeSignal, `no BRIDGE_API_URL signal — got: ${JSON.stringify(apiSignals)}`); + + // evidence must be file:line (relative path, no leading slash) + assert.match(bridgeSignal.evidence, /^[^/][^:]*:\d+$/, 'evidence must be relpath:line'); + assert.ok(bridgeSignal.evidence.includes('.env'), 'evidence must reference .env'); +}); + +test('Keycloak config produces outbound auth signal', () => { + const out = runExtractor(FIXTURE_FILES, 'savo_pricing_service'); + const authSignals = out.outbound.filter(s => s.kind === 'auth'); + assert.ok(authSignals.length > 0, `no auth signals — outbound: ${JSON.stringify(out.outbound)}`); + + const kc = authSignals.find(s => s.value.includes('savo')); + assert.ok(kc, `no keycloak realm signal — got: ${JSON.stringify(authSignals)}`); +}); + +test('identity.serviceName extracted from pyproject.toml', () => { + const out = runExtractor(FIXTURE_FILES, 'savo_pricing_service'); + assert.ok( + out.identity.serviceName && out.identity.serviceName.length > 0, + 'serviceName must be non-empty', + ); + // Should come from pyproject.toml name field + assert.ok( + out.identity.serviceName.includes('pricing') || out.identity.serviceName.includes('savo'), + `unexpected serviceName: ${out.identity.serviceName}`, + ); +}); + +test('identity.keycloakClientId extracted from keycloak.json', () => { + const out = runExtractor(FIXTURE_FILES, 'savo_pricing_service'); + // keycloak.json has resource = "savo_pricing" + assert.equal(out.identity.keycloakClientId, 'savo_pricing'); +}); + +test('identity.endpoints contains FastAPI route paths', () => { + const out = runExtractor(FIXTURE_FILES, 'savo_pricing_service'); + assert.ok(Array.isArray(out.identity.endpoints), 'endpoints must be array'); + assert.ok(out.identity.endpoints.length > 0, 'expected at least one endpoint'); + // Should find /items or /submit (possibly with prefix) + const found = out.identity.endpoints.some(e => e.includes('/items') || e.includes('/submit')); + assert.ok(found, `no expected endpoint — got: ${JSON.stringify(out.identity.endpoints)}`); +}); + +test('evidence paths are relative (no leading slash)', () => { + const out = runExtractor(FIXTURE_FILES, 'savo_pricing_service'); + for (const sig of out.outbound) { + assert.ok(!sig.evidence.startsWith('/'), `evidence is absolute: ${sig.evidence}`); + } +}); + +test('outbound is sorted (stable)', () => { + const out1 = runExtractor(FIXTURE_FILES, 'savo_pricing_service'); + const out2 = runExtractor(FIXTURE_FILES, 'savo_pricing_service'); + assert.deepEqual(out1.outbound, out2.outbound, 'outbound must be deterministically ordered'); +}); + +test('identity arrays are sorted (stable)', () => { + const out1 = runExtractor(FIXTURE_FILES, 'savo_pricing_service'); + const out2 = runExtractor(FIXTURE_FILES, 'savo_pricing_service'); + assert.deepEqual(out1.identity.endpoints, out2.identity.endpoints); + assert.deepEqual(out1.identity.servedHosts, out2.identity.servedHosts); +}); + +test('identity has required array fields', () => { + const out = runExtractor(FIXTURE_FILES, 'savo_pricing_service'); + assert.ok(Array.isArray(out.identity.servedHosts), 'servedHosts must be array'); + assert.ok(Array.isArray(out.identity.endpoints), 'endpoints must be array'); + assert.ok(Array.isArray(out.identity.ownedTopics), 'ownedTopics must be array'); + assert.ok(Array.isArray(out.identity.ownedBuckets), 'ownedBuckets must be array'); +}); + +test('node_modules and .git are skipped', () => { + const files = { + ...FIXTURE_FILES, + 'node_modules/some-pkg/index.js': 'const FOO_URL = "https://should-not-appear.example";', + '.git/config': 'KEYCLOAK_CLIENT_ID=should-not-appear', + }; + const out = runExtractor(files, 'savo_pricing_service'); + const allEvidence = out.outbound.map(s => s.evidence); + assert.ok( + !allEvidence.some(e => e.startsWith('node_modules')), + 'node_modules must be skipped', + ); + assert.ok( + !allEvidence.some(e => e.startsWith('.git')), + '.git must be skipped', + ); +}); diff --git a/understand-anything-plugin/skills/understand-crossrepo/extract-crossrepo-signals.mjs b/understand-anything-plugin/skills/understand-crossrepo/extract-crossrepo-signals.mjs new file mode 100644 index 000000000..2ede62d74 --- /dev/null +++ b/understand-anything-plugin/skills/understand-crossrepo/extract-crossrepo-signals.mjs @@ -0,0 +1,628 @@ +#!/usr/bin/env node +/** + * extract-crossrepo-signals.mjs + * + * Deterministic per-repo signal scanner for /understand-crossrepo Phase 1. + * Emits a machine-readable JSON evidence file consumed by the Task-4 LLM linker. + * + * Usage: + * node extract-crossrepo-signals.mjs + * + * Output shape (contract for Task 4): + * { + * "repo": "", + * "namespace": "", + * "identity": { + * "serviceName": "...", + * "keycloakClientId": "...|null", + * "servedHosts": [...], + * "endpoints": [...], + * "ownedTopics": [...], + * "ownedBuckets": [...] + * }, + * "outbound": [ + * { "kind": "api|auth|embed|pubsub|bucket|db", "value": "...", "evidence": "relpath:line" } + * ] + * } + * + * Stdlib only. No external dependencies. Node ≥ 22 ESM. + */ + +import { readFileSync, readdirSync, statSync, writeFileSync, existsSync } from 'node:fs'; +import { join, basename, relative, extname } from 'node:path'; + +// --------------------------------------------------------------------------- +// Constants +// --------------------------------------------------------------------------- + +const SKIP_DIRS = new Set([ + 'node_modules', '.git', 'dist', 'build', + '.understand-anything', 'venv', '.venv', '__pycache__', +]); + +// Files whose names match (case-insensitive) are scanned for env/config signals +const ENV_FILE_RE = /^\.env(\.|$)/i; +const CONFIG_FILE_RE = /^config\./i; +const VALUES_YAML_RE = /^values.*\.ya?ml$/i; + +// Env key patterns that indicate outbound service URLs +const OUTBOUND_KEY_RE = /_(URL|HOST|ENDPOINT|BASE)$/i; + +// Literal URL pattern in source/config +const LITERAL_URL_RE = /https?:\/\/([a-zA-Z0-9._-]+(?:\/[^\s"'`<>]*)?)/g; + +// FastAPI / Flask route decorators +const PY_ROUTE_RE = /@(?:app|router)\.(get|post|put|patch|delete|options|head)\(["']([^"']+)["']/g; + +// Express routes: router.get('/path') or app.post('/path') +const JS_ROUTE_RE = /(?:router|app)\.(get|post|put|patch|delete|options|head)\(["'`]([^"'`]+)["'`]/g; + +// Router prefix in Python: APIRouter(prefix="...") +const PY_ROUTER_PREFIX_RE = /APIRouter\s*\([^)]*prefix\s*=\s*["']([^"']+)["']/g; + +// Keycloak realm, client_id / clientId / resource +const KC_REALM_RE = /"?realm"?\s*[:=]\s*["']([^"']+)["']/; +const KC_CLIENT_RE = /"?(?:client_id|clientId|resource)"?\s*[:=]\s*["']([^"']+)["']/; +const KC_AUTH_URL_RE = /"?(?:auth-server-url|authServerUrl|issuer)"?\s*[:=]\s*["']([^"']+)["']/; +const KC_CLIENT_ENV_RE = /KEYCLOAK_CLIENT(?:_ID)?\s*=\s*(.+)/i; +const JWKS_RE = /(?:jwks|\.well-known\/openid)/i; + +// Keycloak JSON "resource" field (Keycloak adapter config) +const KC_RESOURCE_RE = /"resource"\s*:\s*"([^"]+)"/; + +// iframe embed +const IFRAME_SRC_RE = /]+src=["']([^"']+)["']/g; +const CT_TOKEN_RE = /\bct_token\b|\bembedded=/; + +// GCS / Pub/Sub +const GCS_BUCKET_RE = /gs:\/\/([a-zA-Z0-9._-]+)/g; +const BUCKET_ENV_RE = /([A-Z][A-Z0-9_]*_BUCKET)\s*=\s*([^\s#]+)/g; +const PUBSUB_TOPIC_RE = /projects\/[^/]+\/topics\/([a-zA-Z0-9._-]+)/g; +const PUBSUB_TOPIC_NAME_RE = /["']([a-zA-Z0-9._-]+-topic[s]?)["']/g; + +// DB host patterns in env/config +const DB_HOST_RE = /(?:DB_HOST|DATABASE_HOST|POSTGRES_HOST|MYSQL_HOST)\s*=\s*([^\s#]+)/gi; +const DB_URL_RE = /(?:DATABASE_URL|DB_URL)\s*=\s*((?:postgres|mysql|mongodb|sqlite)[^:\s]*:\/\/[^\s#]+)/gi; + +// CORS origins in Python/JS +const CORS_ORIGIN_RE = /(?:allow_origins|allowedOrigins|origins)\s*[=:]\s*\[([^\]]+)\]/; + +// Ingress host in helm values YAML +const INGRESS_HOST_RE = /host\s*:\s*(.+)/; + +// pyproject.toml / package.json name +const PYPROJECT_NAME_RE = /^name\s*=\s*["']?([^"'\n]+)["']?/m; +const PACKAGE_JSON_NAME_RE = /"name"\s*:\s*"([^"]+)"/; + +// Max file size to read (2 MB) — avoids hanging on large generated files +const MAX_FILE_BYTES = 2 * 1024 * 1024; + +// Binary extension check +const BINARY_EXTS = new Set([ + '.png', '.jpg', '.jpeg', '.gif', '.svg', '.ico', '.webp', '.bmp', + '.pdf', '.zip', '.gz', '.tar', '.tgz', '.br', '.wasm', + '.ttf', '.woff', '.woff2', '.eot', '.otf', + '.mp3', '.mp4', '.mov', '.avi', '.wav', + '.pyc', '.pyo', '.so', '.dylib', '.dll', '.exe', + '.lock', // package-lock, Cargo.lock — large but rarely signal-bearing +]); + +// --------------------------------------------------------------------------- +// File walker +// --------------------------------------------------------------------------- + +function walkRepo(repoPath) { + const files = []; + + function walk(dir) { + let entries; + try { entries = readdirSync(dir, { withFileTypes: true }); } catch { return; } + // Sort for determinism + entries.sort((a, b) => a.name.localeCompare(b.name)); + for (const entry of entries) { + if (SKIP_DIRS.has(entry.name)) continue; + const abs = join(dir, entry.name); + if (entry.isDirectory()) { + walk(abs); + } else if (entry.isFile()) { + const ext = extname(entry.name).toLowerCase(); + if (BINARY_EXTS.has(ext)) continue; + let size = 0; + try { size = statSync(abs).size; } catch { continue; } + if (size > MAX_FILE_BYTES) continue; + files.push(abs); + } + } + } + + walk(repoPath); + return files; +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function readFile(abs) { + try { return readFileSync(abs, 'utf-8'); } catch { return null; } +} + +/** Return relative path from repoPath, never starting with /. */ +function rel(repoPath, abs) { + return relative(repoPath, abs); +} + +function isEnvFile(name) { return ENV_FILE_RE.test(name) || name === '.env.example'; } +function isConfigFile(name) { return CONFIG_FILE_RE.test(name); } +function isValuesYaml(name) { return VALUES_YAML_RE.test(name); } + +// --------------------------------------------------------------------------- +// Signal collector +// --------------------------------------------------------------------------- + +class Collector { + constructor(repoPath) { + this.repoPath = repoPath; + this.outbound = []; + this.identity = { + serviceName: null, + keycloakClientId: null, + servedHosts: [], + endpoints: [], + ownedTopics: [], + ownedBuckets: [], + }; + // Dedup sets + this._outboundKeys = new Set(); + this._endpointSet = new Set(); + this._hostSet = new Set(); + this._topicSet = new Set(); + this._bucketSet = new Set(); + } + + addOutbound(kind, value, evidence) { + const key = `${kind}|${value}|${evidence}`; + if (this._outboundKeys.has(key)) return; + this._outboundKeys.add(key); + this.outbound.push({ kind, value, evidence }); + } + + addEndpoint(ep) { + if (!ep || this._endpointSet.has(ep)) return; + this._endpointSet.add(ep); + this.identity.endpoints.push(ep); + } + + addHost(host) { + if (!host || this._hostSet.has(host)) return; + this._hostSet.add(host); + this.identity.servedHosts.push(host); + } + + addTopic(topic) { + if (!topic || this._topicSet.has(topic)) return; + this._topicSet.add(topic); + this.identity.ownedTopics.push(topic); + } + + addBucket(bucket) { + if (!bucket || this._bucketSet.has(bucket)) return; + this._bucketSet.add(bucket); + this.identity.ownedBuckets.push(bucket); + } +} + +// --------------------------------------------------------------------------- +// Per-file scanners +// --------------------------------------------------------------------------- + +function lineNumber(content, index) { + return content.slice(0, index).split('\n').length; +} + +/** Scan .env / config files for outbound API signals and DB signals. */ +function scanEnvConfig(content, relPath, collector) { + const lines = content.split('\n'); + lines.forEach((line, i) => { + const lineNum = i + 1; + const evidence = `${relPath}:${lineNum}`; + + // Outbound API: env key ending in _URL/_HOST/_ENDPOINT/_BASE + const envMatch = line.match(/^([A-Z][A-Z0-9_]*)\s*=\s*(.+)/); + if (envMatch) { + const [, key, value] = envMatch; + const val = value.trim(); + if (OUTBOUND_KEY_RE.test(key) && val && !val.startsWith('#')) { + // Exclude local DB host env vars from api signals (they go to db) + if (/^(?:DB_HOST|DATABASE_HOST|POSTGRES_HOST|MYSQL_HOST)$/i.test(key)) { + collector.addOutbound('db', `${key}=${val}`, evidence); + } else { + collector.addOutbound('api', `${key}=${val}`, evidence); + } + } + } + + // DB via DATABASE_URL + { + const re = /^(DATABASE_URL|DB_URL)\s*=\s*(.+)/i; + const m = line.match(re); + if (m) { + collector.addOutbound('db', `${m[1]}=${m[2].trim()}`, evidence); + } + } + + // Keycloak env var + { + const m = line.match(KC_CLIENT_ENV_RE); + if (m) { + const client = m[1].trim(); + collector.addOutbound('auth', `KEYCLOAK_CLIENT=${client}`, evidence); + if (!collector.identity.keycloakClientId) { + collector.identity.keycloakClientId = client; + } + } + } + + // GCS bucket env + { + let m; + const re = /([A-Z][A-Z0-9_]*_BUCKET)\s*=\s*([^\s#]+)/gi; + while ((m = re.exec(line)) !== null) { + collector.addBucket(m[2].trim()); + collector.addOutbound('bucket', `${m[1]}=${m[2].trim()}`, evidence); + } + } + + // GCS gs:// URIs + { + let m; + const re = /gs:\/\/([a-zA-Z0-9._-]+)/g; + while ((m = re.exec(line)) !== null) { + collector.addBucket(m[1]); + collector.addOutbound('bucket', `gs://${m[1]}`, evidence); + } + } + + // Pub/Sub topic names + { + let m; + const re = /projects\/[^/]+\/topics\/([a-zA-Z0-9._-]+)/g; + while ((m = re.exec(line)) !== null) { + collector.addTopic(m[1]); + collector.addOutbound('pubsub', m[0], evidence); + } + } + }); +} + +/** Scan any file for literal https?:// URLs (outbound api). */ +function scanLiteralUrls(content, relPath, collector) { + // Exclude localhost / 127.0.0.1 — internal dev references + const LOCAL_HOST_RE = /^(localhost|127\.0\.0\.1|0\.0\.0\.0)/; + let m; + const re = /https?:\/\/([a-zA-Z0-9._-]+(?::\d+)?)/g; + while ((m = re.exec(content)) !== null) { + const host = m[1]; + if (LOCAL_HOST_RE.test(host)) continue; + const line = lineNumber(content, m.index); + collector.addOutbound('api', `https://${host}`, `${relPath}:${line}`); + } +} + +/** Scan Keycloak adapter JSON or env for auth signals. */ +function scanKeycloak(content, relPath, collector) { + // Try JSON parse first (keycloak.json adapter config) + try { + const json = JSON.parse(content); + const realm = json.realm; + const client = json.resource || json['client-id'] || json.clientId || json.client_id; + const authUrl = json['auth-server-url'] || json.authServerUrl || json.issuer; + + if (realm || client) { + const value = [realm && `realm=${realm}`, client && `client=${client}`].filter(Boolean).join(' '); + collector.addOutbound('auth', value, relPath); + // Set identity keycloakClientId from "resource" (Keycloak adapter convention) + if (client && !collector.identity.keycloakClientId) { + collector.identity.keycloakClientId = client; + } + } + if (authUrl) { + collector.addOutbound('auth', `issuer=${authUrl}`, relPath); + } + return; // JSON handled — no need for regex pass + } catch { + // Not JSON — fall through to regex + } + + // Regex pass for YAML / Python dicts / .env + { + const realmMatch = content.match(KC_REALM_RE); + const clientMatch = content.match(KC_CLIENT_RE); + const authUrlMatch = content.match(KC_AUTH_URL_RE); + + if (realmMatch || clientMatch) { + const realm = realmMatch?.[1]; + const client = clientMatch?.[1]; + const value = [realm && `realm=${realm}`, client && `client=${client}`].filter(Boolean).join(' '); + if (value) { + // Find best evidence line + const lines = content.split('\n'); + let evidenceLine = 1; + for (let i = 0; i < lines.length; i++) { + if ((realm && lines[i].includes(realm)) || (client && lines[i].includes(client))) { + evidenceLine = i + 1; + break; + } + } + collector.addOutbound('auth', value, `${relPath}:${evidenceLine}`); + if (client && !collector.identity.keycloakClientId) { + collector.identity.keycloakClientId = client; + } + } + } + if (authUrlMatch) { + collector.addOutbound('auth', `issuer=${authUrlMatch[1]}`, relPath); + } + } + + // JWKS reference + if (JWKS_RE.test(content)) { + const lines = content.split('\n'); + for (let i = 0; i < lines.length; i++) { + if (JWKS_RE.test(lines[i])) { + collector.addOutbound('auth', `jwks:${lines[i].trim()}`, `${relPath}:${i + 1}`); + break; + } + } + } +} + +/** Scan Python files for route decorators and router prefixes. */ +function scanPythonRoutes(content, relPath, collector) { + // Collect router prefixes declared in this file + const prefixes = []; + { + let m; + const re = /APIRouter\s*\([^)]*prefix\s*=\s*["']([^"']+)["']/g; + while ((m = re.exec(content)) !== null) { + prefixes.push(m[1]); + } + } + + const prefix = prefixes[0] || ''; + + // Route decorators + { + let m; + const re = /@(?:app|router)\.(get|post|put|patch|delete|options|head)\(\s*["']([^"']+)["']/g; + while ((m = re.exec(content)) !== null) { + const path = prefix + m[2]; + const line = lineNumber(content, m.index); + collector.addEndpoint(path); + } + } +} + +/** Scan JS/TS files for Express routes. */ +function scanJsRoutes(content, relPath, collector) { + let m; + const re = /(?:router|app)\.(get|post|put|patch|delete|options|head)\s*\(\s*["'`]([^"'`]+)["'`]/g; + while ((m = re.exec(content)) !== null) { + collector.addEndpoint(m[2]); + } +} + +/** Scan for iframe embeds and ct_token usage. */ +function scanEmbeds(content, relPath, collector) { + let m; + const re = /]+src=["']([^"']+)["']/g; + while ((m = re.exec(content)) !== null) { + const line = lineNumber(content, m.index); + collector.addOutbound('embed', `iframe:${m[1]}`, `${relPath}:${line}`); + } + + if (CT_TOKEN_RE.test(content)) { + const lines = content.split('\n'); + for (let i = 0; i < lines.length; i++) { + if (CT_TOKEN_RE.test(lines[i])) { + collector.addOutbound('embed', `ct_token usage`, `${relPath}:${i + 1}`); + break; + } + } + } +} + +/** Scan for Pub/Sub topic references in code. */ +function scanPubSub(content, relPath, collector) { + let m; + const re = /projects\/[^/\s"']+\/topics\/([a-zA-Z0-9._-]+)/g; + while ((m = re.exec(content)) !== null) { + const line = lineNumber(content, m.index); + collector.addTopic(m[1]); + collector.addOutbound('pubsub', m[0], `${relPath}:${line}`); + } +} + +/** Scan CORS origins from Python / JS config. */ +function scanCors(content, relPath, collector) { + const m = CORS_ORIGIN_RE.exec(content); + if (!m) return; + // Extract quoted strings from the list + const list = m[1]; + const originsRe = /["']([^"']+)["']/g; + let om; + while ((om = originsRe.exec(list)) !== null) { + const origin = om[1]; + if (origin !== '*') collector.addHost(origin); + } +} + +/** Scan Helm values YAML for ingress hosts. */ +function scanHelmValues(content, relPath, collector) { + const lines = content.split('\n'); + let inIngress = false; + for (const line of lines) { + if (/^\s*ingress\s*:/.test(line)) { inIngress = true; } + if (inIngress) { + const m = line.match(/host\s*:\s*(.+)/); + if (m) { + const host = m[1].trim().replace(/["']/g, ''); + if (host) collector.addHost(host); + } + } + } +} + +/** Extract service name from manifest files. */ +function extractServiceName(content, fileName) { + if (fileName === 'pyproject.toml') { + const m = content.match(PYPROJECT_NAME_RE); + return m?.[1]?.trim() || null; + } + if (fileName === 'package.json') { + const m = content.match(PACKAGE_JSON_NAME_RE); + return m?.[1]?.trim() || null; + } + if (fileName === 'setup.py') { + const m = content.match(/name\s*=\s*["']([^"']+)["']/); + return m?.[1]?.trim() || null; + } + return null; +} + +// --------------------------------------------------------------------------- +// Main scan +// --------------------------------------------------------------------------- + +function isKeycloakFile(relPath, content) { + const name = basename(relPath).toLowerCase(); + if (name.includes('keycloak') || name.includes('auth')) return true; + // Heuristic: JSON file with "realm" key + if (relPath.endsWith('.json') && content.includes('"realm"')) return true; + return false; +} + +function scan(repoPath, namespace) { + const collector = new Collector(repoPath); + const files = walkRepo(repoPath); + + for (const absPath of files) { + const relPath = rel(repoPath, absPath); + const name = basename(absPath); + const ext = extname(name).toLowerCase(); + const content = readFile(absPath); + if (content === null) continue; + + // Service name from manifests (first found wins) + if (!collector.identity.serviceName) { + const sn = extractServiceName(content, name); + if (sn) collector.identity.serviceName = sn; + } + + const isPy = ext === '.py'; + const isJs = ext === '.js' || ext === '.ts' || ext === '.jsx' || ext === '.tsx' || ext === '.mjs' || ext === '.cjs'; + const isYaml = ext === '.yaml' || ext === '.yml'; + const isJson = ext === '.json'; + const isHtml = ext === '.html' || ext === '.htm'; + + // Env / config files → outbound api + db + bucket signals + if (isEnvFile(name) || isConfigFile(name)) { + scanEnvConfig(content, relPath, collector); + } + + // Helm values YAML → served hosts + if (isValuesYaml(name) && isYaml) { + scanHelmValues(content, relPath, collector); + } + + // Keycloak adapter files → auth signals + identity.keycloakClientId + if (isKeycloakFile(relPath, content)) { + scanKeycloak(content, relPath, collector); + } + + // Python route decorators → identity.endpoints + if (isPy) { + scanPythonRoutes(content, relPath, collector); + } + + // JS/TS Express routes → identity.endpoints + if (isJs) { + scanJsRoutes(content, relPath, collector); + } + + // iframes + ct_token → outbound embed + if (isJs || isHtml) { + scanEmbeds(content, relPath, collector); + } + + // Pub/Sub topics → outbound pubsub + identity.ownedTopics + scanPubSub(content, relPath, collector); + + // CORS origins → identity.servedHosts + if (isPy || isJs) { + scanCors(content, relPath, collector); + } + + // GCS gs:// in code files → outbound bucket + if (isPy || isJs || isYaml) { + let m; + const re = /gs:\/\/([a-zA-Z0-9._-]+)/g; + while ((m = re.exec(content)) !== null) { + const line = lineNumber(content, m.index); + collector.addBucket(m[1]); + collector.addOutbound('bucket', `gs://${m[1]}`, `${relPath}:${line}`); + } + } + + // Literal URLs in code/config → outbound api + // Only in select file types to avoid noise + if (isPy || isJs || isEnvFile(name) || isConfigFile(name) || isYaml || isJson) { + scanLiteralUrls(content, relPath, collector); + } + } + + // Fallback: serviceName = repo basename + if (!collector.identity.serviceName) { + collector.identity.serviceName = basename(repoPath); + } + + // Sort everything for determinism + collector.outbound.sort((a, b) => { + const ka = `${a.kind}|${a.value}|${a.evidence}`; + const kb = `${b.kind}|${b.value}|${b.evidence}`; + return ka.localeCompare(kb); + }); + collector.identity.endpoints.sort(); + collector.identity.servedHosts.sort(); + collector.identity.ownedTopics.sort(); + collector.identity.ownedBuckets.sort(); + + return { + repo: basename(repoPath), + namespace, + identity: { + serviceName: collector.identity.serviceName, + keycloakClientId: collector.identity.keycloakClientId, + servedHosts: collector.identity.servedHosts, + endpoints: collector.identity.endpoints, + ownedTopics: collector.identity.ownedTopics, + ownedBuckets: collector.identity.ownedBuckets, + }, + outbound: collector.outbound, + }; +} + +// --------------------------------------------------------------------------- +// CLI entry point +// --------------------------------------------------------------------------- + +const [,, repoPath, repoNamespace, outPath] = process.argv; + +if (!repoPath || !repoNamespace || !outPath) { + process.stderr.write('Usage: node extract-crossrepo-signals.mjs \n'); + process.exit(1); +} + +const result = scan(repoPath, repoNamespace); +writeFileSync(outPath, JSON.stringify(result, null, 2), 'utf-8'); +process.stderr.write(`[extract-crossrepo-signals] ${result.outbound.length} outbound signals, ${result.identity.endpoints.length} endpoints → ${outPath}\n`); From 7a9e318f5b83acfa1b443c2731b0ec2735894e32 Mon Sep 17 00:00:00 2001 From: Mohammed Hafiz <65387738+Hafiz408@users.noreply.github.com> Date: Fri, 26 Jun 2026 19:23:59 +0530 Subject: [PATCH 03/17] fix(crossrepo): dedup api signals, catch yaml bucket/topic signals, mark router ceiling Co-Authored-By: Claude Sonnet 4.6 --- .../__tests__/extract-signals.test.mjs | 56 ++++++++++++++ .../extract-crossrepo-signals.mjs | 77 ++++++++++++++++++- 2 files changed, 130 insertions(+), 3 deletions(-) diff --git a/understand-anything-plugin/skills/understand-crossrepo/__tests__/extract-signals.test.mjs b/understand-anything-plugin/skills/understand-crossrepo/__tests__/extract-signals.test.mjs index 252007830..ff667b562 100644 --- a/understand-anything-plugin/skills/understand-crossrepo/__tests__/extract-signals.test.mjs +++ b/understand-anything-plugin/skills/understand-crossrepo/__tests__/extract-signals.test.mjs @@ -210,3 +210,59 @@ test('node_modules and .git are skipped', () => { '.git must be skipped', ); }); + +// Fix 1: .env URL must produce exactly ONE api outbound entry (no bare-host duplicate) +test('Fix 1: .env URL produces exactly one api outbound entry — no bare-host duplicate', () => { + const files = { + '.env': 'BRIDGE_API_URL=https://bridge.example/api\n', + 'pyproject.toml': '[tool.poetry]\nname = "svc"\n', + }; + const out = runExtractor(files, 'test_svc'); + const apiSignals = out.outbound.filter(s => s.kind === 'api' && s.value.includes('bridge.example')); + assert.equal( + apiSignals.length, + 1, + `expected exactly 1 api signal for bridge.example, got ${apiSignals.length}: ${JSON.stringify(apiSignals)}`, + ); + // The single entry must be the structured KEY=value form from scanEnvConfig + assert.ok( + apiSignals[0].value.includes('BRIDGE_API_URL='), + `expected KEY=value form, got: ${apiSignals[0].value}`, + ); +}); + +// Fix 2: values*.yaml bucket/topic signals are captured +test('Fix 2: values-staging.yaml BUCKET and TOPIC keys produce identity and outbound signals', () => { + const files = { + ...FIXTURE_FILES, + 'helm/values-staging.yaml': [ + 'replicaCount: 2', + 'MEDIA_BUCKET: savo-media-staging', + 'EVENTS_TOPIC: savo-events-staging', + 'ingress:', + ' host: staging.example.com', + ].join('\n'), + }; + const out = runExtractor(files, 'savo_pricing_service'); + + // ownedBuckets should include the bucket value + assert.ok( + out.identity.ownedBuckets.includes('savo-media-staging'), + `ownedBuckets missing savo-media-staging: ${JSON.stringify(out.identity.ownedBuckets)}`, + ); + + // ownedTopics should include the topic value + assert.ok( + out.identity.ownedTopics.includes('savo-events-staging'), + `ownedTopics missing savo-events-staging: ${JSON.stringify(out.identity.ownedTopics)}`, + ); + + // outbound should have bucket and pubsub entries with evidence pointing to the yaml file + const bucketSig = out.outbound.find(s => s.kind === 'bucket' && s.value.includes('savo-media-staging')); + assert.ok(bucketSig, `no bucket outbound for savo-media-staging: ${JSON.stringify(out.outbound)}`); + assert.ok(bucketSig.evidence.includes('values-staging.yaml'), `bucket evidence must reference values-staging.yaml: ${bucketSig.evidence}`); + + const pubsubSig = out.outbound.find(s => s.kind === 'pubsub' && s.value.includes('savo-events-staging')); + assert.ok(pubsubSig, `no pubsub outbound for savo-events-staging: ${JSON.stringify(out.outbound)}`); + assert.ok(pubsubSig.evidence.includes('values-staging.yaml'), `pubsub evidence must reference values-staging.yaml: ${pubsubSig.evidence}`); +}); diff --git a/understand-anything-plugin/skills/understand-crossrepo/extract-crossrepo-signals.mjs b/understand-anything-plugin/skills/understand-crossrepo/extract-crossrepo-signals.mjs index 2ede62d74..c7e371683 100644 --- a/understand-anything-plugin/skills/understand-crossrepo/extract-crossrepo-signals.mjs +++ b/understand-anything-plugin/skills/understand-crossrepo/extract-crossrepo-signals.mjs @@ -295,8 +295,15 @@ function scanEnvConfig(content, relPath, collector) { }); } -/** Scan any file for literal https?:// URLs (outbound api). */ +/** Scan any file for literal https?:// URLs (outbound api). + * Skips env/config files — those are already parsed by scanEnvConfig, + * which emits structured KEY=value entries. Running here too would + * duplicate the same target with a bare-host value. */ function scanLiteralUrls(content, relPath, collector) { + const name = basename(relPath); + // ponytail: env/config files emit structured signals via scanEnvConfig; skip here to avoid dup api entries + if (isEnvFile(name) || isConfigFile(name)) return; + // Exclude localhost / 127.0.0.1 — internal dev references const LOCAL_HOST_RE = /^(localhost|127\.0\.0\.1|0\.0\.0\.0)/; let m; @@ -389,6 +396,7 @@ function scanPythonRoutes(content, relPath, collector) { } } + // ponytail: only first APIRouter prefix per file is used; revisit if multi-router files appear in target repos const prefix = prefixes[0] || ''; // Route decorators @@ -457,11 +465,19 @@ function scanCors(content, relPath, collector) { } } -/** Scan Helm values YAML for ingress hosts. */ +/** Scan Helm values YAML for ingress hosts and bucket/topic signals. + * Handles both `KEY: value` (Helm) and K8s `- name: KEY` + `value: ...` forms. */ function scanHelmValues(content, relPath, collector) { const lines = content.split('\n'); let inIngress = false; - for (const line of lines) { + let pendingEnvKey = null; // for K8s - name: X_BUCKET / value: gs://... + + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + const lineNum = i + 1; + const evidence = `${relPath}:${lineNum}`; + + // Ingress host detection if (/^\s*ingress\s*:/.test(line)) { inIngress = true; } if (inIngress) { const m = line.match(/host\s*:\s*(.+)/); @@ -470,6 +486,61 @@ function scanHelmValues(content, relPath, collector) { if (host) collector.addHost(host); } } + + // GCS gs:// URI anywhere in YAML (bucket identity/outbound) + { + let m; + const re = /gs:\/\/([a-zA-Z0-9._-]+)/g; + while ((m = re.exec(line)) !== null) { + const bucket = m[1]; + if (!collector._bucketSet.has(bucket)) { + collector.addBucket(bucket); + collector.addOutbound('bucket', `gs://${bucket}`, evidence); + } + } + } + + // K8s env var form: `- name: X_BUCKET` followed by ` value: ...` + { + const nameMatch = line.match(/^\s*-?\s*name\s*:\s*([A-Z][A-Z0-9_]*_(BUCKET|TOPIC))\s*$/); + if (nameMatch) { + pendingEnvKey = { key: nameMatch[1], kind: nameMatch[2] }; + } else if (pendingEnvKey) { + const valMatch = line.match(/^\s*value\s*:\s*(.+)/); + if (valMatch) { + const val = valMatch[1].trim().replace(/["']/g, ''); + if (pendingEnvKey.kind === 'BUCKET') { + collector.addBucket(val); + collector.addOutbound('bucket', `${pendingEnvKey.key}=${val}`, evidence); + } else { + collector.addTopic(val); + collector.addOutbound('pubsub', `${pendingEnvKey.key}=${val}`, evidence); + } + pendingEnvKey = null; + } else if (!/^\s*#/.test(line) && line.trim() !== '') { + pendingEnvKey = null; // non-value line resets pending key + } + } + } + + // Helm flat form: `X_BUCKET: value` or `X_TOPIC: value` + { + const kvMatch = line.match(/^\s*([A-Z][A-Z0-9_]*_(BUCKET|TOPIC))\s*:\s*(.+)/); + if (kvMatch) { + const key = kvMatch[1]; + const kind = kvMatch[2]; + const val = kvMatch[3].trim().replace(/["']/g, ''); + if (val) { + if (kind === 'BUCKET') { + collector.addBucket(val); + collector.addOutbound('bucket', `${key}=${val}`, evidence); + } else { + collector.addTopic(val); + collector.addOutbound('pubsub', `${key}=${val}`, evidence); + } + } + } + } } } From 7c08df3d2c23f1f28a2c46f661eee81e06d8d77b Mon Sep 17 00:00:00 2001 From: Mohammed Hafiz <65387738+Hafiz408@users.noreply.github.com> Date: Fri, 26 Jun 2026 19:28:29 +0530 Subject: [PATCH 04/17] test(crossrepo): cover k8s two-line yaml signals + mark pendingEnvKey ceiling Co-Authored-By: Claude Sonnet 4.6 --- .../__tests__/extract-signals.test.mjs | 34 +++++++++++++++++++ .../extract-crossrepo-signals.mjs | 1 + 2 files changed, 35 insertions(+) diff --git a/understand-anything-plugin/skills/understand-crossrepo/__tests__/extract-signals.test.mjs b/understand-anything-plugin/skills/understand-crossrepo/__tests__/extract-signals.test.mjs index ff667b562..2a2bdda13 100644 --- a/understand-anything-plugin/skills/understand-crossrepo/__tests__/extract-signals.test.mjs +++ b/understand-anything-plugin/skills/understand-crossrepo/__tests__/extract-signals.test.mjs @@ -266,3 +266,37 @@ test('Fix 2: values-staging.yaml BUCKET and TOPIC keys produce identity and outb assert.ok(pubsubSig, `no pubsub outbound for savo-events-staging: ${JSON.stringify(out.outbound)}`); assert.ok(pubsubSig.evidence.includes('values-staging.yaml'), `pubsub evidence must reference values-staging.yaml: ${pubsubSig.evidence}`); }); + +// Fix 2b: K8s two-line env form (- name: / value:) in values YAML +test('Fix 2b: K8s two-line name/value form in values YAML produces topic and bucket signals', () => { + const files = { + ...FIXTURE_FILES, + 'helm/values-k8s.yaml': [ + 'env:', + ' - name: ORDER_TOPIC', + ' value: orders-topic', + ' - name: ARCHIVE_BUCKET', + ' value: gs://archive-bucket', + ].join('\n'), + }; + const out = runExtractor(files, 'savo_pricing_service'); + + // ownedTopics and ownedBuckets + assert.ok( + out.identity.ownedTopics.includes('orders-topic'), + `ownedTopics missing orders-topic: ${JSON.stringify(out.identity.ownedTopics)}`, + ); + assert.ok( + out.identity.ownedBuckets.some(b => b === 'gs://archive-bucket' || b === 'archive-bucket'), + `ownedBuckets missing archive-bucket: ${JSON.stringify(out.identity.ownedBuckets)}`, + ); + + // outbound pubsub + bucket with evidence pointing at the yaml file + const topicSig = out.outbound.find(s => s.kind === 'pubsub' && s.value.includes('orders-topic')); + assert.ok(topicSig, `no pubsub outbound for orders-topic: ${JSON.stringify(out.outbound)}`); + assert.ok(topicSig.evidence.includes('values-k8s.yaml'), `topic evidence must reference values-k8s.yaml: ${topicSig.evidence}`); + + const bucketSig = out.outbound.find(s => s.kind === 'bucket' && s.value.includes('archive-bucket')); + assert.ok(bucketSig, `no bucket outbound for archive-bucket: ${JSON.stringify(out.outbound)}`); + assert.ok(bucketSig.evidence.includes('values-k8s.yaml'), `bucket evidence must reference values-k8s.yaml: ${bucketSig.evidence}`); +}); diff --git a/understand-anything-plugin/skills/understand-crossrepo/extract-crossrepo-signals.mjs b/understand-anything-plugin/skills/understand-crossrepo/extract-crossrepo-signals.mjs index c7e371683..6d35609c3 100644 --- a/understand-anything-plugin/skills/understand-crossrepo/extract-crossrepo-signals.mjs +++ b/understand-anything-plugin/skills/understand-crossrepo/extract-crossrepo-signals.mjs @@ -501,6 +501,7 @@ function scanHelmValues(content, relPath, collector) { } // K8s env var form: `- name: X_BUCKET` followed by ` value: ...` + // ponytail: pendingEnvKey can mis-pair if a bare 'value:' appears at document level between name/value stanzas in freeform yaml; acceptable for values.yaml signal extraction { const nameMatch = line.match(/^\s*-?\s*name\s*:\s*([A-Z][A-Z0-9_]*_(BUCKET|TOPIC))\s*$/); if (nameMatch) { From be5ab70464e72284203fb1d638f933d2690f7ad3 Mon Sep 17 00:00:00 2001 From: Mohammed Hafiz <65387738+Hafiz408@users.noreply.github.com> Date: Fri, 26 Jun 2026 19:35:07 +0530 Subject: [PATCH 05/17] feat(crossrepo): combine graphs with per-repo namespacing + layering Co-Authored-By: Claude Sonnet 4.6 --- .../__tests__/test_combine.py | 283 ++++++++++++++++++ .../understand-crossrepo/combine-graphs.py | 271 +++++++++++++++++ 2 files changed, 554 insertions(+) create mode 100644 understand-anything-plugin/skills/understand-crossrepo/__tests__/test_combine.py create mode 100644 understand-anything-plugin/skills/understand-crossrepo/combine-graphs.py diff --git a/understand-anything-plugin/skills/understand-crossrepo/__tests__/test_combine.py b/understand-anything-plugin/skills/understand-crossrepo/__tests__/test_combine.py new file mode 100644 index 000000000..5664576bc --- /dev/null +++ b/understand-anything-plugin/skills/understand-crossrepo/__tests__/test_combine.py @@ -0,0 +1,283 @@ +#!/usr/bin/env python3 +""" +test_combine.py — TDD tests for combine-graphs.py + +Two fixture repos both containing a `file:Dockerfile` node and an edge. +Assert: + - Distinct namespaced ids (file:/Dockerfile per repo) + - Zero id collisions in the union + - Both layer: and layer: present with correct nodeIds + - Edge endpoints rewritten to namespaced ids + - module: node exists per repo +""" + +import json +import subprocess +import sys +import tempfile +from pathlib import Path + + +def make_repo(tmp: Path, ns: str) -> Path: + """Build a minimal .understand-anything/knowledge-graph.json in a temp repo dir.""" + repo = tmp / ns + ua = repo / ".understand-anything" + ua.mkdir(parents=True) + + graph = { + "version": "1.0.0", + "project": {"name": ns, "languages": ["Python"], "frameworks": []}, + "nodes": [ + { + "id": "file:Dockerfile", + "type": "file", + "name": "Dockerfile", + "filePath": "Dockerfile", + "summary": f"Docker build for {ns}", + "tags": [], + }, + { + "id": f"file:src/main.py", + "type": "file", + "name": "main.py", + "filePath": "src/main.py", + "summary": "Entry point", + "tags": [], + }, + { + "id": f"function:src/main.py:run", + "type": "function", + "name": "run", + "filePath": "src/main.py", + "summary": "Main runner", + "tags": [], + }, + ], + "edges": [ + { + "source": "file:Dockerfile", + "target": "file:src/main.py", + "type": "references", + "direction": "forward", + "weight": 0.5, + } + ], + "layers": [ + { + "id": "layer:internal", + "name": "Internal Layer", + "description": "Should be discarded", + "nodeIds": ["file:Dockerfile", "file:src/main.py"], + } + ], + "tour": [], + } + (ua / "knowledge-graph.json").write_text(json.dumps(graph), encoding="utf-8") + return repo + + +def run_combine(out: Path, *repo_ns_pairs: str) -> tuple[dict, dict]: + """Run combine-graphs.py, return (combined-graph, id-map).""" + script = Path(__file__).parent.parent / "combine-graphs.py" + cmd = [sys.executable, str(script), str(out)] + list(repo_ns_pairs) + result = subprocess.run(cmd, capture_output=True, text=True) + assert result.returncode == 0, f"combine-graphs.py failed:\nstdout: {result.stdout}\nstderr: {result.stderr}" + + combined = json.loads((out / ".understand-anything" / "intermediate" / "combined-graph.json").read_text()) + id_map = json.loads((out / ".understand-anything" / "intermediate" / "id-map.json").read_text()) + return combined, id_map + + +def test_no_id_collisions_and_distinct_dockerfiles(): + with tempfile.TemporaryDirectory() as tmp_str: + tmp = Path(tmp_str) + repo_a = make_repo(tmp, "repo_a") + repo_b = make_repo(tmp, "repo_b") + out = tmp / "out" + + combined, id_map = run_combine(out, f"{repo_a}:repo_a", f"{repo_b}:repo_b") + + node_ids = [n["id"] for n in combined["nodes"]] + + # Both Dockerfiles must be distinct namespaced ids + assert "file:repo_a/Dockerfile" in node_ids, f"Missing file:repo_a/Dockerfile in {node_ids}" + assert "file:repo_b/Dockerfile" in node_ids, f"Missing file:repo_b/Dockerfile in {node_ids}" + + # Old bare id must NOT be present + assert "file:Dockerfile" not in node_ids, "Bare file:Dockerfile still present (no namespace)" + + # Zero id collisions — all ids are unique + assert len(node_ids) == len(set(node_ids)), f"Duplicate ids found: {[x for x in node_ids if node_ids.count(x) > 1]}" + + +def test_both_layers_present_with_correct_nodeids(): + with tempfile.TemporaryDirectory() as tmp_str: + tmp = Path(tmp_str) + repo_a = make_repo(tmp, "repo_a") + repo_b = make_repo(tmp, "repo_b") + out = tmp / "out" + + combined, _ = run_combine(out, f"{repo_a}:repo_a", f"{repo_b}:repo_b") + + layer_ids = {l["id"] for l in combined["layers"]} + assert "layer:repo_a" in layer_ids, f"Missing layer:repo_a in {layer_ids}" + assert "layer:repo_b" in layer_ids, f"Missing layer:repo_b in {layer_ids}" + + # Internal layers from repos must be discarded + assert "layer:internal" not in layer_ids, "Repo's internal layer must be discarded" + + # Each layer's nodeIds must contain at least the file-level nodes for that repo + layer_a = next(l for l in combined["layers"] if l["id"] == "layer:repo_a") + layer_b = next(l for l in combined["layers"] if l["id"] == "layer:repo_b") + + # Dockerfile (file-level) must be in each repo's layer + assert "file:repo_a/Dockerfile" in layer_a["nodeIds"], f"Dockerfile not in layer_a nodeIds: {layer_a['nodeIds']}" + assert "file:repo_b/Dockerfile" in layer_b["nodeIds"], f"Dockerfile not in layer_b nodeIds: {layer_b['nodeIds']}" + + # function nodes must NOT be in the layer (only top-level) + assert not any(nid.startswith("function:") for nid in layer_a["nodeIds"]), \ + "Function nodes should not be in repo layer" + + # module anchor must be in each layer + assert "module:repo_a" in layer_a["nodeIds"], f"module:repo_a missing from layer nodeIds: {layer_a['nodeIds']}" + assert "module:repo_b" in layer_b["nodeIds"], f"module:repo_b missing from layer nodeIds: {layer_b['nodeIds']}" + + +def test_edge_endpoints_rewritten(): + with tempfile.TemporaryDirectory() as tmp_str: + tmp = Path(tmp_str) + repo_a = make_repo(tmp, "repo_a") + repo_b = make_repo(tmp, "repo_b") + out = tmp / "out" + + combined, _ = run_combine(out, f"{repo_a}:repo_a", f"{repo_b}:repo_b") + + # Find the references edge from repo_a (Dockerfile → main.py) + ref_edges = [e for e in combined["edges"] if e["type"] == "references"] + assert ref_edges, "No references edges found" + + # All edge endpoints must be namespaced + for edge in combined["edges"]: + src, tgt = edge["source"], edge["target"] + # No bare file:Dockerfile — must have a namespace segment + assert not src == "file:Dockerfile", f"Bare source id in edge: {edge}" + assert not tgt == "file:Dockerfile", f"Bare target id in edge: {edge}" + + # Specifically: there must be a references edge repo_a/Dockerfile → repo_a/src/main.py + a_ref = next( + (e for e in ref_edges + if e["source"] == "file:repo_a/Dockerfile" and e["target"] == "file:repo_a/src/main.py"), + None, + ) + assert a_ref is not None, "Expected namespaced edge file:repo_a/Dockerfile → file:repo_a/src/main.py not found" + + +def test_module_anchor_node_per_repo(): + with tempfile.TemporaryDirectory() as tmp_str: + tmp = Path(tmp_str) + repo_a = make_repo(tmp, "repo_a") + repo_b = make_repo(tmp, "repo_b") + out = tmp / "out" + + combined, _ = run_combine(out, f"{repo_a}:repo_a", f"{repo_b}:repo_b") + + node_ids = {n["id"] for n in combined["nodes"]} + assert "module:repo_a" in node_ids, f"module:repo_a missing from nodes: {node_ids}" + assert "module:repo_b" in node_ids, f"module:repo_b missing from nodes: {node_ids}" + + mod_a = next(n for n in combined["nodes"] if n["id"] == "module:repo_a") + assert mod_a["type"] == "module" + assert mod_a["name"] == "repo_a" + assert mod_a.get("repo") == "repo_a" + + +def test_id_map_is_keyed_by_ns(): + with tempfile.TemporaryDirectory() as tmp_str: + tmp = Path(tmp_str) + repo_a = make_repo(tmp, "repo_a") + repo_b = make_repo(tmp, "repo_b") + out = tmp / "out" + + _, id_map = run_combine(out, f"{repo_a}:repo_a", f"{repo_b}:repo_b") + + # id-map must be keyed by ns to avoid collisions + assert "repo_a" in id_map, f"repo_a key missing from id-map: {list(id_map.keys())}" + assert "repo_b" in id_map, f"repo_b key missing from id-map: {list(id_map.keys())}" + + # Within repo_a, old id → new id + a_map = id_map["repo_a"] + assert "file:Dockerfile" in a_map, f"file:Dockerfile missing from repo_a id-map: {a_map}" + assert a_map["file:Dockerfile"] == "file:repo_a/Dockerfile" + + +def test_scan_artifact_nodes_dropped(): + """Nodes whose filePath contains .understand-anything must be dropped.""" + with tempfile.TemporaryDirectory() as tmp_str: + tmp = Path(tmp_str) + repo = tmp / "repo_c" + ua = repo / ".understand-anything" + ua.mkdir(parents=True) + + graph = { + "version": "1.0.0", + "project": {"name": "repo_c"}, + "nodes": [ + {"id": "file:src/app.py", "type": "file", "name": "app.py", "filePath": "src/app.py", "summary": "", "tags": []}, + # Scan artifact — must be dropped + {"id": "file:.understand-anything/meta.json", "type": "file", "name": "meta.json", + "filePath": ".understand-anything/meta.json", "summary": "", "tags": []}, + ], + "edges": [], + "layers": [], + "tour": [], + } + (ua / "knowledge-graph.json").write_text(json.dumps(graph), encoding="utf-8") + + out = tmp / "out2" + combined, _ = run_combine(out, f"{repo}:repo_c") + + node_ids = [n["id"] for n in combined["nodes"]] + assert not any(".understand-anything" in nid for nid in node_ids), \ + f".understand-anything artifact node not dropped: {node_ids}" + assert "file:repo_c/src/app.py" in node_ids + + +def test_output_is_deterministic(): + """Two identical runs produce byte-identical output.""" + with tempfile.TemporaryDirectory() as tmp_str: + tmp = Path(tmp_str) + repo_a = make_repo(tmp, "repo_a") + repo_b = make_repo(tmp, "repo_b") + + out1 = tmp / "out1" + out2 = tmp / "out2" + run_combine(out1, f"{repo_a}:repo_a", f"{repo_b}:repo_b") + run_combine(out2, f"{repo_a}:repo_a", f"{repo_b}:repo_b") + + g1 = (out1 / ".understand-anything" / "intermediate" / "combined-graph.json").read_text() + g2 = (out2 / ".understand-anything" / "intermediate" / "combined-graph.json").read_text() + assert g1 == g2, "Output is not deterministic across two runs" + + +if __name__ == "__main__": + tests = [ + test_no_id_collisions_and_distinct_dockerfiles, + test_both_layers_present_with_correct_nodeids, + test_edge_endpoints_rewritten, + test_module_anchor_node_per_repo, + test_id_map_is_keyed_by_ns, + test_scan_artifact_nodes_dropped, + test_output_is_deterministic, + ] + passed = 0 + failed = 0 + for t in tests: + try: + t() + print(f" PASS {t.__name__}") + passed += 1 + except Exception as e: + print(f" FAIL {t.__name__}: {e}") + failed += 1 + print(f"\n{passed} passed, {failed} failed") + sys.exit(0 if failed == 0 else 1) diff --git a/understand-anything-plugin/skills/understand-crossrepo/combine-graphs.py b/understand-anything-plugin/skills/understand-crossrepo/combine-graphs.py new file mode 100644 index 000000000..e0f93d60a --- /dev/null +++ b/understand-anything-plugin/skills/understand-crossrepo/combine-graphs.py @@ -0,0 +1,271 @@ +#!/usr/bin/env python3 +""" +combine-graphs.py — Namespace + union per-repo knowledge graphs into one combined graph. + +Usage: + python combine-graphs.py : : ... + +For each : arg: + - Loads /.understand-anything/knowledge-graph.json + - Namespaces every node id and filePath with / + - Rewrites all edge endpoints through the per-repo id map + - Discards the repo's internal layers; creates one layer: per repo + - Adds a module: anchor node per repo + +Union + dedup across all repos: + - Nodes by id (last wins) + - Edges by (source, target, type) — higher weight wins + +Output: + /.understand-anything/intermediate/combined-graph.json + /.understand-anything/intermediate/id-map.json + +Task 3 of /understand-crossrepo. Task 5 adds cross-repo edges to this substrate. +""" + +import json +import sys +from pathlib import Path +from typing import Any + + +_SCAN_ARTIFACT_MARKERS = (".understand-anything", ".trash-") + + +def _num(v: Any) -> float: + try: + return float(v) + except (TypeError, ValueError): + return 0.0 + + +def _namespace_id(node_id: str, ns: str) -> str: + """Insert / after the FIRST colon only. + + file:Dockerfile → file:/Dockerfile + endpoint:api/x.py → endpoint:/api/x.py + function:api/x.py:foo → function:/api/x.py:foo + """ + kind, rest = node_id.split(":", 1) + return f"{kind}:{ns}/{rest}" + + +def _is_scan_artifact(node: dict) -> bool: + fp = node.get("filePath", "") or "" + nid = node.get("id", "") or "" + for marker in _SCAN_ARTIFACT_MARKERS: + if marker in fp or marker in nid: + return True + return False + + +def _is_top_level(namespaced_id: str) -> bool: + """True for file/endpoint/service nodes — not function/class members.""" + kind = namespaced_id.split(":")[0] + return kind in ("file", "endpoint", "service") + + +def load_graph(path: Path) -> dict | None: + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as e: + print(f" Warning: skipping {path}: {e}", file=sys.stderr) + return None + if not isinstance(data.get("nodes"), list) or not isinstance(data.get("edges"), list): + print(f" Warning: skipping {path}: missing nodes or edges", file=sys.stderr) + return None + return data + + +def namespace_repo(graph: dict, ns: str) -> tuple[dict, dict]: + """Namespace all ids in `graph` for `ns`. Returns (namespaced_graph, id_map). + + id_map: old_id → new_id (for this repo only — no global collision risk). + """ + id_map: dict[str, str] = {} + + # Phase 1: drop scan artifacts, build id_map for survivors + kept_nodes: list[dict] = [] + for node in graph.get("nodes", []): + if _is_scan_artifact(node): + continue + old_id = node.get("id", "") + if not old_id: + continue + new_id = _namespace_id(old_id, ns) + id_map[old_id] = new_id + kept_nodes.append(node) + + # Phase 2: rewrite node fields + ns_nodes: list[dict] = [] + for node in kept_nodes: + n = dict(node) + n["id"] = id_map[node["id"]] + if "filePath" in n and n["filePath"]: + n["filePath"] = f"{ns}/{n['filePath']}" + # Tag with repo + tags = list(n.get("tags") or []) + if f"repo:{ns}" not in tags: + tags.append(f"repo:{ns}") + n["tags"] = tags + n["repo"] = ns + ns_nodes.append(n) + + # Phase 3: rewrite edges, drop dangling (apply id_map; same split rule for unknowns) + dropped_edges = 0 + ns_edges: list[dict] = [] + for edge in graph.get("edges", []): + src_old = edge.get("source", "") + tgt_old = edge.get("target", "") + + if src_old in id_map: + new_src = id_map[src_old] + else: + # Dangling: apply split rule best-effort, then drop + dropped_edges += 1 + continue + + if tgt_old in id_map: + new_tgt = id_map[tgt_old] + else: + dropped_edges += 1 + continue + + e = dict(edge) + e["source"] = new_src + e["target"] = new_tgt + ns_edges.append(e) + + if dropped_edges: + print(f" [{ns}] dropped {dropped_edges} dangling edges", file=sys.stderr) + + return {"nodes": ns_nodes, "edges": ns_edges}, id_map + + +def build_layer(ns: str, ns_nodes: list[dict]) -> dict: + """One layer: whose nodeIds = file-level nodes + module: anchor.""" + top_level_ids = sorted( + n["id"] for n in ns_nodes if _is_top_level(n["id"]) + ) + anchor_id = f"module:{ns}" + node_ids = sorted(set(top_level_ids + [anchor_id])) + return { + "id": f"layer:{ns}", + "name": ns, + "nodeIds": node_ids, + } + + +def build_module_anchor(ns: str) -> dict: + return { + "id": f"module:{ns}", + "type": "module", + "name": ns, + "repo": ns, + } + + +def combine(out: Path, repo_ns_pairs: list[tuple[Path, str]]) -> None: + intermediate = out / ".understand-anything" / "intermediate" + intermediate.mkdir(parents=True, exist_ok=True) + + all_id_maps: dict[str, dict] = {} # ns → {old → new} + # Global union structures + nodes_by_id: dict[str, dict] = {} + edges_by_key: dict[tuple, dict] = {} # (src, tgt, type) → edge + layers: list[dict] = [] + + for repo, ns in repo_ns_pairs: + kg_path = repo / ".understand-anything" / "knowledge-graph.json" + graph = load_graph(kg_path) + if graph is None: + print(f" Skipping {ns} (could not load {kg_path})", file=sys.stderr) + continue + + ns_graph, id_map = namespace_repo(graph, ns) + all_id_maps[ns] = id_map + + # Add module anchor node + anchor = build_module_anchor(ns) + ns_graph["nodes"].append(anchor) + + # Union nodes (last wins on collision) + for node in ns_graph["nodes"]: + nodes_by_id[node["id"]] = node + + # Union edges (higher weight wins on collision) + for edge in ns_graph["edges"]: + key = (edge.get("source", ""), edge.get("target", ""), edge.get("type", "")) + existing = edges_by_key.get(key) + if existing is None or _num(edge.get("weight", 0)) > _num(existing.get("weight", 0)): + edges_by_key[key] = edge + + # Build per-repo layer (pass ns_nodes without anchor to avoid double-add in _is_top_level) + layers.append(build_layer(ns, ns_graph["nodes"])) + + # Drop edges referencing nodes not in the union (cross-repo dangling — Task 5 adds those) + node_ids_set = set(nodes_by_id.keys()) + valid_edges = [ + e for e in edges_by_key.values() + if e["source"] in node_ids_set and e["target"] in node_ids_set + ] + + # Stable sort for determinism + sorted_nodes = sorted(nodes_by_id.values(), key=lambda n: n["id"]) + sorted_edges = sorted(valid_edges, key=lambda e: (e["source"], e["target"], e.get("type", ""))) + sorted_layers = sorted(layers, key=lambda l: l["id"]) + + combined = { + "version": "1.0.0", + "project": {"name": "combined", "repos": [ns for _, ns in repo_ns_pairs]}, + "nodes": sorted_nodes, + "edges": sorted_edges, + "layers": sorted_layers, + } + + (intermediate / "combined-graph.json").write_text( + json.dumps(combined, indent=2, ensure_ascii=False), encoding="utf-8" + ) + (intermediate / "id-map.json").write_text( + json.dumps(all_id_maps, indent=2, ensure_ascii=False), encoding="utf-8" + ) + + print( + f"combined-graph: {len(sorted_nodes)} nodes, {len(sorted_edges)} edges, " + f"{len(sorted_layers)} layers → {intermediate / 'combined-graph.json'}", + file=sys.stderr, + ) + + +def parse_args(argv: list[str]) -> tuple[Path, list[tuple[Path, str]]]: + if len(argv) < 3: + print( + "Usage: python combine-graphs.py : : ...", + file=sys.stderr, + ) + sys.exit(1) + + out = Path(argv[1]).resolve() + pairs: list[tuple[Path, str]] = [] + for arg in argv[2:]: + if ":" in arg: + # Split on the LAST colon that separates path from ns + # (paths on Windows may have drive letters, but we're on POSIX anyway) + # Use the FIRST colon only if arg looks like /abs/path:ns or rel/path:ns + # Safest: rsplit on ":" with maxsplit=1 + repo_str, ns = arg.rsplit(":", 1) + else: + repo_str = arg + ns = Path(arg).name + pairs.append((Path(repo_str).resolve(), ns)) + + return out, pairs + + +def main() -> None: + out, pairs = parse_args(sys.argv) + combine(out, pairs) + + +if __name__ == "__main__": + main() From d0a24cfa89400bedda989eb0fb394dab86afead9 Mon Sep 17 00:00:00 2001 From: Mohammed Hafiz <65387738+Hafiz408@users.noreply.github.com> Date: Fri, 26 Jun 2026 19:41:11 +0530 Subject: [PATCH 06/17] fix(crossrepo): tag module anchor, tighten trash-path skip, cover endpoint/service layer membership Co-Authored-By: Claude Sonnet 4.6 --- .../__tests__/test_combine.py | 26 +++++++++++++++++-- .../understand-crossrepo/combine-graphs.py | 9 ++++--- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/understand-anything-plugin/skills/understand-crossrepo/__tests__/test_combine.py b/understand-anything-plugin/skills/understand-crossrepo/__tests__/test_combine.py index 5664576bc..00a3b50ef 100644 --- a/understand-anything-plugin/skills/understand-crossrepo/__tests__/test_combine.py +++ b/understand-anything-plugin/skills/understand-crossrepo/__tests__/test_combine.py @@ -37,7 +37,7 @@ def make_repo(tmp: Path, ns: str) -> Path: "tags": [], }, { - "id": f"file:src/main.py", + "id": "file:src/main.py", "type": "file", "name": "main.py", "filePath": "src/main.py", @@ -45,13 +45,29 @@ def make_repo(tmp: Path, ns: str) -> Path: "tags": [], }, { - "id": f"function:src/main.py:run", + "id": "function:src/main.py:run", "type": "function", "name": "run", "filePath": "src/main.py", "summary": "Main runner", "tags": [], }, + { + "id": "endpoint:api/v1/items", + "type": "endpoint", + "name": "GET /api/v1/items", + "filePath": "src/main.py", + "summary": "List items endpoint", + "tags": [], + }, + { + "id": "service:ItemService", + "type": "service", + "name": "ItemService", + "filePath": "src/main.py", + "summary": "Item business logic", + "tags": [], + }, ], "edges": [ { @@ -138,6 +154,12 @@ def test_both_layers_present_with_correct_nodeids(): assert not any(nid.startswith("function:") for nid in layer_a["nodeIds"]), \ "Function nodes should not be in repo layer" + # endpoint: and service: nodes MUST be in the layer (Task 5 relies on this) + assert "endpoint:repo_a/api/v1/items" in layer_a["nodeIds"], \ + f"endpoint node missing from layer_a nodeIds: {layer_a['nodeIds']}" + assert "service:repo_a/ItemService" in layer_a["nodeIds"], \ + f"service node missing from layer_a nodeIds: {layer_a['nodeIds']}" + # module anchor must be in each layer assert "module:repo_a" in layer_a["nodeIds"], f"module:repo_a missing from layer nodeIds: {layer_a['nodeIds']}" assert "module:repo_b" in layer_b["nodeIds"], f"module:repo_b missing from layer nodeIds: {layer_b['nodeIds']}" diff --git a/understand-anything-plugin/skills/understand-crossrepo/combine-graphs.py b/understand-anything-plugin/skills/understand-crossrepo/combine-graphs.py index e0f93d60a..12f4de7a0 100644 --- a/understand-anything-plugin/skills/understand-crossrepo/combine-graphs.py +++ b/understand-anything-plugin/skills/understand-crossrepo/combine-graphs.py @@ -53,9 +53,11 @@ def _namespace_id(node_id: str, ns: str) -> str: def _is_scan_artifact(node: dict) -> bool: fp = node.get("filePath", "") or "" nid = node.get("id", "") or "" - for marker in _SCAN_ARTIFACT_MARKERS: - if marker in fp or marker in nid: - return True + if ".understand-anything" in fp or ".understand-anything" in nid: + return True + # ponytail: path-segment check — avoid matching "src/.trash-backup/file.py" mid-path + if fp.startswith(".trash-") or "/.trash-" in fp or nid.startswith(".trash-") or "/.trash-" in nid: + return True return False @@ -162,6 +164,7 @@ def build_module_anchor(ns: str) -> dict: "type": "module", "name": ns, "repo": ns, + "tags": [f"repo:{ns}"], } From 604cba6e2a68af744871bf100328d8d228aa17b7 Mon Sep 17 00:00:00 2001 From: Mohammed Hafiz <65387738+Hafiz408@users.noreply.github.com> Date: Fri, 26 Jun 2026 19:45:00 +0530 Subject: [PATCH 07/17] feat(crossrepo): LLM cross-repo linker agent Co-Authored-By: Claude Sonnet 4.6 --- .../agents/crossrepo-linker.md | 132 ++++++++++++++++++ 1 file changed, 132 insertions(+) create mode 100644 understand-anything-plugin/agents/crossrepo-linker.md diff --git a/understand-anything-plugin/agents/crossrepo-linker.md b/understand-anything-plugin/agents/crossrepo-linker.md new file mode 100644 index 000000000..09b510840 --- /dev/null +++ b/understand-anything-plugin/agents/crossrepo-linker.md @@ -0,0 +1,132 @@ +--- +name: crossrepo-linker +description: | + Given all repos' SIGNAL files and the combined graph's node-id list, emits + typed cross-repo edges (calls, embeds, authenticates_via, publishes, + subscribes, reads_from, writes_to, depends_on) to + /.understand-anything/intermediate/crossrepo-edges.json. +--- + +# Cross-Repo Linker Agent + +You are a cross-repository integration analyst. Your job is to read every repo's +outbound signals alongside the collected identity claims and emit typed, +evidence-backed edges that represent real runtime integrations — HTTP calls, +SSO, iframe embeds, Pub/Sub, shared buckets/DBs. + +## Input + +The dispatching skill provides two things: + +**1. Signal files** — one per repo, JSON of shape: +```json +{ + "repo": "", + "namespace": "", + "identity": { + "serviceName": "", + "keycloakClientId": "|null", + "servedHosts": [""], + "endpoints": [""], + "ownedTopics": [""], + "ownedBuckets": [""] + }, + "outbound": [ + { "kind": "api|auth|embed|pubsub|bucket|db", "value": "", "evidence": "file:line" } + ] +} +``` + +**2. Combined-graph node-id list** — the set of `module:` anchor nodes (one +per repo) and `endpoint:/...` served-endpoint nodes available for +`fineTarget` precision. + +The dispatching skill names the output path as ``. + +## Matching Rules + +For each repo A's outbound signal, attempt a match against every other repo B's +identity or a clearly-shared external service. Apply rules top-to-bottom; stop +at the first match. + +| Signal kind | Match condition | Edge type | Target | +|---|---|---|---| +| `api` | signal value matches B's `identity.servedHosts` or `identity.endpoints` (substring or prefix) | `calls` | `module:` | +| `auth` | signal value matches a Keycloak realm/client/JWKS URL and B's `identity.keycloakClientId` is that client, OR no repo owns it | `authenticates_via` | `module:` or `external:keycloak` | +| `embed` | iframe src matches B's `identity.servedHosts` (UI→UI embed) | `embeds` | `module:` | +| `pubsub` | signal value ∈ B's `identity.ownedTopics` | `publishes` (A→B) or `subscribes` (B→A) | `module:` | +| `bucket` | signal value ∈ B's `identity.ownedBuckets` | `reads_from` or `writes_to` | `module:` | +| `db` | signal value matches a clearly-shared DB owned by B | `reads_from` or `writes_to` | `module:` | +| any | no repo match, but signal points to recognisable shared infra (Bridge data hub, GCP, ZeptoMail, Redis, BigQuery, etc.) | `depends_on` | `external:` | + +Naming for `external` targets: use lowercase kebab — `external:keycloak`, +`external:gcp`, `external:bridge-core`, `external:zeptomail`, `external:redis`, +`external:bigquery`, etc. + +Allowed `type` values: `calls | embeds | authenticates_via | publishes | +subscribes | reads_from | writes_to | depends_on`. + +For `publishes`/`subscribes`: if A's outbound names a topic owned by B, emit +`publishes` (A→B). If B's outbound names a topic owned by A, emit `subscribes` +(B→A, meaning B listens to A's topic). + +When a specific `endpoint:/...` node from the node-id list matches the +outbound path, set `fineTarget` to that node id. + +## Output Schema + +Write a JSON array to `/.understand-anything/intermediate/crossrepo-edges.json`: + +```json +[ + { + "source": "module:", + "target": "module:", + "type": "calls", + "label": "", + "weight": 0.8, + "direction": "forward", + "fineTarget": "endpoint:/api/v1/...", + "confidence": 0.9, + "evidence": "" + } +] +``` + +Field rules: +- `source` — always `module:` (the originating repo's anchor). +- `target` — `module:` for a matched repo; `external:` for shared infra. +- `direction` — always `"forward"`. +- `weight` — edge importance (0.0–1.0): use 0.9 for direct API calls, 0.8 for + auth/embed, 0.7 for pub/sub, 0.6 for bucket/db, 0.5 for external infra. +- `confidence` — match strength (0.0–1.0): exact host/client-id match = 0.9–1.0; + prefix/substring match = 0.6–0.8; heuristic/env-var name match = 0.3–0.5. +- `fineTarget` — omit when no specific endpoint node matches. +- `evidence` — required; cite the `evidence` field from the outbound signal and + the identity field it matched. + +## Self-Grounding + +**Only emit edges with real signal basis.** + +- Do NOT invent an edge because two repos seem related by name. +- DO emit a plausible-but-uncertain edge with low confidence (0.3–0.5) rather + than dropping it — Task 5 renders low-confidence edges faded, so they are + useful as weak hints. +- Do NOT emit an edge with confidence below 0.3 — that is fabrication. +- Prefer `fineTarget` when the outbound path precisely matches a served-endpoint + node id in the provided list. +- Every edge must include `evidence` quoting which outbound signal matched which + identity field, including the file:line from the signal. +- No self-edges: `source` and `target` must differ. +- No duplicate edges: if two signals produce the same `(source, target, type)` + pair, merge them into one edge with the higher confidence and combined evidence. + +## Writing Results + +1. Write the JSON array to `/.understand-anything/intermediate/crossrepo-edges.json`. + Create the `intermediate/` directory if it does not exist. +2. Respond with ONLY a brief text summary: total edges emitted, breakdown by + `type`, count of `external:*` targets, and any low-confidence edges flagged. + +Do NOT include the full JSON in your text response. From feb16d01159f7b97dd090daad6eb52ecf7b4e07a Mon Sep 17 00:00:00 2001 From: Mohammed Hafiz <65387738+Hafiz408@users.noreply.github.com> Date: Fri, 26 Jun 2026 19:55:01 +0530 Subject: [PATCH 08/17] feat(crossrepo): apply interlinks, synthesize external infra, assemble + validate Co-Authored-By: Claude Sonnet 4.6 --- .../__tests__/test_apply.py | 367 +++++++++++++++++ .../understand-crossrepo/apply-interlinks.py | 374 ++++++++++++++++++ 2 files changed, 741 insertions(+) create mode 100644 understand-anything-plugin/skills/understand-crossrepo/__tests__/test_apply.py create mode 100644 understand-anything-plugin/skills/understand-crossrepo/apply-interlinks.py diff --git a/understand-anything-plugin/skills/understand-crossrepo/__tests__/test_apply.py b/understand-anything-plugin/skills/understand-crossrepo/__tests__/test_apply.py new file mode 100644 index 000000000..cd5a0ee3b --- /dev/null +++ b/understand-anything-plugin/skills/understand-crossrepo/__tests__/test_apply.py @@ -0,0 +1,367 @@ +#!/usr/bin/env python3 +""" +test_apply.py — TDD tests for apply-interlinks.py (Task 5). + +Fixture: two-repo combined-graph + crossrepo-edges with: + - one edge targeting external:keycloak + - one valid module→module edge + - one DANGLING edge (target not in node set) + +Assertions: + a) service:external/keycloak node exists with summary+tags, in layer:external-shared-infra + b) dangling edge was dropped + c) valid + external edges present with direction:"forward" and x ids + d) bundled validator on knowledge-graph.json → ZERO issues + e) meta.json written +""" + +import json +import subprocess +import sys +import tempfile +from pathlib import Path + + +SCRIPT = Path(__file__).parent.parent / "apply-interlinks.py" + + +def make_combined_graph(out: Path) -> None: + """Write a minimal combined-graph.json with two repos and module anchors.""" + intermediate = out / ".understand-anything" / "intermediate" + intermediate.mkdir(parents=True, exist_ok=True) + + graph = { + "version": "1.0.0", + "project": { + "name": "test-combined", + "languages": ["Python"], + "frameworks": [], + "description": "Test fixture combined graph", + }, + "nodes": [ + # Repo A: file node + module anchor + { + "id": "file:svc_a/src/main.py", + "type": "file", + "name": "main.py", + "filePath": "svc_a/src/main.py", + "summary": "Entry point for svc_a", + "tags": ["repo:svc_a"], + "repo": "svc_a", + }, + { + "id": "module:svc_a", + "type": "module", + "name": "svc_a", + "summary": "Module anchor for svc_a", + "tags": ["repo:svc_a"], + "repo": "svc_a", + }, + # Repo B: file node + module anchor + { + "id": "file:svc_b/src/app.py", + "type": "file", + "name": "app.py", + "filePath": "svc_b/src/app.py", + "summary": "Entry point for svc_b", + "tags": ["repo:svc_b"], + "repo": "svc_b", + }, + { + "id": "module:svc_b", + "type": "module", + "name": "svc_b", + "summary": "Module anchor for svc_b", + "tags": ["repo:svc_b"], + "repo": "svc_b", + }, + ], + "edges": [ + # Intra-repo edge (svc_a) + { + "source": "module:svc_a", + "target": "file:svc_a/src/main.py", + "type": "contains", + "direction": "forward", + "weight": 0.8, + }, + # Intra-repo edge (svc_b) + { + "source": "module:svc_b", + "target": "file:svc_b/src/app.py", + "type": "contains", + "direction": "forward", + "weight": 0.8, + }, + ], + "layers": [ + { + "id": "layer:svc_a", + "name": "svc_a", + "description": "Layer for svc_a", + "nodeIds": ["file:svc_a/src/main.py", "module:svc_a"], + }, + { + "id": "layer:svc_b", + "name": "svc_b", + "description": "Layer for svc_b", + "nodeIds": ["file:svc_b/src/app.py", "module:svc_b"], + }, + ], + } + (intermediate / "combined-graph.json").write_text( + json.dumps(graph, indent=2), encoding="utf-8" + ) + + +def make_crossrepo_edges(out: Path) -> None: + """Write crossrepo-edges.json with three edge cases.""" + intermediate = out / ".understand-anything" / "intermediate" + edges = [ + # Valid: module→external (keycloak) + { + "source": "module:svc_a", + "target": "external:keycloak", + "type": "auth-dependency", + "label": "svc_a authenticates via Keycloak", + "weight": 0.9, + "direction": "forward", + "confidence": 0.85, + "evidence": "Uses Keycloak OIDC", + }, + # Valid: module→module cross-repo + { + "source": "module:svc_a", + "target": "module:svc_b", + "type": "api-call", + "label": "svc_a calls svc_b API", + "weight": 0.7, + "direction": "forward", + "confidence": 0.75, + "evidence": "HTTP client import", + }, + # Dangling: target endpoint does not exist in node set + { + "source": "module:svc_a", + "target": "endpoint:svc_b/api/v1/nonexistent", + "type": "api-call", + "label": "Dangling edge — target missing", + "weight": 0.5, + "direction": "forward", + "confidence": 0.6, + "evidence": "Stale reference", + }, + # Low-confidence: must be kept but tagged lowConfidence + { + "source": "module:svc_b", + "target": "module:svc_a", + "type": "event-dependency", + "label": "svc_b maybe consumes svc_a events", + "weight": 0.3, + "direction": "forward", + "confidence": 0.3, + "evidence": "Speculative", + }, + ] + (intermediate / "crossrepo-edges.json").write_text( + json.dumps(edges, indent=2), encoding="utf-8" + ) + + +def run_apply(out: Path) -> subprocess.CompletedProcess: + return subprocess.run( + [sys.executable, str(SCRIPT), str(out)], + capture_output=True, + text=True, + ) + + +# ── Tests ────────────────────────────────────────────────────────────────────── + + +def test_external_node_and_layer_created(): + """(a) service:external/keycloak exists with summary+tags, in layer:external-shared-infra.""" + with tempfile.TemporaryDirectory() as tmp_str: + out = Path(tmp_str) + make_combined_graph(out) + make_crossrepo_edges(out) + + result = run_apply(out) + assert result.returncode == 0, f"apply-interlinks failed:\n{result.stderr}" + + kg = json.loads((out / ".understand-anything" / "knowledge-graph.json").read_text()) + node_ids = {n["id"]: n for n in kg["nodes"]} + + # Node exists + assert "service:external/keycloak" in node_ids, ( + f"service:external/keycloak missing from nodes: {list(node_ids.keys())}" + ) + kc = node_ids["service:external/keycloak"] + assert kc["summary"], "External keycloak node missing summary" + assert kc["tags"], "External keycloak node missing tags" + assert "external" in kc["tags"], "External node tags must include 'external'" + + # In the external layer + layers = {l["id"]: l for l in kg["layers"]} + assert "layer:external-shared-infra" in layers, ( + f"layer:external-shared-infra missing: {list(layers.keys())}" + ) + ext_layer = layers["layer:external-shared-infra"] + assert "service:external/keycloak" in ext_layer["nodeIds"], ( + f"keycloak not in external layer nodeIds: {ext_layer['nodeIds']}" + ) + + +def test_dangling_edge_dropped(): + """(b) edge with non-existent target endpoint is dropped.""" + with tempfile.TemporaryDirectory() as tmp_str: + out = Path(tmp_str) + make_combined_graph(out) + make_crossrepo_edges(out) + + result = run_apply(out) + assert result.returncode == 0, f"apply-interlinks failed:\n{result.stderr}" + + kg = json.loads((out / ".understand-anything" / "knowledge-graph.json").read_text()) + node_ids = {n["id"] for n in kg["nodes"]} + + for edge in kg["edges"]: + assert edge["target"] in node_ids, ( + f"Dangling edge not dropped: target '{edge['target']}' not in node set" + ) + assert edge["source"] in node_ids, ( + f"Dangling edge not dropped: source '{edge['source']}' not in node set" + ) + + # Specifically the dangling endpoint edge must be absent + dangling_targets = [ + e for e in kg["edges"] + if e.get("target") == "endpoint:svc_b/api/v1/nonexistent" + ] + assert not dangling_targets, "Dangling edge with missing endpoint target was not dropped" + + +def test_valid_and_external_edges_present(): + """(c) valid module→module and external edges present with direction:forward and x ids.""" + with tempfile.TemporaryDirectory() as tmp_str: + out = Path(tmp_str) + make_combined_graph(out) + make_crossrepo_edges(out) + + result = run_apply(out) + assert result.returncode == 0, f"apply-interlinks failed:\n{result.stderr}" + + kg = json.loads((out / ".understand-anything" / "knowledge-graph.json").read_text()) + + cross_edges = [e for e in kg["edges"] if e.get("id", "").startswith("x")] + assert cross_edges, "No cross-repo edges (x ids) found" + + # All cross edges have direction:forward and numeric weight + for e in cross_edges: + assert e.get("direction") == "forward", f"Edge missing direction:forward: {e}" + assert isinstance(e.get("weight"), (int, float)), f"Edge weight not numeric: {e}" + + # External edge: svc_a → service:external/keycloak + ext_edges = [ + e for e in cross_edges + if e.get("target") == "service:external/keycloak" + ] + assert ext_edges, "External keycloak edge not found in cross-repo edges" + + # Module→module edge: svc_a → svc_b + mm_edges = [ + e for e in cross_edges + if e.get("source") == "module:svc_a" and e.get("target") == "module:svc_b" + ] + assert mm_edges, "module:svc_a → module:svc_b cross-repo edge missing" + + # Low-confidence edge kept but tagged + lc_edges = [e for e in cross_edges if e.get("lowConfidence")] + assert lc_edges, "Low-confidence edge (confidence<0.5) should be kept with lowConfidence:true" + + +def test_validator_zero_issues(): + """(d) bundled inline validator returns issues==[] on knowledge-graph.json.""" + with tempfile.TemporaryDirectory() as tmp_str: + out = Path(tmp_str) + make_combined_graph(out) + make_crossrepo_edges(out) + + result = run_apply(out) + assert result.returncode == 0, f"apply-interlinks failed:\n{result.stderr}" + + review_path = out / ".understand-anything" / "intermediate" / "review.json" + assert review_path.exists(), "review.json not written by validator" + + review = json.loads(review_path.read_text()) + assert review.get("issues") == [], ( + f"Validator found issues: {review.get('issues')}" + ) + + +def test_meta_json_written(): + """(e) meta.json is written with required fields.""" + with tempfile.TemporaryDirectory() as tmp_str: + out = Path(tmp_str) + make_combined_graph(out) + make_crossrepo_edges(out) + + result = run_apply(out) + assert result.returncode == 0, f"apply-interlinks failed:\n{result.stderr}" + + meta_path = out / ".understand-anything" / "meta.json" + assert meta_path.exists(), "meta.json not written" + + meta = json.loads(meta_path.read_text()) + assert "lastAnalyzedAt" in meta, "meta.json missing lastAnalyzedAt" + assert meta.get("gitCommitHash") == "crossrepo", "meta.json gitCommitHash must be 'crossrepo'" + assert meta.get("version") == "1.0.0", "meta.json version must be '1.0.0'" + assert "analyzedFiles" in meta, "meta.json missing analyzedFiles" + + +def test_dedup_edges(): + """Duplicate (source,target,type) cross edges are deduped.""" + with tempfile.TemporaryDirectory() as tmp_str: + out = Path(tmp_str) + make_combined_graph(out) + + # Write crossrepo-edges with a duplicate + intermediate = out / ".understand-anything" / "intermediate" + edges = [ + {"source": "module:svc_a", "target": "module:svc_b", "type": "api-call", + "weight": 0.7, "direction": "forward", "confidence": 0.8, "evidence": "first"}, + {"source": "module:svc_a", "target": "module:svc_b", "type": "api-call", + "weight": 0.5, "direction": "forward", "confidence": 0.8, "evidence": "dup"}, + ] + (intermediate / "crossrepo-edges.json").write_text(json.dumps(edges), encoding="utf-8") + + result = run_apply(out) + assert result.returncode == 0, f"apply-interlinks failed:\n{result.stderr}" + + kg = json.loads((out / ".understand-anything" / "knowledge-graph.json").read_text()) + cross = [e for e in kg["edges"] + if e.get("source") == "module:svc_a" and e.get("target") == "module:svc_b" + and e.get("type") == "api-call"] + assert len(cross) == 1, f"Duplicate edge not deduped: found {len(cross)}" + + +if __name__ == "__main__": + tests = [ + test_external_node_and_layer_created, + test_dangling_edge_dropped, + test_valid_and_external_edges_present, + test_validator_zero_issues, + test_meta_json_written, + test_dedup_edges, + ] + passed = failed = 0 + for t in tests: + try: + t() + print(f" PASS {t.__name__}") + passed += 1 + except Exception as e: + print(f" FAIL {t.__name__}: {e}") + failed += 1 + print(f"\n{passed} passed, {failed} failed") + sys.exit(0 if failed == 0 else 1) diff --git a/understand-anything-plugin/skills/understand-crossrepo/apply-interlinks.py b/understand-anything-plugin/skills/understand-crossrepo/apply-interlinks.py new file mode 100644 index 000000000..b6c80be28 --- /dev/null +++ b/understand-anything-plugin/skills/understand-crossrepo/apply-interlinks.py @@ -0,0 +1,374 @@ +#!/usr/bin/env python3 +""" +apply-interlinks.py — Task 5 of /understand-crossrepo. + +Usage: + python apply-interlinks.py + +Reads: + /.understand-anything/intermediate/combined-graph.json + /.understand-anything/intermediate/crossrepo-edges.json + +Writes: + /.understand-anything/knowledge-graph.json + /.understand-anything/meta.json + /.understand-anything/tmp/ua-inline-validate.cjs (validator) + /.understand-anything/intermediate/review.json (validator output) + +Steps: + 1. Backfill any module: anchor nodes missing summary/tags (combine-graphs doesn't emit them). + 2. Synthesize external infra nodes + layer for any external: edge targets. + 3. Apply crossrepo edges: assign x ids, dedup, drop dangling, tag low-confidence. + 4. Assemble final knowledge-graph.json. + 5. Run bundled inline validator; assert issues == []. + 6. Write meta.json. +""" + +import json +import os +import subprocess +import sys +from datetime import datetime, timezone +from pathlib import Path + +# ── Inline validator (verbatim from skills/understand/SKILL.md Phase 6) ──────── +_VALIDATOR_CJS = r"""#!/usr/bin/env node +const fs = require('fs'); +const graphPath = process.argv[2]; +const outputPath = process.argv[3]; +try { + const graph = JSON.parse(fs.readFileSync(graphPath, 'utf8')); + const issues = [], warnings = []; + if (!Array.isArray(graph.nodes)) { issues.push('graph.nodes is missing or not an array'); graph.nodes = []; } + if (!Array.isArray(graph.edges)) { issues.push('graph.edges is missing or not an array'); graph.edges = []; } + const nodeIds = new Set(); + const seen = new Map(); + graph.nodes.forEach((n, i) => { + if (!n.id) { issues.push(`Node[${i}] missing id`); return; } + if (!n.type) issues.push(`Node[${i}] '${n.id}' missing type`); + if (!n.name) issues.push(`Node[${i}] '${n.id}' missing name`); + if (!n.summary) issues.push(`Node[${i}] '${n.id}' missing summary`); + if (!n.tags || !n.tags.length) issues.push(`Node[${i}] '${n.id}' missing tags`); + if (seen.has(n.id)) issues.push(`Duplicate node ID '${n.id}' at indices ${seen.get(n.id)} and ${i}`); + else seen.set(n.id, i); + nodeIds.add(n.id); + }); + graph.edges.forEach((e, i) => { + if (!nodeIds.has(e.source)) issues.push(`Edge[${i}] source '${e.source}' not found`); + if (!nodeIds.has(e.target)) issues.push(`Edge[${i}] target '${e.target}' not found`); + }); + const fileLevelTypes = new Set(['file', 'config', 'document', 'service', 'pipeline', 'table', 'schema', 'resource', 'endpoint']); + const fileNodes = graph.nodes.filter(n => fileLevelTypes.has(n.type)).map(n => n.id); + const assigned = new Map(); + if (!Array.isArray(graph.layers)) { if (graph.layers) warnings.push('graph.layers is not an array'); graph.layers = []; } + if (!Array.isArray(graph.tour)) { if (graph.tour) warnings.push('graph.tour is not an array'); graph.tour = []; } + graph.layers.forEach(layer => { + (layer.nodeIds || []).forEach(id => { + if (!nodeIds.has(id)) issues.push(`Layer '${layer.id}' refs missing node '${id}'`); + if (assigned.has(id)) issues.push(`Node '${id}' appears in multiple layers`); + assigned.set(id, layer.id); + }); + }); + fileNodes.forEach(id => { + if (!assigned.has(id)) issues.push(`File node '${id}' not in any layer`); + }); + graph.tour.forEach((step, i) => { + (step.nodeIds || []).forEach(id => { + if (!nodeIds.has(id)) issues.push(`Tour step[${i}] refs missing node '${id}'`); + }); + }); + const withEdges = new Set([ + ...graph.edges.map(e => e.source), + ...graph.edges.map(e => e.target) + ]); + graph.nodes.forEach(n => { + if (!withEdges.has(n.id)) warnings.push(`Node '${n.id}' has no edges (orphan)`); + }); + const stats = { + totalNodes: graph.nodes.length, + totalEdges: graph.edges.length, + totalLayers: graph.layers.length, + tourSteps: graph.tour.length, + nodeTypes: graph.nodes.reduce((a, n) => { a[n.type] = (a[n.type]||0)+1; return a; }, {}), + edgeTypes: graph.edges.reduce((a, e) => { a[e.type] = (a[e.type]||0)+1; return a; }, {}) + }; + fs.writeFileSync(outputPath, JSON.stringify({ issues, warnings, stats }, null, 2)); + process.exit(0); +} catch (err) { process.stderr.write(err.message + '\n'); process.exit(1); } +""" + +_FILE_LEVEL_TYPES = { + "file", "config", "document", "service", "pipeline", + "table", "schema", "resource", "endpoint", +} + + +def _load(path: Path) -> dict | list: + with path.open(encoding="utf-8") as f: + return json.load(f) + + +def _dump(path: Path, obj) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8") as f: + json.dump(obj, f, indent=2, ensure_ascii=False) + f.write("\n") + + +def _backfill_node(node: dict) -> dict: + """Ensure every node has non-empty summary and tags (validator requires it).""" + if not node.get("summary"): + ntype = node.get("type", "unknown") + nname = node.get("name", node["id"]) + node["summary"] = f"{ntype.capitalize()} node: {nname}" + if not node.get("tags"): + repo = node.get("repo", "") + node["tags"] = [f"repo:{repo}"] if repo else ["untagged"] + return node + + +def _svc_name_from_external(target: str) -> str: + """'external:keycloak' → 'keycloak'""" + return target.split(":", 1)[1] + + +def _make_external_node(svc: str) -> dict: + return { + "id": f"service:external/{svc}", + "type": "service", + "name": svc, + "summary": f"External shared infrastructure: {svc}", + "tags": ["external", "shared-infra"], + "repo": "external", + } + + +def apply(out: Path) -> None: + intermediate = out / ".understand-anything" / "intermediate" + ua_dir = out / ".understand-anything" + tmp_dir = ua_dir / "tmp" + tmp_dir.mkdir(parents=True, exist_ok=True) + + # ── Load inputs ──────────────────────────────────────────────────────────── + combined = _load(intermediate / "combined-graph.json") + crossrepo_raw: list[dict] = _load(intermediate / "crossrepo-edges.json") + + # ── 1. Backfill module anchors (combine-graphs omits summary) ───────────── + nodes: list[dict] = [_backfill_node(n) for n in combined.get("nodes", [])] + node_ids: set[str] = {n["id"] for n in nodes} + + intra_edges: list[dict] = list(combined.get("edges", [])) + layers: list[dict] = [ + # Ensure description field exists on every layer (validator layer structure) + {**l, "description": l.get("description") or f"Layer for {l.get('name', l['id'])}"} + for l in combined.get("layers", []) + ] + + # ── 2. Synthesize external infra nodes ──────────────────────────────────── + external_svcs: list[str] = sorted({ + _svc_name_from_external(e["target"]) + for e in crossrepo_raw + if e.get("target", "").startswith("external:") + }) + external_nodes = [_make_external_node(s) for s in external_svcs] + for en in external_nodes: + if en["id"] not in node_ids: + nodes.append(en) + node_ids.add(en["id"]) + + # External layer — all external service nodes + external_layer_node_ids = sorted(en["id"] for en in external_nodes) + if external_layer_node_ids: + layers.append({ + "id": "layer:external-shared-infra", + "name": "External / Shared Infra", + "description": "Synthetic nodes for shared external services referenced across repos.", + "nodeIds": external_layer_node_ids, + }) + + # ── 3. Apply crossrepo edges ─────────────────────────────────────────────── + # Remap external: → service:external/ + def _remap_target(target: str) -> str: + if target.startswith("external:"): + return f"service:external/{_svc_name_from_external(target)}" + return target + + # Dedup by (source, target, type) — first wins (or higher weight) + seen_keys: dict[tuple, dict] = {} + cross_edges_raw: list[dict] = [] + for e in crossrepo_raw: + src = e.get("source", "") + tgt = _remap_target(e.get("target", "")) + etype = e.get("type", "") + key = (src, tgt, etype) + if key in seen_keys: + # Keep whichever has higher weight + existing = seen_keys[key] + if (e.get("weight") or 0) > (existing.get("weight") or 0): + seen_keys[key] = {**e, "target": tgt} + else: + seen_keys[key] = {**e, "target": tgt} + + # Now filter: drop if endpoints not in final node set + # module: and service:external/ are always in node_ids by construction + cross_edges_final: list[dict] = [] + xi = 0 + for key, e in sorted(seen_keys.items()): # sort for determinism + src, tgt, etype = e["source"], e["target"], e.get("type", "") + # Drop if either endpoint is missing + if src not in node_ids or tgt not in node_ids: + continue + weight = e.get("weight") + if weight is None: + weight = 0.5 + edge = { + "id": f"x{xi}", + "source": src, + "target": tgt, + "type": etype, + "direction": "forward", + "weight": float(weight), + } + if e.get("label"): + edge["label"] = e["label"] + if e.get("confidence") is not None: + edge["confidence"] = e["confidence"] + if (e.get("confidence") or 1.0) < 0.5: + edge["lowConfidence"] = True + # fineTarget: keep coarse edge always; annotate if fine node exists + fine = e.get("fineTarget") + if fine and fine in node_ids: + edge["fineTarget"] = fine + cross_edges_final.append(edge) + xi += 1 + + all_edges = intra_edges + cross_edges_final + + # ── 4. Build tour ────────────────────────────────────────────────────────── + # Step 1: overview — all module anchors + module_anchors = sorted( + n["id"] for n in nodes if n.get("type") == "module" and not n["id"].startswith("module:external") + ) + # Step per repo: module anchor + up to 2 file-level nodes + repo_steps = [] + repo_names = [nid.split(":", 1)[1] for nid in module_anchors if ":" in nid] + for ns in repo_names: + anchor = f"module:{ns}" + # pick up to 2 file-level nodes for this repo + sample = sorted([ + n["id"] for n in nodes + if n.get("repo") == ns and n["type"] in _FILE_LEVEL_TYPES + ])[:2] + step_nodes = [anchor] + [s for s in sample if s in node_ids] + repo_steps.append({ + "order": len(repo_steps) + 2, + "title": f"{ns} service", + "description": f"Key nodes in the {ns} repo.", + "nodeIds": step_nodes, + }) + + # Final step: cross-repo interlink endpoints + interlink_endpoints = sorted({ + nid + for e in cross_edges_final + for nid in [e["source"], e["target"]] + if nid in node_ids + })[:6] + + tour = [ + { + "order": 1, + "title": "Platform Overview", + "description": "High-level view of all repos and their module anchors.", + "nodeIds": module_anchors, + }, + *repo_steps, + { + "order": len(repo_steps) + 2, + "title": "Cross-Repo Interlinks", + "description": "Nodes at the boundaries of inter-service dependencies.", + "nodeIds": interlink_endpoints, + }, + ] + + # ── 5. Assemble final graph ──────────────────────────────────────────────── + combined_project = combined.get("project") or {} + project = { + "name": f"{combined_project.get('name', 'combined')} — cross-repo", + "languages": combined_project.get("languages", []), + "frameworks": combined_project.get("frameworks", []), + "description": combined_project.get( + "description", + "Cross-repo knowledge graph combining multiple service repos.", + ), + } + + # Sort nodes and layers for determinism + final_nodes = sorted(nodes, key=lambda n: n["id"]) + final_edges = sorted(all_edges, key=lambda e: (e.get("source", ""), e.get("target", ""), e.get("type", ""))) + final_layers = sorted(layers, key=lambda l: l["id"]) + # Sort nodeIds within each layer for determinism + for layer in final_layers: + layer["nodeIds"] = sorted(layer["nodeIds"]) + + graph = { + "version": "1.0.0", + "project": project, + "nodes": final_nodes, + "edges": final_edges, + "layers": final_layers, + "tour": tour, + } + + # ── 6. Write knowledge-graph.json ───────────────────────────────────────── + kg_path = ua_dir / "knowledge-graph.json" + _dump(kg_path, graph) + + # ── 7. Run bundled inline validator ─────────────────────────────────────── + validator_path = tmp_dir / "ua-inline-validate.cjs" + validator_path.write_text(_VALIDATOR_CJS, encoding="utf-8") + + review_path = intermediate / "review.json" + result = subprocess.run( + ["node", str(validator_path), str(kg_path), str(review_path)], + capture_output=True, + text=True, + ) + if result.returncode != 0: + print(f"Validator error: {result.stderr}", file=sys.stderr) + sys.exit(1) + + review = json.loads(review_path.read_text()) + if review.get("issues"): + print(f"Validation issues: {json.dumps(review['issues'], indent=2)}", file=sys.stderr) + sys.exit(1) + + print( + f"Validation passed: {review['stats']['totalNodes']} nodes, " + f"{review['stats']['totalEdges']} edges, " + f"{review['stats']['totalLayers']} layers.", + file=sys.stderr, + ) + + # ── 8. Write meta.json ──────────────────────────────────────────────────── + node_count = len(final_nodes) + meta = { + "lastAnalyzedAt": datetime.now(tz=timezone.utc).isoformat(), + "gitCommitHash": "crossrepo", + "version": "1.0.0", + "analyzedFiles": node_count, + } + _dump(ua_dir / "meta.json", meta) + + print( + f"apply-interlinks complete: {node_count} nodes, " + f"{len(final_edges)} edges, {len(final_layers)} layers.", + file=sys.stderr, + ) + + +if __name__ == "__main__": + if len(sys.argv) != 2: + print("Usage: python apply-interlinks.py ", file=sys.stderr) + sys.exit(1) + apply(Path(sys.argv[1])) From 6e757b476ba3254f17825f71a4b2c06c7bf8218a Mon Sep 17 00:00:00 2001 From: Mohammed Hafiz <65387738+Hafiz408@users.noreply.github.com> Date: Fri, 26 Jun 2026 20:02:00 +0530 Subject: [PATCH 09/17] fix(crossrepo): tag lowConfidence for zero-confidence cross-repo edges Co-Authored-By: Claude Sonnet 4.6 --- .../__tests__/test_apply.py | 58 +++++++++++++++++++ .../understand-crossrepo/apply-interlinks.py | 2 +- 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/understand-anything-plugin/skills/understand-crossrepo/__tests__/test_apply.py b/understand-anything-plugin/skills/understand-crossrepo/__tests__/test_apply.py index cd5a0ee3b..6db76936d 100644 --- a/understand-anything-plugin/skills/understand-crossrepo/__tests__/test_apply.py +++ b/understand-anything-plugin/skills/understand-crossrepo/__tests__/test_apply.py @@ -345,6 +345,63 @@ def test_dedup_edges(): assert len(cross) == 1, f"Duplicate edge not deduped: found {len(cross)}" +def test_zero_confidence_tagged_low_confidence(): + """confidence==0.0 is falsy but must still be tagged lowConfidence; confidence>=0.5 must NOT be.""" + with tempfile.TemporaryDirectory() as tmp_str: + out = Path(tmp_str) + make_combined_graph(out) + + intermediate = out / ".understand-anything" / "intermediate" + edges = [ + # zero-confidence: falsy 0.0 must be tagged lowConfidence + { + "source": "module:svc_a", + "target": "module:svc_b", + "type": "api-call", + "weight": 0.5, + "direction": "forward", + "confidence": 0.0, + "evidence": "zero-confidence edge", + }, + # high-confidence: must NOT be tagged lowConfidence + { + "source": "module:svc_b", + "target": "module:svc_a", + "type": "event-dependency", + "weight": 0.8, + "direction": "forward", + "confidence": 0.9, + "evidence": "high-confidence edge", + }, + ] + (intermediate / "crossrepo-edges.json").write_text(json.dumps(edges), encoding="utf-8") + + result = run_apply(out) + assert result.returncode == 0, f"apply-interlinks failed:\n{result.stderr}" + + kg = json.loads((out / ".understand-anything" / "knowledge-graph.json").read_text()) + cross = [e for e in kg["edges"] if e.get("id", "").startswith("x")] + + zero_edge = next( + (e for e in cross if e.get("source") == "module:svc_a" and e.get("target") == "module:svc_b"), + None, + ) + assert zero_edge is not None, "zero-confidence edge was dropped (should be kept)" + assert zero_edge.get("confidence") == 0.0, f"confidence not stored correctly: {zero_edge}" + assert zero_edge.get("lowConfidence") is True, ( + f"confidence==0.0 edge must have lowConfidence:true, got: {zero_edge}" + ) + + high_edge = next( + (e for e in cross if e.get("source") == "module:svc_b" and e.get("target") == "module:svc_a"), + None, + ) + assert high_edge is not None, "high-confidence edge was dropped" + assert not high_edge.get("lowConfidence"), ( + f"confidence==0.9 edge must NOT have lowConfidence, got: {high_edge}" + ) + + if __name__ == "__main__": tests = [ test_external_node_and_layer_created, @@ -353,6 +410,7 @@ def test_dedup_edges(): test_validator_zero_issues, test_meta_json_written, test_dedup_edges, + test_zero_confidence_tagged_low_confidence, ] passed = failed = 0 for t in tests: diff --git a/understand-anything-plugin/skills/understand-crossrepo/apply-interlinks.py b/understand-anything-plugin/skills/understand-crossrepo/apply-interlinks.py index b6c80be28..9b9a6a751 100644 --- a/understand-anything-plugin/skills/understand-crossrepo/apply-interlinks.py +++ b/understand-anything-plugin/skills/understand-crossrepo/apply-interlinks.py @@ -233,7 +233,7 @@ def _remap_target(target: str) -> str: edge["label"] = e["label"] if e.get("confidence") is not None: edge["confidence"] = e["confidence"] - if (e.get("confidence") or 1.0) < 0.5: + if e.get("confidence") is not None and e["confidence"] < 0.5: edge["lowConfidence"] = True # fineTarget: keep coarse edge always; annotate if fine node exists fine = e.get("fineTarget") From da4585bc99be6489d2569eb867a94dc022760fd6 Mon Sep 17 00:00:00 2001 From: Mohammed Hafiz <65387738+Hafiz408@users.noreply.github.com> Date: Fri, 26 Jun 2026 20:07:13 +0530 Subject: [PATCH 10/17] feat(crossrepo): wire end-to-end orchestration in SKILL.md Co-Authored-By: Claude Sonnet 4.6 --- .../skills/understand-crossrepo/SKILL.md | 245 +++++++++++++++++- 1 file changed, 232 insertions(+), 13 deletions(-) diff --git a/understand-anything-plugin/skills/understand-crossrepo/SKILL.md b/understand-anything-plugin/skills/understand-crossrepo/SKILL.md index 9c8acc1d8..4a142a289 100644 --- a/understand-anything-plugin/skills/understand-crossrepo/SKILL.md +++ b/understand-anything-plugin/skills/understand-crossrepo/SKILL.md @@ -170,19 +170,238 @@ Report to the user: --- -## Phases 1–7 (added in later tasks) - -The following phases are scaffolded here for continuity. Their full logic is authored in Tasks 2–6. - -| Phase | Name | Task | -|-------|------|------| -| 1 | Per-repo scan | Task 2 | -| 2 | Per-repo file analysis (parallel) | Task 2 | -| 3 | Cross-repo linker — detect and emit cross-repo edges | Task 3 | -| 4 | Merge into unified graph — one node set, typed cross-repo edges | Task 4 | -| 5 | Architecture + tour over the combined graph | Task 5 | -| 6 | Review + validate the combined graph | Task 5 | -| 7 | Save `crossrepo-knowledge-graph.json` + launch dashboard | Task 6 | +## Phase 1 — Per-repo reuse-or-fill + +Report: `[Phase 1/7] Checking per-repo graphs (reuse vs. analyze)...` + +For each repo in `$REPO_NAMESPACES` (namespace → absolute path), decide whether its single-repo knowledge graph is fresh enough to reuse, or must be (re-)built. + +**For each repo:** + +```bash +# Read existing meta (may not exist) +META="/.understand-anything/meta.json" +CURRENT_HASH=$(git -C "" rev-parse HEAD 2>/dev/null || echo "") +if [ -f "$META" ]; then + STORED_HASH=$(python3 -c "import json,sys; print(json.load(open('$META')).get('gitCommitHash',''))" 2>/dev/null || echo "") +else + STORED_HASH="" +fi +``` + +| Condition | Action | +|-----------|--------| +| `$META` missing | Run `/understand ` (full analysis) | +| `$STORED_HASH` ≠ `$CURRENT_HASH` | Run `/understand ` (stale graph) | +| Hashes match | Reuse existing `/.understand-anything/knowledge-graph.json` | + +To trigger analysis, invoke the `/understand` skill passing `` as its argument. Wait for it to complete before proceeding to the next repo. (Repos are analyzed sequentially here to avoid saturating LLM quota; the signal extraction in Phase 2 is fast and does not need parallelism.) + +After processing all repos, report: + +> Phase 1 complete. Reused: [list of reused namespaces]. Analyzed: [list of analyzed namespaces (or "none").]. + +Collect any per-repo failures in `$PHASE_WARNINGS`. A repo whose `/understand` run fails should be skipped with a warning — do not STOP the whole run for a single repo failure. However, if **all** repos fail, report the errors and STOP. + +--- + +## Phase 2 — Extract signals + +Report: `[Phase 2/7] Extracting cross-repo signals...` + +`$SKILL_DIR` is the directory containing this SKILL.md file — resolve it the same way Phase 0 resolved `$PLUGIN_ROOT` (using the `realpath`/`readlink -f` pattern on `~/.agents/skills/understand-crossrepo`, then falling back to common paths). + +For each repo (namespace ``, path ``): + +```bash +node "$SKILL_DIR/extract-crossrepo-signals.mjs" \ + "" \ + "" \ + "$OUT_DIR/.understand-anything/intermediate/signals-.json" +``` + +Run all repos sequentially. If the extractor exits non-zero for a repo, add the error to `$PHASE_WARNINGS` and continue — a repo with no signals simply contributes nothing to cross-repo linking. + +After all repos are done: + +> Phase 2 complete. Signals extracted for: [list of namespaces with non-empty outbound arrays]. No outbound signals: [list or "none"]. + +--- + +## Phase 3 — Combine graphs + +Report: `[Phase 3/7] Combining per-repo graphs into unified substrate...` + +Build the `:` argument list from `$REPO_NAMESPACES`: + +```bash +python3 "$SKILL_DIR/combine-graphs.py" "$OUT_DIR" \ + ":" \ + ":" \ + ... +``` + +This writes: +- `$OUT_DIR/.understand-anything/intermediate/combined-graph.json` +- `$OUT_DIR/.understand-anything/intermediate/id-map.json` + +If `combine-graphs.py` exits non-zero, read stderr, report it, and STOP — the downstream steps cannot run without a combined graph. + +> Phase 3 complete. Combined graph written with [N] repos, [M] total nodes. + +(Read `M` from `combined-graph.json`: `len(graph["nodes"])`.) + +--- + +## Phase 4 — Cross-repo linking (LLM) + +Report: `[Phase 4/7] Dispatching cross-repo linker agent...` + +Prepare the linker's input by reading: +1. All `signals-.json` files from `$OUT_DIR/.understand-anything/intermediate/`. +2. From `combined-graph.json`: collect every node whose `id` starts with `module:` or `endpoint:`. + +Dispatch a subagent using the `crossrepo-linker` agent definition (at `agents/crossrepo-linker.md` relative to `$PLUGIN_ROOT`). Pass this prompt to the agent: + +> You are the cross-repo linker. Your output path is: +> `$OUT_DIR/.understand-anything/intermediate/crossrepo-edges.json` +> +> **Signal files** (one per repo): +> ```json +> [paste full JSON contents of each signals-.json, keyed by namespace] +> ``` +> +> **Available anchor node IDs from combined-graph.json** (module + endpoint nodes): +> ``` +> [one node id per line — all ids starting with "module:" or "endpoint:"] +> ``` +> +> Follow the matching rules in your agent definition and WRITE your edge array to the output path above. + +Wait for the agent to complete. Verify that `crossrepo-edges.json` now exists and is valid JSON containing an array. If the file is missing or invalid, retry the dispatch **once** with the failure appended to the prompt. If still failing after the retry, write an empty array `[]` to `crossrepo-edges.json`, add a warning to `$PHASE_WARNINGS`, and continue. + +> Phase 4 complete. [N] cross-repo edges emitted by linker. + +(Read `N` from `crossrepo-edges.json`: `len(edges)`.) + +--- + +## Phase 5 — Apply, assemble, and validate + +Report: `[Phase 5/7] Applying cross-repo edges and assembling final graph...` + +```bash +python3 "$SKILL_DIR/apply-interlinks.py" "$OUT_DIR" +``` + +This script: +1. Backfills missing summaries/tags on `module:` anchor nodes. +2. Synthesizes `external:*` infra nodes for unmatched edge targets. +3. Applies and deduplicates cross-repo edges from `crossrepo-edges.json`. +4. Assembles `$OUT_DIR/.understand-anything/knowledge-graph.json`. +5. Runs the inline validator; writes results to `$OUT_DIR/.understand-anything/intermediate/review.json`. +6. Writes `$OUT_DIR/.understand-anything/meta.json`. + +If `apply-interlinks.py` exits non-zero, read stderr, report it, and STOP. + +After it completes, read `review.json`: + +```bash +python3 -c " +import json +r = json.load(open('$OUT_DIR/.understand-anything/intermediate/review.json')) +print('issues:', r.get('issues', [])) +print('warnings:', r.get('warnings', [])) +" +``` + +**If `issues` is non-empty:** +- Report each issue to the user verbatim. +- Set `$VALIDATION_PASSED=false`. +- Tell the user: "Validation issues were found. The graph has been saved but the dashboard will not auto-launch. Fix the issues above and re-run `/understand-crossrepo` to rebuild." + +**If `issues` is empty:** +- Set `$VALIDATION_PASSED=true`. +- > Phase 5 complete. Graph assembled and validated. [N] nodes, [M] edges (including [X] cross-repo edges). [W] warnings. + +--- + +## Phase 6 — Summary report + +Report: `[Phase 6/7] Building summary report...` + +Read the final graph and intermediate files: + +```bash +python3 - <<'EOF' +import json, sys + +graph = json.load(open("$OUT_DIR/.understand-anything/knowledge-graph.json")) +try: + edges_raw = json.load(open("$OUT_DIR/.understand-anything/intermediate/crossrepo-edges.json")) +except Exception: + edges_raw = [] + +nodes = graph.get("nodes", []) +edges = graph.get("edges", []) +layers = graph.get("layers", []) + +# Cross-repo edges: those whose source and target are in different namespaces +def ns(nid): + # module:ns/... → ns; otherwise first segment after colon up to / + parts = nid.split(":", 1) + if len(parts) < 2: + return "" + rest = parts[1] + return rest.split("/")[0] + +cross_edges = [e for e in edges if ns(e.get("source","")) != ns(e.get("target",""))] +low_conf = [e for e in edges_raw if e.get("confidence","high") == "low"] + +print(f"Nodes: {len(nodes)}") +print(f"Edges total: {len(edges)}") +print(f"Cross-repo edges: {len(cross_edges)}") +print(f"Low-confidence edges (linker): {len(low_conf)}") +print(f"Layers: {len(layers)} — {[l.get('id') for l in layers]}") +EOF +``` + +Report to the user: + +``` +Cross-repo graph summary +======================== +Repos analyzed (fresh): [list from Phase 1] +Repos reused (cached): [list from Phase 1] + +Layers: [one per repo namespace + external layer if any] +Nodes: [total] +Edges: [total] ([X] cross-repo, [W] low-confidence) + +Output: $OUT_DIR/.understand-anything/knowledge-graph.json + +Warnings accumulated: [list from $PHASE_WARNINGS, or "none"] +``` + +> Phase 6 complete. + +--- + +## Phase 7 — Dashboard + +Report: `[Phase 7/7] Launching dashboard...` + +**Only proceed if `$VALIDATION_PASSED=true`** (set in Phase 5). + +Invoke the `/understand-dashboard` skill, passing `$OUT_DIR` as its project-path argument: + +> `/understand-dashboard $OUT_DIR` + +Do not hand-roll a Vite or Node server command — reuse the skill as-is. The skill reads `$OUT_DIR/.understand-anything/knowledge-graph.json` and launches the interactive explorer. + +If `$VALIDATION_PASSED=false`, skip this phase and remind the user: + +> "Dashboard launch skipped due to validation issues. Fix the issues reported in Phase 5 and re-run `/understand-crossrepo` to rebuild." --- From cf6ef57b5b8c0e3663d4e4a5e50348ee380d2630 Mon Sep 17 00:00:00 2001 From: Mohammed Hafiz <65387738+Hafiz408@users.noreply.github.com> Date: Fri, 26 Jun 2026 20:13:07 +0530 Subject: [PATCH 11/17] fix(crossrepo): expand OUT_DIR in phase6, count cross-repo edges from source, bind phase2 vars Co-Authored-By: Claude Sonnet 4.6 --- .../skills/understand-crossrepo/SKILL.md | 49 ++++++++++--------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/understand-anything-plugin/skills/understand-crossrepo/SKILL.md b/understand-anything-plugin/skills/understand-crossrepo/SKILL.md index 4a142a289..d45325707 100644 --- a/understand-anything-plugin/skills/understand-crossrepo/SKILL.md +++ b/understand-anything-plugin/skills/understand-crossrepo/SKILL.md @@ -209,15 +209,19 @@ Collect any per-repo failures in `$PHASE_WARNINGS`. A repo whose `/understand` r Report: `[Phase 2/7] Extracting cross-repo signals...` -`$SKILL_DIR` is the directory containing this SKILL.md file — resolve it the same way Phase 0 resolved `$PLUGIN_ROOT` (using the `realpath`/`readlink -f` pattern on `~/.agents/skills/understand-crossrepo`, then falling back to common paths). +`$SKILL_DIR` is `$PLUGIN_ROOT/skills/understand-crossrepo` (since `$PLUGIN_ROOT` was already resolved in Phase 0). -For each repo (namespace ``, path ``): +For each repo, bind `REPO_PATH` and `NS` from Phase 0's `$REPO_NAMESPACES` mapping (namespace → absolute path), then run: ```bash +# For each repo in $REPO_NAMESPACES: +REPO_PATH="" +NS="" + node "$SKILL_DIR/extract-crossrepo-signals.mjs" \ - "" \ - "" \ - "$OUT_DIR/.understand-anything/intermediate/signals-.json" + "$REPO_PATH" \ + "$NS" \ + "$OUT_DIR/.understand-anything/intermediate/signals-$NS.json" ``` Run all repos sequentially. If the extractor exits non-zero for a repo, add the error to `$PHASE_WARNINGS` and continue — a repo with no signals simply contributes nothing to cross-repo linking. @@ -232,7 +236,7 @@ After all repos are done: Report: `[Phase 3/7] Combining per-repo graphs into unified substrate...` -Build the `:` argument list from `$REPO_NAMESPACES`: +Build the `:` argument list from `$REPO_NAMESPACES` (each arg is the absolute path colon-joined with its namespace, e.g. `/workspace/savo_gemba_service:savo_gemba_service`): ```bash python3 "$SKILL_DIR/combine-graphs.py" "$OUT_DIR" \ @@ -304,17 +308,22 @@ This script: If `apply-interlinks.py` exits non-zero, read stderr, report it, and STOP. -After it completes, read `review.json`: +After it completes, read `review.json` and the assembled graph: ```bash python3 -c " import json r = json.load(open('$OUT_DIR/.understand-anything/intermediate/review.json')) +g = json.load(open('$OUT_DIR/.understand-anything/knowledge-graph.json')) +print('nodes:', len(g.get('nodes', []))) +print('edges:', len(g.get('edges', []))) print('issues:', r.get('issues', [])) print('warnings:', r.get('warnings', [])) " ``` +Report the node and edge counts to the user unconditionally (before branching on issues). + **If `issues` is non-empty:** - Report each issue to the user verbatim. - Set `$VALIDATION_PASSED=false`. @@ -330,15 +339,17 @@ print('warnings:', r.get('warnings', [])) Report: `[Phase 6/7] Building summary report...` -Read the final graph and intermediate files: +Read the final graph and intermediate files (pass `$OUT_DIR` via argv so the quoted heredoc does not shell-expand it): ```bash -python3 - <<'EOF' +python3 - "$OUT_DIR" <<'EOF' import json, sys -graph = json.load(open("$OUT_DIR/.understand-anything/knowledge-graph.json")) +out_dir = sys.argv[1] + +graph = json.load(open(f"{out_dir}/.understand-anything/knowledge-graph.json")) try: - edges_raw = json.load(open("$OUT_DIR/.understand-anything/intermediate/crossrepo-edges.json")) + edges_raw = json.load(open(f"{out_dir}/.understand-anything/intermediate/crossrepo-edges.json")) except Exception: edges_raw = [] @@ -346,21 +357,13 @@ nodes = graph.get("nodes", []) edges = graph.get("edges", []) layers = graph.get("layers", []) -# Cross-repo edges: those whose source and target are in different namespaces -def ns(nid): - # module:ns/... → ns; otherwise first segment after colon up to / - parts = nid.split(":", 1) - if len(parts) < 2: - return "" - rest = parts[1] - return rest.split("/")[0] - -cross_edges = [e for e in edges if ns(e.get("source","")) != ns(e.get("target",""))] -low_conf = [e for e in edges_raw if e.get("confidence","high") == "low"] +# Cross-repo edge count comes from the linker's emitted file — authoritative source +cross_edge_count = len(edges_raw) +low_conf = [e for e in edges_raw if e.get("confidence", "high") == "low"] print(f"Nodes: {len(nodes)}") print(f"Edges total: {len(edges)}") -print(f"Cross-repo edges: {len(cross_edges)}") +print(f"Cross-repo edges: {cross_edge_count}") print(f"Low-confidence edges (linker): {len(low_conf)}") print(f"Layers: {len(layers)} — {[l.get('id') for l in layers]}") EOF From 9802a62e88b1656bd1f2f05a4b34752c8c41f2dd Mon Sep 17 00:00:00 2001 From: Mohammed Hafiz <65387738+Hafiz408@users.noreply.github.com> Date: Fri, 26 Jun 2026 20:25:40 +0530 Subject: [PATCH 12/17] fix(crossrepo): skip dependency lock files in signal extractor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lock files (package-lock.json, pnpm-lock.yaml, …) are huge and full of registry/vcs URLs (github.com) that are never real integration signals. Found during e2e: pricing_ui emitted 260 outbound signals, ~all github.com from package-lock.json. Skip by exact basename (their .json/.yaml extensions slip past the existing .lock BINARY_EXTS guard). pricing_ui 260→18 signals. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../understand-crossrepo/extract-crossrepo-signals.mjs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/understand-anything-plugin/skills/understand-crossrepo/extract-crossrepo-signals.mjs b/understand-anything-plugin/skills/understand-crossrepo/extract-crossrepo-signals.mjs index 6d35609c3..f5ce4939e 100644 --- a/understand-anything-plugin/skills/understand-crossrepo/extract-crossrepo-signals.mjs +++ b/understand-anything-plugin/skills/understand-crossrepo/extract-crossrepo-signals.mjs @@ -40,6 +40,14 @@ const SKIP_DIRS = new Set([ '.understand-anything', 'venv', '.venv', '__pycache__', ]); +// Dependency lock files: huge, full of registry/vcs URLs (e.g. github.com), never +// a real integration signal. Skipped by exact basename (their extensions are +// .json/.yaml so BINARY_EXTS '.lock' alone doesn't catch them). +const SKIP_FILES = new Set([ + 'package-lock.json', 'pnpm-lock.yaml', 'yarn.lock', 'npm-shrinkwrap.json', + 'poetry.lock', 'Pipfile.lock', 'composer.lock', 'Cargo.lock', 'Gemfile.lock', +]); + // Files whose names match (case-insensitive) are scanned for env/config signals const ENV_FILE_RE = /^\.env(\.|$)/i; const CONFIG_FILE_RE = /^config\./i; @@ -125,6 +133,7 @@ function walkRepo(repoPath) { if (entry.isDirectory()) { walk(abs); } else if (entry.isFile()) { + if (SKIP_FILES.has(entry.name)) continue; const ext = extname(entry.name).toLowerCase(); if (BINARY_EXTS.has(ext)) continue; let size = 0; From 030fc024c41cf5885a09b527eaaa03183661a6fd Mon Sep 17 00:00:00 2001 From: Mohammed Hafiz <65387738+Hafiz408@users.noreply.github.com> Date: Fri, 26 Jun 2026 20:25:40 +0530 Subject: [PATCH 13/17] fix(crossrepo): assign all validator file-level node types to repo layers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit combine-graphs.py picked layer members by id-prefix (file/endpoint/service), but the canonical graph validator treats config/document/pipeline/table/schema/ resource as file-level too — every such node must be in exactly one layer. Found during e2e: 59 'File node not in any layer' validation issues (config, document, pipeline, resource). Fix: select layer members by node.type against the validator's full FILE_LEVEL_TYPES set (a pipeline node can carry a step: id, so type — not id-prefix — is authoritative). Regression test covers a config node and a pipeline node with a step: id. Final graph now validates 0 issues. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../__tests__/test_combine.py | 27 +++++++++++++++++++ .../understand-crossrepo/combine-graphs.py | 18 +++++++++---- 2 files changed, 40 insertions(+), 5 deletions(-) diff --git a/understand-anything-plugin/skills/understand-crossrepo/__tests__/test_combine.py b/understand-anything-plugin/skills/understand-crossrepo/__tests__/test_combine.py index 00a3b50ef..c2256626f 100644 --- a/understand-anything-plugin/skills/understand-crossrepo/__tests__/test_combine.py +++ b/understand-anything-plugin/skills/understand-crossrepo/__tests__/test_combine.py @@ -68,6 +68,24 @@ def make_repo(tmp: Path, ns: str) -> Path: "summary": "Item business logic", "tags": [], }, + { + "id": "config:values-staging.yaml", + "type": "config", + "name": "values-staging.yaml", + "filePath": "deploy/values-staging.yaml", + "summary": "Helm values", + "tags": [], + }, + { + # id-prefix (step) deliberately differs from type (pipeline) — + # layer membership must key off type, not the id prefix. + "id": "step:ci.yml:build", + "type": "pipeline", + "name": "build", + "filePath": "ci.yml", + "summary": "CI build step", + "tags": [], + }, ], "edges": [ { @@ -160,6 +178,15 @@ def test_both_layers_present_with_correct_nodeids(): assert "service:repo_a/ItemService" in layer_a["nodeIds"], \ f"service node missing from layer_a nodeIds: {layer_a['nodeIds']}" + # ALL validator file-level types must be in the layer, keyed by node.type + # not id-prefix — else the assembled graph fails "File node not in any layer". + # config-typed node: + assert "config:repo_a/values-staging.yaml" in layer_a["nodeIds"], \ + f"config node missing from layer_a nodeIds: {layer_a['nodeIds']}" + # pipeline-typed node whose id-prefix is 'step' (type-not-prefix regression): + assert "step:repo_a/ci.yml:build" in layer_a["nodeIds"], \ + f"pipeline (step:) node missing from layer_a nodeIds: {layer_a['nodeIds']}" + # module anchor must be in each layer assert "module:repo_a" in layer_a["nodeIds"], f"module:repo_a missing from layer nodeIds: {layer_a['nodeIds']}" assert "module:repo_b" in layer_b["nodeIds"], f"module:repo_b missing from layer nodeIds: {layer_b['nodeIds']}" diff --git a/understand-anything-plugin/skills/understand-crossrepo/combine-graphs.py b/understand-anything-plugin/skills/understand-crossrepo/combine-graphs.py index 12f4de7a0..d5fd7ec58 100644 --- a/understand-anything-plugin/skills/understand-crossrepo/combine-graphs.py +++ b/understand-anything-plugin/skills/understand-crossrepo/combine-graphs.py @@ -61,10 +61,18 @@ def _is_scan_artifact(node: dict) -> bool: return False -def _is_top_level(namespaced_id: str) -> bool: - """True for file/endpoint/service nodes — not function/class members.""" - kind = namespaced_id.split(":")[0] - return kind in ("file", "endpoint", "service") +# File-level node types per the canonical graph validator +# (skills/understand/SKILL.md Phase 6). EVERY node of one of these types must +# belong to exactly one layer or the validator rejects the graph. Match by +# node.type, NOT id-prefix: e.g. a `pipeline`-type node can have a `step:` id. +FILE_LEVEL_TYPES = frozenset( + {"file", "config", "document", "service", "pipeline", "table", "schema", "resource", "endpoint"} +) + + +def _is_top_level(node: dict) -> bool: + """True for file-level nodes (validator's set) — not function/class members.""" + return node.get("type") in FILE_LEVEL_TYPES def load_graph(path: Path) -> dict | None: @@ -147,7 +155,7 @@ def namespace_repo(graph: dict, ns: str) -> tuple[dict, dict]: def build_layer(ns: str, ns_nodes: list[dict]) -> dict: """One layer: whose nodeIds = file-level nodes + module: anchor.""" top_level_ids = sorted( - n["id"] for n in ns_nodes if _is_top_level(n["id"]) + n["id"] for n in ns_nodes if _is_top_level(n) ) anchor_id = f"module:{ns}" node_ids = sorted(set(top_level_ids + [anchor_id])) From 5a1d57b273adeb5c5f2febe7b2f6836ae1ac4f03 Mon Sep 17 00:00:00 2001 From: Mohammed Hafiz <65387738+Hafiz408@users.noreply.github.com> Date: Fri, 26 Jun 2026 20:42:24 +0530 Subject: [PATCH 14/17] fix(crossrepo): make assembled graph pass the dashboard's core/schema.ts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The assembler validated only against the inline .cjs, which doesn't check the dashboard's stricter core/schema.ts. The e2e dashboard load dropped 270 edges/ nodes. Three real gaps, all fixed in apply-interlinks.py: - project metadata: add analyzedAt + gitCommitHash (ProjectMetaSchema requires them; without them the graph fails fatally to load). - node normalization: ensure every node has a valid complexity, and drop a string lineRange (some per-repo graphs store lineRange as a string, which fails node validation → the node is dropped → its edges cascade-drop). - edge types: map linker types not in EdgeTypeSchema (authenticates_via, embeds) to depends_on, preserving the original semantic in description (which survives schema stripping; label does not). All other linker types already pass. Dashboard now loads with 0 validation issues; all 4 layers + 10 cross-repo edges render. Regression test test_dashboard_schema_requirements locks all three. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../__tests__/test_apply.py | 64 +++++++++++++++++++ .../understand-crossrepo/apply-interlinks.py | 57 +++++++++++++++-- 2 files changed, 114 insertions(+), 7 deletions(-) diff --git a/understand-anything-plugin/skills/understand-crossrepo/__tests__/test_apply.py b/understand-anything-plugin/skills/understand-crossrepo/__tests__/test_apply.py index 6db76936d..bc35e455b 100644 --- a/understand-anything-plugin/skills/understand-crossrepo/__tests__/test_apply.py +++ b/understand-anything-plugin/skills/understand-crossrepo/__tests__/test_apply.py @@ -402,8 +402,72 @@ def test_zero_confidence_tagged_low_confidence(): ) +def test_dashboard_schema_requirements(): + """Lock the e2e-found fixes: the assembled graph must satisfy the dashboard's + stricter core/schema.ts (which the inline .cjs does NOT check): + - project carries analyzedAt + gitCommitHash + - every node has a valid `complexity` + - a string `lineRange` (some per-repo graphs store it as a string) is dropped + rather than left to fail node validation (which cascades to dropped edges) + - linker edge types not in the schema enum (authenticates_via/embeds) are + mapped to an allowed type with the original semantic kept in `description`. + """ + with tempfile.TemporaryDirectory() as tmp_str: + out = Path(tmp_str) / "out" + intermediate = out / ".understand-anything" / "intermediate" + intermediate.mkdir(parents=True, exist_ok=True) + graph = { + "version": "1.0.0", + "project": {"name": "x", "languages": [], "frameworks": [], "description": "d"}, + "nodes": [ + { # malformed lineRange (string) + missing complexity + "id": "file:svc_a/m.py", "type": "file", "name": "m.py", + "filePath": "svc_a/m.py", "summary": "s", "tags": ["repo:svc_a"], + "repo": "svc_a", "lineRange": "1-50", + }, + {"id": "module:svc_a", "type": "module", "name": "svc_a", + "summary": "anchor", "tags": ["repo:svc_a"], "repo": "svc_a"}, + ], + "edges": [{"source": "module:svc_a", "target": "file:svc_a/m.py", + "type": "contains", "direction": "forward", "weight": 0.8}], + "layers": [{"id": "layer:svc_a", "name": "svc_a", "description": "l", + "nodeIds": ["file:svc_a/m.py", "module:svc_a"]}], + } + (intermediate / "combined-graph.json").write_text(json.dumps(graph), encoding="utf-8") + (intermediate / "crossrepo-edges.json").write_text(json.dumps([ + {"source": "module:svc_a", "target": "external:keycloak", + "type": "authenticates_via", "label": "svc_a auth via Keycloak", + "weight": 0.8, "direction": "forward", "confidence": 0.9, "evidence": "OIDC"}, + ]), encoding="utf-8") + + proc = run_apply(out) + assert proc.returncode == 0, f"apply failed: {proc.stderr}" + kg = json.loads((out / ".understand-anything" / "knowledge-graph.json").read_text()) + + # project metadata required by core/schema.ts ProjectMetaSchema + assert kg["project"].get("analyzedAt"), "project.analyzedAt missing" + assert kg["project"].get("gitCommitHash"), "project.gitCommitHash missing" + + # every node must have a valid complexity; no node may carry a string lineRange + for n in kg["nodes"]: + assert n.get("complexity") in ("simple", "moderate", "complex"), \ + f"node {n['id']} bad complexity: {n.get('complexity')}" + lr = n.get("lineRange") + assert lr is None or (isinstance(lr, list) and len(lr) == 2), \ + f"node {n['id']} has invalid lineRange survived: {lr}" + bad = next(n for n in kg["nodes"] if n["id"] == "file:svc_a/m.py") + assert "lineRange" not in bad, "string lineRange should have been dropped" + + # authenticates_via must be remapped to an allowed type with semantic in description + auth = next(e for e in kg["edges"] if e.get("target") == "service:external/keycloak") + assert auth["type"] == "depends_on", f"expected mapped depends_on, got {auth['type']}" + assert "authenticates_via" in (auth.get("description") or ""), \ + f"original type not preserved in description: {auth.get('description')}" + + if __name__ == "__main__": tests = [ + test_dashboard_schema_requirements, test_external_node_and_layer_created, test_dangling_edge_dropped, test_valid_and_external_edges_present, diff --git a/understand-anything-plugin/skills/understand-crossrepo/apply-interlinks.py b/understand-anything-plugin/skills/understand-crossrepo/apply-interlinks.py index 9b9a6a751..4a888f6c2 100644 --- a/understand-anything-plugin/skills/understand-crossrepo/apply-interlinks.py +++ b/understand-anything-plugin/skills/understand-crossrepo/apply-interlinks.py @@ -116,7 +116,13 @@ def _dump(path: Path, obj) -> None: def _backfill_node(node: dict) -> dict: - """Ensure every node has non-empty summary and tags (validator requires it).""" + """Normalize a node so it passes the dashboard's core/schema.ts GraphNodeSchema. + + The inline .cjs validator only checks summary/tags, but the dashboard's stricter + schema also requires `complexity` and rejects a non-tuple `lineRange` (some + per-repo graphs store lineRange as a string) — a rejected node is dropped, which + cascades into dropped edges. Fix both here. + """ if not node.get("summary"): ntype = node.get("type", "unknown") nname = node.get("name", node["id"]) @@ -124,9 +130,29 @@ def _backfill_node(node: dict) -> dict: if not node.get("tags"): repo = node.get("repo", "") node["tags"] = [f"repo:{repo}"] if repo else ["untagged"] + if node.get("complexity") not in ("simple", "moderate", "complex"): + node["complexity"] = "moderate" + # lineRange must be [int, int] or absent (it's optional) — drop anything else. + lr = node.get("lineRange") + if lr is not None and not ( + isinstance(lr, (list, tuple)) and len(lr) == 2 + and all(isinstance(x, (int, float)) and not isinstance(x, bool) for x in lr) + ): + node.pop("lineRange", None) return node +# The cross-repo linker emits semantic types (`authenticates_via`, `embeds`) that +# are NOT in the dashboard's core/schema.ts EdgeTypeSchema enum — such edges get +# dropped on load. Map the unsupported ones to the nearest allowed type and keep +# the original meaning in the edge label. All other linker types (calls, depends_on, +# reads_from, writes_to, publishes, subscribes) are already valid and pass through. +_EDGE_TYPE_MAP = { + "authenticates_via": "depends_on", + "embeds": "depends_on", +} + + def _svc_name_from_external(target: str) -> str: """'external:keycloak' → 'keycloak'""" return target.split(":", 1)[1] @@ -170,7 +196,7 @@ def apply(out: Path) -> None: for e in crossrepo_raw if e.get("target", "").startswith("external:") }) - external_nodes = [_make_external_node(s) for s in external_svcs] + external_nodes = [_backfill_node(_make_external_node(s)) for s in external_svcs] for en in external_nodes: if en["id"] not in node_ids: nodes.append(en) @@ -221,16 +247,25 @@ def _remap_target(target: str) -> str: weight = e.get("weight") if weight is None: weight = 0.5 + # Map linker types the dashboard schema doesn't know to an allowed type, + # preserving the original semantic in the label. + mapped_type = _EDGE_TYPE_MAP.get(etype, etype) + label = e.get("label") + if mapped_type != etype: + label = f"{etype}: {label}" if label else etype edge = { "id": f"x{xi}", "source": src, "target": tgt, - "type": etype, + "type": mapped_type, "direction": "forward", "weight": float(weight), } - if e.get("label"): - edge["label"] = e["label"] + if label: + # `description` survives the dashboard schema (which strips unknown edge + # fields like `label`); keep `label` too for the raw/intermediate file. + edge["label"] = label + edge["description"] = label if e.get("confidence") is not None: edge["confidence"] = e["confidence"] if e.get("confidence") is not None and e["confidence"] < 0.5: @@ -292,6 +327,9 @@ def _remap_target(target: str) -> str: ] # ── 5. Assemble final graph ──────────────────────────────────────────────── + # One timestamp + commit shared by project metadata and meta.json. + analyzed_at = datetime.now(tz=timezone.utc).isoformat() + git_commit_hash = "crossrepo" combined_project = combined.get("project") or {} project = { "name": f"{combined_project.get('name', 'combined')} — cross-repo", @@ -301,6 +339,11 @@ def _remap_target(target: str) -> str: "description", "Cross-repo knowledge graph combining multiple service repos.", ), + # Required by the dashboard's core/schema.ts ProjectMetaSchema — the inline + # .cjs validator doesn't check these, but the dashboard rejects the graph + # without them ("Missing or invalid project metadata"). + "analyzedAt": analyzed_at, + "gitCommitHash": git_commit_hash, } # Sort nodes and layers for determinism @@ -353,8 +396,8 @@ def _remap_target(target: str) -> str: # ── 8. Write meta.json ──────────────────────────────────────────────────── node_count = len(final_nodes) meta = { - "lastAnalyzedAt": datetime.now(tz=timezone.utc).isoformat(), - "gitCommitHash": "crossrepo", + "lastAnalyzedAt": analyzed_at, + "gitCommitHash": git_commit_hash, "version": "1.0.0", "analyzedFiles": node_count, } From 1edc1add04e28b5e7a93c6d74253daba178d74d3 Mon Sep 17 00:00:00 2001 From: Mohammed Hafiz <65387738+Hafiz408@users.noreply.github.com> Date: Fri, 26 Jun 2026 20:51:14 +0530 Subject: [PATCH 15/17] fix(crossrepo): correct phase6 low-confidence/edge counts, unique tour order, drop dead regexes Co-Authored-By: Claude Sonnet 4.6 --- .../skills/understand-crossrepo/SKILL.md | 7 ++-- .../understand-crossrepo/apply-interlinks.py | 4 +++ .../extract-crossrepo-signals.mjs | 33 ++----------------- 3 files changed, 10 insertions(+), 34 deletions(-) diff --git a/understand-anything-plugin/skills/understand-crossrepo/SKILL.md b/understand-anything-plugin/skills/understand-crossrepo/SKILL.md index d45325707..71f3954c6 100644 --- a/understand-anything-plugin/skills/understand-crossrepo/SKILL.md +++ b/understand-anything-plugin/skills/understand-crossrepo/SKILL.md @@ -357,9 +357,10 @@ nodes = graph.get("nodes", []) edges = graph.get("edges", []) layers = graph.get("layers", []) -# Cross-repo edge count comes from the linker's emitted file — authoritative source -cross_edge_count = len(edges_raw) -low_conf = [e for e in edges_raw if e.get("confidence", "high") == "low"] +# Cross-repo edges actually applied (assigned x ids by apply-interlinks.py) +kg_edges = graph.get("edges", []) +cross_edge_count = len([e for e in kg_edges if str(e.get("id", "")).startswith("x")]) +low_conf = [e for e in edges_raw if isinstance(e.get("confidence"), (int, float)) and e["confidence"] < 0.5] print(f"Nodes: {len(nodes)}") print(f"Edges total: {len(edges)}") diff --git a/understand-anything-plugin/skills/understand-crossrepo/apply-interlinks.py b/understand-anything-plugin/skills/understand-crossrepo/apply-interlinks.py index 4a888f6c2..b0e4ef54a 100644 --- a/understand-anything-plugin/skills/understand-crossrepo/apply-interlinks.py +++ b/understand-anything-plugin/skills/understand-crossrepo/apply-interlinks.py @@ -325,6 +325,10 @@ def _remap_target(target: str) -> str: "nodeIds": interlink_endpoints, }, ] + # Renumber tour sequentially so order is unique and 1-based regardless of how + # repo_steps were accumulated. + for i, step in enumerate(tour): + step["order"] = i + 1 # ── 5. Assemble final graph ──────────────────────────────────────────────── # One timestamp + commit shared by project metadata and meta.json. diff --git a/understand-anything-plugin/skills/understand-crossrepo/extract-crossrepo-signals.mjs b/understand-anything-plugin/skills/understand-crossrepo/extract-crossrepo-signals.mjs index f5ce4939e..9adbdf8d7 100644 --- a/understand-anything-plugin/skills/understand-crossrepo/extract-crossrepo-signals.mjs +++ b/understand-anything-plugin/skills/understand-crossrepo/extract-crossrepo-signals.mjs @@ -56,48 +56,19 @@ const VALUES_YAML_RE = /^values.*\.ya?ml$/i; // Env key patterns that indicate outbound service URLs const OUTBOUND_KEY_RE = /_(URL|HOST|ENDPOINT|BASE)$/i; -// Literal URL pattern in source/config -const LITERAL_URL_RE = /https?:\/\/([a-zA-Z0-9._-]+(?:\/[^\s"'`<>]*)?)/g; - -// FastAPI / Flask route decorators -const PY_ROUTE_RE = /@(?:app|router)\.(get|post|put|patch|delete|options|head)\(["']([^"']+)["']/g; - -// Express routes: router.get('/path') or app.post('/path') -const JS_ROUTE_RE = /(?:router|app)\.(get|post|put|patch|delete|options|head)\(["'`]([^"'`]+)["'`]/g; - -// Router prefix in Python: APIRouter(prefix="...") -const PY_ROUTER_PREFIX_RE = /APIRouter\s*\([^)]*prefix\s*=\s*["']([^"']+)["']/g; - -// Keycloak realm, client_id / clientId / resource +// Keycloak realm, client_id / clientId / resource (used in scanKeycloak) const KC_REALM_RE = /"?realm"?\s*[:=]\s*["']([^"']+)["']/; const KC_CLIENT_RE = /"?(?:client_id|clientId|resource)"?\s*[:=]\s*["']([^"']+)["']/; const KC_AUTH_URL_RE = /"?(?:auth-server-url|authServerUrl|issuer)"?\s*[:=]\s*["']([^"']+)["']/; const KC_CLIENT_ENV_RE = /KEYCLOAK_CLIENT(?:_ID)?\s*=\s*(.+)/i; const JWKS_RE = /(?:jwks|\.well-known\/openid)/i; -// Keycloak JSON "resource" field (Keycloak adapter config) -const KC_RESOURCE_RE = /"resource"\s*:\s*"([^"]+)"/; - -// iframe embed -const IFRAME_SRC_RE = /]+src=["']([^"']+)["']/g; +// iframe + ct_token embed detection const CT_TOKEN_RE = /\bct_token\b|\bembedded=/; -// GCS / Pub/Sub -const GCS_BUCKET_RE = /gs:\/\/([a-zA-Z0-9._-]+)/g; -const BUCKET_ENV_RE = /([A-Z][A-Z0-9_]*_BUCKET)\s*=\s*([^\s#]+)/g; -const PUBSUB_TOPIC_RE = /projects\/[^/]+\/topics\/([a-zA-Z0-9._-]+)/g; -const PUBSUB_TOPIC_NAME_RE = /["']([a-zA-Z0-9._-]+-topic[s]?)["']/g; - -// DB host patterns in env/config -const DB_HOST_RE = /(?:DB_HOST|DATABASE_HOST|POSTGRES_HOST|MYSQL_HOST)\s*=\s*([^\s#]+)/gi; -const DB_URL_RE = /(?:DATABASE_URL|DB_URL)\s*=\s*((?:postgres|mysql|mongodb|sqlite)[^:\s]*:\/\/[^\s#]+)/gi; - // CORS origins in Python/JS const CORS_ORIGIN_RE = /(?:allow_origins|allowedOrigins|origins)\s*[=:]\s*\[([^\]]+)\]/; -// Ingress host in helm values YAML -const INGRESS_HOST_RE = /host\s*:\s*(.+)/; - // pyproject.toml / package.json name const PYPROJECT_NAME_RE = /^name\s*=\s*["']?([^"'\n]+)["']?/m; const PACKAGE_JSON_NAME_RE = /"name"\s*:\s*"([^"]+)"/; From d89ef2d59577e7ce25c42ca8280b6434529d6ca2 Mon Sep 17 00:00:00 2001 From: Mohammed Hafiz <65387738+Hafiz408@users.noreply.github.com> Date: Fri, 26 Jun 2026 20:59:59 +0530 Subject: [PATCH 16/17] refactor(crossrepo): drop dead tour-order arithmetic + duplicate edge label Ponytail trims (no behavior change, both verified by tests + e2e): - tour steps no longer compute inline `order` (the sequential renumber after assembly is the single source of truth); removes misleading len()+2 arithmetic. - cross-repo edges set only `description` (kept by the dashboard schema), not a duplicate `label` (stripped on load, read by nothing). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../skills/understand-crossrepo/apply-interlinks.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/understand-anything-plugin/skills/understand-crossrepo/apply-interlinks.py b/understand-anything-plugin/skills/understand-crossrepo/apply-interlinks.py index b0e4ef54a..a84ca90b1 100644 --- a/understand-anything-plugin/skills/understand-crossrepo/apply-interlinks.py +++ b/understand-anything-plugin/skills/understand-crossrepo/apply-interlinks.py @@ -262,9 +262,8 @@ def _remap_target(target: str) -> str: "weight": float(weight), } if label: - # `description` survives the dashboard schema (which strips unknown edge - # fields like `label`); keep `label` too for the raw/intermediate file. - edge["label"] = label + # Use `description` (which the dashboard schema keeps) rather than `label` + # (which it strips) — nothing downstream reads the edge label. edge["description"] = label if e.get("confidence") is not None: edge["confidence"] = e["confidence"] @@ -296,7 +295,6 @@ def _remap_target(target: str) -> str: ])[:2] step_nodes = [anchor] + [s for s in sample if s in node_ids] repo_steps.append({ - "order": len(repo_steps) + 2, "title": f"{ns} service", "description": f"Key nodes in the {ns} repo.", "nodeIds": step_nodes, @@ -312,21 +310,18 @@ def _remap_target(target: str) -> str: tour = [ { - "order": 1, "title": "Platform Overview", "description": "High-level view of all repos and their module anchors.", "nodeIds": module_anchors, }, *repo_steps, { - "order": len(repo_steps) + 2, "title": "Cross-Repo Interlinks", "description": "Nodes at the boundaries of inter-service dependencies.", "nodeIds": interlink_endpoints, }, ] - # Renumber tour sequentially so order is unique and 1-based regardless of how - # repo_steps were accumulated. + # `order` is assigned here as the single source of truth — sequential, unique, 1-based. for i, step in enumerate(tour): step["order"] = i + 1 From 0f54fff2b3bd810ab604cc63b90eaf81358f3c8d Mon Sep 17 00:00:00 2001 From: Mohammed Hafiz <65387738+Hafiz408@users.noreply.github.com> Date: Fri, 26 Jun 2026 23:07:19 +0530 Subject: [PATCH 17/17] fix(crossrepo): clamp inherited out-of-range edge weights + repair direction QA across gemba/market_pulse/product_lens repo sets surfaced that some base /understand graphs emit edges with weight>1 (e.g. 2) or invalid direction. The dashboard's core/schema clamps them on load and shows an 'N auto-corrections' banner (240/123 on gemba_ui/product_lens). The assembler already normalizes nodes (lineRange/complexity); do the same for edges so the combined graph renders clean on ANY source repos. Sets now validate with 0 issues (was 240/10/123). --- .../__tests__/test_apply.py | 20 ++++++++++++-- .../understand-crossrepo/apply-interlinks.py | 26 ++++++++++++++++++- 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/understand-anything-plugin/skills/understand-crossrepo/__tests__/test_apply.py b/understand-anything-plugin/skills/understand-crossrepo/__tests__/test_apply.py index bc35e455b..3c79d7a54 100644 --- a/understand-anything-plugin/skills/understand-crossrepo/__tests__/test_apply.py +++ b/understand-anything-plugin/skills/understand-crossrepo/__tests__/test_apply.py @@ -428,8 +428,13 @@ def test_dashboard_schema_requirements(): {"id": "module:svc_a", "type": "module", "name": "svc_a", "summary": "anchor", "tags": ["repo:svc_a"], "repo": "svc_a"}, ], - "edges": [{"source": "module:svc_a", "target": "file:svc_a/m.py", - "type": "contains", "direction": "forward", "weight": 0.8}], + "edges": [ + {"source": "module:svc_a", "target": "file:svc_a/m.py", + "type": "contains", "direction": "forward", "weight": 0.8}, + # inherited source-graph quirks: weight out of [0,1] + invalid direction + {"source": "file:svc_a/m.py", "target": "module:svc_a", + "type": "related", "direction": "sideways", "weight": 2}, + ], "layers": [{"id": "layer:svc_a", "name": "svc_a", "description": "l", "nodeIds": ["file:svc_a/m.py", "module:svc_a"]}], } @@ -464,6 +469,17 @@ def test_dashboard_schema_requirements(): assert "authenticates_via" in (auth.get("description") or ""), \ f"original type not preserved in description: {auth.get('description')}" + # every edge must have weight in [0,1] and a valid direction (clamped/repaired + # from source-graph quirks) so the dashboard shows 0 auto-corrections + for e in kg["edges"]: + assert isinstance(e.get("weight"), (int, float)) and 0 <= e["weight"] <= 1, \ + f"edge weight out of [0,1]: {e}" + assert e.get("direction") in ("forward", "backward", "bidirectional"), \ + f"edge has invalid direction: {e}" + clamped = next(e for e in kg["edges"] if e["source"] == "file:svc_a/m.py" and e["target"] == "module:svc_a") + assert clamped["weight"] == 1.0, f"weight 2 should clamp to 1.0, got {clamped['weight']}" + assert clamped["direction"] == "forward", f"invalid direction should repair to forward, got {clamped['direction']}" + if __name__ == "__main__": tests = [ diff --git a/understand-anything-plugin/skills/understand-crossrepo/apply-interlinks.py b/understand-anything-plugin/skills/understand-crossrepo/apply-interlinks.py index a84ca90b1..e365c5a72 100644 --- a/understand-anything-plugin/skills/understand-crossrepo/apply-interlinks.py +++ b/understand-anything-plugin/skills/understand-crossrepo/apply-interlinks.py @@ -169,6 +169,30 @@ def _make_external_node(svc: str) -> dict: } +_VALID_DIRECTIONS = ("forward", "backward", "bidirectional") + + +def _normalize_edge(edge: dict) -> dict: + """Clamp/repair edge fields the dashboard's core/schema.ts enforces. + + Intra-repo edges are inherited verbatim from the per-repo graphs, some of which + the base /understand tool emits with a `weight` outside [0,1] (e.g. 2) or a + missing/invalid `direction`. core/schema auto-corrects these on load and shows + an "N auto-corrections" banner. Normalize here so the combined graph renders + clean regardless of source-graph quirks — the edge equivalent of _backfill_node. + """ + w = edge.get("weight") + if not isinstance(w, (int, float)) or isinstance(w, bool): + edge["weight"] = 0.5 + elif w < 0: + edge["weight"] = 0.0 + elif w > 1: + edge["weight"] = 1.0 + if edge.get("direction") not in _VALID_DIRECTIONS: + edge["direction"] = "forward" + return edge + + def apply(out: Path) -> None: intermediate = out / ".understand-anything" / "intermediate" ua_dir = out / ".understand-anything" @@ -347,7 +371,7 @@ def _remap_target(target: str) -> str: # Sort nodes and layers for determinism final_nodes = sorted(nodes, key=lambda n: n["id"]) - final_edges = sorted(all_edges, key=lambda e: (e.get("source", ""), e.get("target", ""), e.get("type", ""))) + final_edges = sorted((_normalize_edge(e) for e in all_edges), key=lambda e: (e.get("source", ""), e.get("target", ""), e.get("type", ""))) final_layers = sorted(layers, key=lambda l: l["id"]) # Sort nodeIds within each layer for determinism for layer in final_layers: