package graph generator to understand dependencies

.gitignore

@@ -108,3 +108,6 @@ packages/zql-integration-tests/Chinook_Sqlite.sqlite
 *.ignore.*
 package-lock-commit-deps-report.html
 packages/zql-integration-tests/Pagila_PostgreSql-5ba5a57aeb15.sql
+
+# generated by `pnpm graph` (docs/PACKAGE-GRAPH.md is committed)
+docs/graph/

.oxfmtrc.json

@@ -31,6 +31,7 @@
     "apps/zbugs/public/um.js",
     "apps/zbugs/db/migrations/",
     "**/__snapshots__/**",
+    "docs/PACKAGE-GRAPH.md",
     "packages/zero-client/src/client/zero-stress-queries-test.ts",
     "packages/zql-benchmarks/src/planner-cost.bench.ts"
   ]

AGENTS.md

@@ -179,8 +179,38 @@ const user = table('user')
   - `zero-client`, `zero-server`, `zero-cache` (higher level - can use zql/schema)
   - `zero` (highest - re-exports for convenience, user-facing only)
 
+  `pnpm graph` renders this hierarchy from the manifests and reports every
+  dependency that points the wrong way up it. See "Package graph" below.
+
 - Re-exports are acceptable in **user-facing packages** for convenience (e.g., `packages/zero/src/mod.ts` → exports from `zero-client`, `zero-server`), but avoid re-exports between internal packages
 
+## Package graph
+
+`pnpm graph` builds a map of the workspace from pnpm membership plus the curated
+layers in `tools/package-graph/src/workspace.ts`, and writes three things:
+
+- `docs/PACKAGE-GRAPH.md` — committed. Mermaid diagrams, the full direct
+  dependency inventory, and a **Layer inversions** table: every dependency that
+  points up the layer stack, i.e. where an abstraction boundary is being broken.
+  A new row here is worth a question in review.
+- `docs/graph/model.json` — gitignored. The extracted model. Read this rather
+  than re-walking pnpm.
+- `docs/graph/index.html` — gitignored. Interactive view: pan/zoom, per-package
+  metrics (fan-in/out, cone, blast radius, instability), dependency and
+  dependent cones, layer filtering, and an inversions-only filter.
+
+```bash
+pnpm graph          # regenerate all three
+pnpm graph:check    # fail if the committed Markdown is stale
+pnpm graph:open     # regenerate and open the interactive view
+```
+
+Adding or renaming a workspace package makes `pnpm graph` fail until the package
+is placed in `LAYERS`. Internal dependencies here are usually `devDependencies`
+(packages import each other's TypeScript source over relative paths), so the
+graph includes them; `pnpm verify-deps` is what keeps those manifests honest
+against the actual imports.
+
 ## Database
 
 ### Zero + PostgreSQL

docs/PACKAGE-GRAPH.md

@@ -0,0 +1,232 @@
+<!-- Generated by `pnpm graph`; do not edit by hand. -->
+
+# Package dependency map
+
+This view comes from pnpm workspace membership plus the curated architectural
+layers in `tools/package-graph/src/workspace.ts`. Arrows point from a consumer to the internal
+package it depends on. External npm packages are omitted; internal devDependencies are not.
+
+Packages in this repo import each other as TypeScript source over relative paths,
+so an internal dependency is usually declared as a **devDependency** — 117
+of the 133 edges below. Those are the architecture here, not
+test scaffolding, so they are drawn like any other edge;
+`tools/verify-package-deps` is what keeps the manifests honest against the
+actual imports.
+
+**39 workspace packages · 133 direct internal dependencies · 44 structural edges**
+
+## Bird's-eye view
+
+The number on an arrow is the count of direct package dependencies crossing
+those two layers.
+
+```mermaid
+flowchart BT
+  layer_foundations["Foundations<br/>6 packages"]
+  layer_query_engine["Query engine &amp; schema<br/>5 packages"]
+  layer_storage["Storage &amp; replication<br/>3 packages"]
+  layer_sdks["Client &amp; server SDKs<br/>3 packages"]
+  layer_bindings["Framework bindings &amp; packaging<br/>5 packages"]
+  layer_apps_tools["Apps, tools &amp; harnesses<br/>17 packages"]
+
+  layer_query_engine -->|16| layer_foundations
+  layer_query_engine -->|2| layer_storage
+  layer_storage -->|10| layer_foundations
+  layer_storage -->|5| layer_query_engine
+  layer_sdks -->|7| layer_foundations
+  layer_sdks -->|7| layer_query_engine
+  layer_sdks -->|3| layer_storage
+  layer_bindings -->|8| layer_foundations
+  layer_bindings -->|7| layer_query_engine
+  layer_bindings -->|6| layer_storage
+  layer_bindings -->|6| layer_sdks
+  layer_apps_tools -->|19| layer_foundations
+  layer_apps_tools -->|9| layer_query_engine
+  layer_apps_tools -->|8| layer_storage
+  layer_apps_tools -->|1| layer_sdks
+  layer_apps_tools -->|2| layer_bindings
+```
+
+## Layer inversions
+
+Layers run low to high and arrows point consumer → dependency, so a dependency
+that climbs to a **higher** layer is a package reaching into something meant to
+sit above it. These are reported, not forbidden — but each one should be a
+deliberate decision, and a new row here is worth a question in review.
+
+| Consumer | Its layer | Depends on | Which sits in | Declared as |
+| --- | --- | --- | --- | --- |
+| `ast-to-zql` | Query engine & schema | `zero-cache` | Storage & replication | dev |
+| `z2s` | Query engine & schema | `zero-cache` | Storage & replication | dev |
+
+## Package paths
+
+This diagram uses the graph's transitive reduction so the architectural paths
+remain legible. For example, when `A → B → C` exists, a direct `A → C`
+shortcut is left out here. The inventory below retains every direct dependency
+declared in package.json.
+
+```mermaid
+flowchart BT
+  subgraph layer_foundations["Foundations"]
+    pkg__rocicorp_zero_events["@rocicorp/zero-events"]
+    pkg_datadog["datadog"]
+    pkg_otel["otel"]
+    pkg_shared["shared"]
+    pkg_zero_protocol["zero-protocol"]
+    pkg_zero_types["zero-types"]
+  end
+
+  subgraph layer_query_engine["Query engine &amp; schema"]
+    pkg_ast_to_zql["ast-to-zql"]
+    pkg_z2s["z2s"]
+    pkg_zero_permissions["zero-permissions"]
+    pkg_zero_schema["zero-schema"]
+    pkg_zql["zql"]
+  end
+
+  subgraph layer_storage["Storage &amp; replication"]
+    pkg_replicache["replicache"]
+    pkg_zero_cache["zero-cache"]
+    pkg_zqlite["zqlite"]
+  end
+
+  subgraph layer_sdks["Client &amp; server SDKs"]
+    pkg_zero_client["zero-client"]
+    pkg_zero_pg["zero-pg"]
+    pkg_zero_server["zero-server"]
+  end
+
+  subgraph layer_bindings["Framework bindings &amp; packaging"]
+    pkg__rocicorp_zero["@rocicorp/zero"]
+    pkg_analyze_query["analyze-query"]
+    pkg_zero_react["zero-react"]
+    pkg_zero_react_native["zero-react-native"]
+    pkg_zero_solid["zero-solid"]
+  end
+
+  subgraph layer_apps_tools["Apps, tools &amp; harnesses"]
+    pkg_client_simulator["client-simulator"]
+    pkg_load_generator["load-generator"]
+    pkg_otel_proxy["otel-proxy"]
+    pkg_package_graph["package-graph"]
+    pkg_process_tracker["process-tracker"]
+    pkg_replicache_doc["replicache-doc"]
+    pkg_replicache_perf["replicache-perf"]
+    pkg_scripts["scripts"]
+    pkg_sqlite_io_yield_simulator["sqlite-io-yield-simulator"]
+    pkg_verify_package_deps["verify-package-deps"]
+    pkg_zbugs["zbugs"]
+    pkg_zero_sst["zero-sst"]
+    pkg_zero_throughput["zero-throughput"]
+    pkg_zql_benchmarks["zql-benchmarks"]
+    pkg_zql_integration_tests["zql-integration-tests"]
+    pkg_zql_viz["zql-viz"]
+    pkg_zqlite_zql_test["zqlite-zql-test"]
+  end
+
+  pkg__rocicorp_zero --> pkg_analyze_query
+  pkg__rocicorp_zero --> pkg_zero_pg
+  pkg__rocicorp_zero --> pkg_zero_react
+  pkg__rocicorp_zero --> pkg_zero_solid
+  pkg__rocicorp_zero_events --> pkg_shared
+  pkg_analyze_query --> pkg_zero_client
+  pkg_ast_to_zql --> pkg_zero_cache
+  pkg_client_simulator --> pkg_zero_protocol
+  pkg_datadog --> pkg_shared
+  pkg_load_generator --> pkg_shared
+  pkg_otel --> pkg_shared
+  pkg_process_tracker --> pkg_shared
+  pkg_replicache --> pkg_shared
+  pkg_replicache_doc --> pkg_replicache
+  pkg_replicache_perf --> pkg_replicache
+  pkg_sqlite_io_yield_simulator --> pkg_zqlite
+  pkg_z2s --> pkg_zero_cache
+  pkg_zbugs --> pkg__rocicorp_zero
+  pkg_zero_cache --> pkg__rocicorp_zero_events
+  pkg_zero_cache --> pkg_zero_permissions
+  pkg_zero_cache --> pkg_zqlite
+  pkg_zero_client --> pkg_ast_to_zql
+  pkg_zero_client --> pkg_datadog
+  pkg_zero_client --> pkg_replicache
+  pkg_zero_permissions --> pkg_zero_schema
+  pkg_zero_permissions --> pkg_zql
+  pkg_zero_pg --> pkg_zero_server
+  pkg_zero_protocol --> pkg_zero_types
+  pkg_zero_react --> pkg_zero_client
+  pkg_zero_react_native --> pkg_replicache
+  pkg_zero_schema --> pkg_zero_protocol
+  pkg_zero_server --> pkg_z2s
+  pkg_zero_solid --> pkg_zero_client
+  pkg_zero_throughput --> pkg__rocicorp_zero
+  pkg_zero_types --> pkg_shared
+  pkg_zql --> pkg_otel
+  pkg_zql --> pkg_zero_protocol
+  pkg_zql_benchmarks --> pkg_zql_integration_tests
+  pkg_zql_integration_tests --> pkg_ast_to_zql
+  pkg_zql_integration_tests --> pkg_zero_server
+  pkg_zql_viz --> pkg_zql
+  pkg_zqlite --> pkg_zero_schema
+  pkg_zqlite --> pkg_zql
+  pkg_zqlite_zql_test --> pkg_zqlite
+```
+
+## Direct dependency inventory
+
+| Layer | Package | Kind | Direct workspace dependencies |
+| --- | --- | --- | --- |
+| Foundations | [`@rocicorp/zero-events`](../packages/zero-events) | published package | `shared` _(dev)_ |
+| Foundations | [`datadog`](../packages/datadog) | internal package | `shared` _(dev)_ |
+| Foundations | [`otel`](../packages/otel) | internal package | `shared` _(dev)_ |
+| Foundations | [`shared`](../packages/shared) | internal package | — |
+| Foundations | [`zero-protocol`](../packages/zero-protocol) | internal package | `shared` _(dev)_, `zero-types` _(dev)_ |
+| Foundations | [`zero-types`](../packages/zero-types) | internal package | `shared` _(dev)_ |
+| Query engine & schema | [`ast-to-zql`](../packages/ast-to-zql) | internal package | `shared` _(dev)_, `zero-cache` _(dev)_, `zero-protocol` _(dev)_, `zero-schema` _(dev)_, `zero-types` _(dev)_ |
+| Query engine & schema | [`z2s`](../packages/z2s) | internal package | `shared` _(dev)_, `zero-cache` _(dev)_, `zero-protocol` _(dev)_, `zero-schema` _(dev)_, `zero-types` _(dev)_, `zql` _(dev)_ |
+| Query engine & schema | [`zero-permissions`](../packages/zero-permissions) | internal package | `shared`, `zero-protocol`, `zero-schema`, `zero-types`, `zql` |
+| Query engine & schema | [`zero-schema`](../packages/zero-schema) | internal package | `shared` _(dev)_, `zero-protocol` _(dev)_, `zero-types` _(dev)_ |
+| Query engine & schema | [`zql`](../packages/zql) | internal package | `otel` _(dev)_, `shared` _(dev)_, `zero-protocol` _(dev)_, `zero-types` _(dev)_ |
+| Storage & replication | [`replicache`](../packages/replicache) | published package | `shared` _(dev)_ |
+| Storage & replication | [`zero-cache`](../packages/zero-cache) | internal package | `@rocicorp/zero-events` _(dev)_, `otel` _(dev)_, `shared` _(dev)_, `zero-permissions` _(dev)_, `zero-protocol`, `zero-schema` _(dev)_, `zero-types`, `zql`, `zqlite` |
+| Storage & replication | [`zqlite`](../packages/zqlite) | internal package | `otel` _(dev)_, `shared` _(dev)_, `zero-protocol` _(dev)_, `zero-schema` _(dev)_, `zero-types` _(dev)_, `zql` |
+| Client & server SDKs | [`zero-client`](../packages/zero-client) | internal package | `ast-to-zql` _(dev)_, `datadog` _(dev)_, `replicache` _(dev)_, `shared` _(dev)_, `zero-permissions` _(dev)_, `zero-protocol` _(dev)_, `zero-schema` _(dev)_, `zero-types` _(dev)_, `zql` _(dev)_ |
+| Client & server SDKs | [`zero-pg`](../packages/zero-pg) | internal package | `zero-cache` _(dev)_, `zero-server` _(dev)_ |
+| Client & server SDKs | [`zero-server`](../packages/zero-server) | internal package | `shared` _(dev)_, `z2s` _(dev)_, `zero-cache` _(dev)_, `zero-protocol` _(dev)_, `zero-schema` _(dev)_, `zero-types` _(dev)_, `zql` _(dev)_ |
+| Framework bindings & packaging | [`@rocicorp/zero`](../packages/zero) | published package | `analyze-query` _(dev)_, `ast-to-zql` _(dev)_, `replicache` _(dev)_, `shared` _(dev)_, `zero-cache` _(dev)_, `zero-client` _(dev)_, `zero-pg` _(dev)_, `zero-react` _(dev)_, `zero-server` _(dev)_, `zero-solid` _(dev)_, `zqlite` _(dev)_ |
+| Framework bindings & packaging | [`analyze-query`](../packages/analyze-query) | internal package | `ast-to-zql` _(dev)_, `otel` _(dev)_, `shared` _(dev)_, `zero-cache` _(dev)_, `zero-client` _(dev)_, `zero-protocol` _(dev)_, `zero-schema` _(dev)_, `zero-types` _(dev)_, `zql` _(dev)_, `zqlite` _(dev)_ |
+| Framework bindings & packaging | [`zero-react`](../packages/zero-react) | internal package | `shared` _(dev)_, `zero-client` _(dev)_, `zero-schema` _(dev)_, `zql` _(dev)_ |
+| Framework bindings & packaging | [`zero-react-native`](../packages/zero-react-native) | internal package | `replicache` _(dev)_, `shared` _(dev)_ |
+| Framework bindings & packaging | [`zero-solid`](../packages/zero-solid) | internal package | `shared` _(dev)_, `zero-client` _(dev)_, `zql` _(dev)_ |
+| Apps, tools & harnesses | [`client-simulator`](../tools/client-simulator) | tool | `shared` _(dev)_, `zero-protocol` _(dev)_ |
+| Apps, tools & harnesses | [`load-generator`](../tools/load-generator) | tool | `shared` _(dev)_ |
+| Apps, tools & harnesses | [`otel-proxy`](../apps/otel-proxy) | app | — |
+| Apps, tools & harnesses | [`package-graph`](../tools/package-graph) | tool | — |
+| Apps, tools & harnesses | [`process-tracker`](../tools/process-tracker) | tool | `shared` _(dev)_ |
+| Apps, tools & harnesses | [`replicache-doc`](../packages/replicache-doc) | internal package | `replicache` _(dev)_ |
+| Apps, tools & harnesses | [`replicache-perf`](../packages/replicache-perf) | internal package | `replicache`, `shared` |
+| Apps, tools & harnesses | [`scripts`](../scripts) | tool | — |
+| Apps, tools & harnesses | [`sqlite-io-yield-simulator`](../tools/sqlite-io-yield-simulator) | tool | `shared` _(dev)_, `zqlite` |
+| Apps, tools & harnesses | [`verify-package-deps`](../tools/verify-package-deps) | tool | — |
+| Apps, tools & harnesses | [`zbugs`](../apps/zbugs) | app | `@rocicorp/zero`, `shared` _(dev)_ |
+| Apps, tools & harnesses | [`zero-sst`](../prod/sst) | deployment | — |
+| Apps, tools & harnesses | [`zero-throughput`](../apps/zero-throughput) | app | `@rocicorp/zero`, `shared` _(dev)_, `zero-protocol` _(dev)_, `zero-schema` _(dev)_, `zql` _(dev)_ |
+| Apps, tools & harnesses | [`zql-benchmarks`](../packages/zql-benchmarks) | internal package | `otel` _(dev)_, `shared` _(dev)_, `zero-cache` _(dev)_, `zero-protocol` _(dev)_, `zero-schema` _(dev)_, `zero-types` _(dev)_, `zql` _(dev)_, `zql-integration-tests` _(dev)_, `zqlite` _(dev)_ |
+| Apps, tools & harnesses | [`zql-integration-tests`](../packages/zql-integration-tests) | internal package | `ast-to-zql` _(dev)_, `otel` _(dev)_, `shared` _(dev)_, `z2s` _(dev)_, `zero-cache` _(dev)_, `zero-protocol` _(dev)_, `zero-schema` _(dev)_, `zero-server` _(dev)_, `zero-types` _(dev)_, `zql` _(dev)_, `zqlite` _(dev)_ |
+| Apps, tools & harnesses | [`zql-viz`](../apps/zql-viz) | app | `zero-types` _(dev)_, `zql` _(dev)_ |
+| Apps, tools & harnesses | [`zqlite-zql-test`](../packages/zqlite-zql-test) | internal package | `shared` _(dev)_, `zqlite` |
+
+## Regenerate
+
+From the repository root:
+
+```sh
+pnpm graph          # this file, plus the gitignored model + interactive view
+pnpm graph:check    # fail if this file is stale
+pnpm graph:open     # regenerate and open the interactive view
+```
+
+`pnpm graph` also writes two build products that are **not**
+committed: `docs/graph/model.json` (the extracted workspace model — every renderer,
+script, and agent query should read this rather than re-walking pnpm) and
+`docs/graph/index.html` (an interactive view of the same model, with per-package
+metrics, dependency/dependent cones, and layer filtering).

notes.md

@@ -0,0 +1 @@
+- the diff that we compute from the change log... fml.

package.json

@@ -17,6 +17,9 @@
     "check-types": "turbo run check-types",
     "check-types:watch": "turbo watch check-types",
     "start-zero-cache": "cd packages/zero-cache && pnpm run start",
+    "graph": "node tools/package-graph/src/main.ts",
+    "graph:check": "node tools/package-graph/src/main.ts --check",
+    "graph:open": "node tools/package-graph/src/main.ts --open",
     "verify-deps": "cd tools/verify-package-deps && pnpm run verify",
     "verify-deps:fix": "cd tools/verify-package-deps && pnpm run verify:fix",
     "api-snapshot": "vitest run --config tools/tsnapi/vitest.config.ts",

tools/package-graph/package.json

@@ -0,0 +1,25 @@
+{
+  "name": "package-graph",
+  "version": "0.0.0",
+  "private": true,
+  "description": "Generates the workspace package dependency map (`pnpm graph`)",
+  "type": "module",
+  "scripts": {
+    "check-format": "oxfmt --check .",
+    "check-fmt": "oxfmt --check .",
+    "check-types": "tsc",
+    "fmt": "oxfmt .",
+    "format": "oxfmt .",
+    "graph": "node src/main.ts",
+    "lint": "oxlint --quiet --config ../../oxlint.config.ts"
+  },
+  "devDependencies": {
+    "@types/node": "^22.10.5",
+    "oxfmt": "^0.65.0",
+    "typescript": "~7.0.2"
+  },
+  "engines": {
+    "node": ">=22"
+  },
+  "packageManager": "pnpm@11.11.0"
+}