feat: complete pre-edit context evaluation gate

.github/workflows/ci.yml

@@ -111,6 +111,9 @@ jobs:
       - name: Run eval harness smoke check (mock embeddings)
         run: npm run eval:smoke
 
+      - name: Run pre-edit context eval gate (mock embeddings)
+        run: npm run eval:pre-edit:ci
+
       - name: Stop mock embeddings server
         if: always()
         run: |

CHANGELOG.md

@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 
 - **Context diagnostics**: `codebase_context` now supports an optional `diagnostic` flag across MCP, OpenCode, and Pi. It exposes bounded routing, retrieval, and evidence-pack traces for troubleshooting without changing normal text output.
+- **Pre-edit evaluation gate**: `codebase_edit_context` now has a documented CI evaluation gate. The budget gate supports a `minGraphNeighborRecall` threshold, and `npm run eval:pre-edit:ci` runs the deterministic mock-embedding baseline (Hit@5 1.0, MRR@10 1.0, graph-neighbor recall 1.0) with thresholds in `benchmarks/budgets/pre-edit.json`.
 
 ### Fixed
 

benchmarks/budgets/pre-edit.json

@@ -0,0 +1,17 @@
+{
+  "name": "pre-edit-context-eval-budget",
+  "failOnMissingBaseline": false,
+  "thresholds": {
+    "p95LatencyMaxAbsoluteMs": 5000,
+    "minHitAt5": 1.0,
+    "minMrrAt10": 1.0,
+    "minGraphNeighborRecall": 1.0,
+    "maxContextResponseTokensAverage": 600,
+    "maxContextResponseTokensP95": 800,
+    "maxContextResponseTokensMax": 1000,
+    "maxContextDuplicateCandidateRatio": 0.5,
+    "minContextSelectedFileRatio": 0.5,
+    "minContextHitAt5Per1kResponseTokens": 1.0,
+    "minContextMrrAt10Per1kResponseTokens": 1.0
+  }
+}

docs/pre-edit-context-design.md

@@ -42,6 +42,26 @@ Add a `pre-edit-context` golden set with definition-led modification tasks. Each
 
 The acceptance gate is no regression in the existing `agent-context` and representative evaluation suites, plus a documented pre-edit baseline for Hit@5, MRR@10, graph-neighbor recall, p95 latency, and returned tokens.
 
+## Baseline (mock embeddings, 2026-08-06)
+
+The gate runs in CI through `npm run eval:pre-edit:ci` with the deterministic mock-embedding provider and `benchmarks/budgets/pre-edit.json`.
+
+| Metric | Baseline | Budget threshold |
+|---|---|---|
+| Hit@5 | 1.0000 | `minHitAt5` 1.0 |
+| MRR@10 | 1.0000 | `minMrrAt10` 1.0 |
+| Graph-neighbor recall | 1.0000 | `minGraphNeighborRecall` 1.0 |
+| p95 latency | 687 ms | `p95LatencyMaxAbsoluteMs` 5000 |
+| Response tokens average | 358.5 | `maxContextResponseTokensAverage` 600 |
+| Response tokens p95 | 399.5 | `maxContextResponseTokensP95` 800 |
+| Response tokens max | 404 | `maxContextResponseTokensMax` 1000 |
+| Duplicate candidate ratio | 0.0 | `maxContextDuplicateCandidateRatio` 0.5 |
+| Selected-file ratio | 0.7857 | `minContextSelectedFileRatio` 0.5 |
+| Hit@5 per 1k response tokens | 2.789 | `minContextHitAt5Per1kResponseTokens` 1.0 |
+| MRR@10 per 1k response tokens | 2.789 | `minContextMrrAt10Per1kResponseTokens` 1.0 |
+
+The budget gate supports the `minGraphNeighborRecall` threshold for datasets whose queries assert graph neighbors; datasets without graph-neighbor expectations are unaffected because the metric is only compared when present.
+
 ## Explicit non-goals
 
 - No new parser or call-resolution heuristic in the first release.

package.json

@@ -48,6 +48,7 @@
     "eval:pre-edit": "npx tsx src/cli.ts eval run --dataset benchmarks/golden/pre-edit-context.json",
     "eval:representative": "npx tsx src/cli.ts eval run --dataset benchmarks/golden/representative.json",
     "eval:agent:ci": "npx tsx src/cli.ts eval run --dataset benchmarks/golden/agent-context.json --reindex --ci --budget benchmarks/budgets/agent-context.json",
+    "eval:pre-edit:ci": "npx tsx src/cli.ts eval run --config .github/eval-config.json --reindex --dataset benchmarks/golden/pre-edit-context.json --ci --budget benchmarks/budgets/pre-edit.json",
     "eval:smoke": "npx tsx src/cli.ts eval run --config .github/eval-config.json --reindex --dataset benchmarks/golden/small.json",
     "eval:ci": "npx tsx src/cli.ts eval run --reindex --ci --budget benchmarks/budgets/default.json --against benchmarks/baselines/eval-baseline-summary.json",
     "eval:ci:ollama": "npx tsx src/cli.ts eval run --config .github/eval-ollama-config.json --reindex --ci --budget benchmarks/budgets/default.json --against benchmarks/baselines/eval-baseline-summary.json",

src/eval/budget.ts

@@ -34,6 +34,17 @@ export function evaluateBudgetGate(
     });
   }
 
+  if (
+    thresholds.minGraphNeighborRecall !== undefined &&
+    summary.metrics.graphNeighborRecall !== undefined &&
+    summary.metrics.graphNeighborRecall < thresholds.minGraphNeighborRecall
+  ) {
+    violations.push({
+      metric: "minGraphNeighborRecall",
+      message: `Graph-neighbor recall ${summary.metrics.graphNeighborRecall.toFixed(4)} is below minimum ${thresholds.minGraphNeighborRecall.toFixed(4)}`,
+    });
+  }
+
   if (comparison) {
     if (
       thresholds.hitAt5MaxDrop !== undefined &&

src/eval/schema.ts

@@ -466,6 +466,11 @@ export function parseBudget(raw: unknown, sourceLabel: string): EvalBudget {
         "minRawDistinctTop3Ratio",
         sourceLabel
       ),
+      minGraphNeighborRecall: parseThresholdValue(
+        thresholds.minGraphNeighborRecall,
+        "minGraphNeighborRecall",
+        sourceLabel
+      ),
       maxContextResponseTokensAverage: parseThresholdValue(
         thresholds.maxContextResponseTokensAverage,
         "maxContextResponseTokensAverage",

src/eval/types.ts

@@ -81,6 +81,7 @@ export interface EvalBudget {
     minHitAt5?: number;
     minMrrAt10?: number;
     minRawDistinctTop3Ratio?: number;
+    minGraphNeighborRecall?: number;
     maxContextResponseTokensAverage?: number;
     maxContextResponseTokensP95?: number;
     maxContextResponseTokensMax?: number;

tests/eval-budget.test.ts

@@ -177,6 +177,66 @@ describe("eval budget gate", () => {
     expect(gate.violations.some((v) => v.metric === "rawDistinctTop3RatioMaxDrop")).toBe(true);
   });
 
+  it("fails when graph-neighbor recall drops below minimum", () => {
+    const budget: EvalBudget = {
+      name: "pre-edit",
+      failOnMissingBaseline: false,
+      thresholds: {
+        minGraphNeighborRecall: 1,
+      },
+    };
+
+    const gate = evaluateBudgetGate(
+      budget,
+      {
+        ...summary(5),
+        metrics: {
+          ...summary(5).metrics,
+          graphNeighborRecall: 0.5,
+        },
+      }
+    );
+    expect(gate.passed).toBe(false);
+    expect(gate.violations.some((v) => v.metric === "minGraphNeighborRecall")).toBe(true);
+  });
+
+  it("passes when graph-neighbor recall meets the minimum", () => {
+    const budget: EvalBudget = {
+      name: "pre-edit",
+      failOnMissingBaseline: false,
+      thresholds: {
+        minGraphNeighborRecall: 0.5,
+      },
+    };
+
+    const gate = evaluateBudgetGate(
+      budget,
+      {
+        ...summary(5),
+        metrics: {
+          ...summary(5).metrics,
+          graphNeighborRecall: 1,
+        },
+      }
+    );
+    expect(gate.passed).toBe(true);
+    expect(gate.violations).toHaveLength(0);
+  });
+
+  it("skips graph-neighbor gate when metric is absent", () => {
+    const budget: EvalBudget = {
+      name: "search-only",
+      failOnMissingBaseline: false,
+      thresholds: {
+        minGraphNeighborRecall: 1,
+      },
+    };
+
+    const gate = evaluateBudgetGate(budget, summary(5));
+    expect(gate.passed).toBe(true);
+    expect(gate.violations).toHaveLength(0);
+  });
+
   it("enforces context response, duplicate, and quality-per-token thresholds", () => {
     const budget: EvalBudget = {
       name: "context",