fix(eval): enforce pre-edit fallback route assertions

CHANGELOG.md

@@ -16,6 +16,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - **Dependency vulnerabilities**: Bumped transitive dependency pins to patched releases so `npm audit` reports 0 vulnerabilities: `hono` 4.12.34 (CORS ReDoS), `ip-address` 10.4.0 (SSRF and trust-boundary bypasses), `fast-uri` 3.1.5 (host confusion), `postcss` 8.5.26 (source-map disclosure), `undici` 8.9.0 via `@earendil-works/pi-coding-agent` 0.84.0 (CRLF injection, cache parsing, cookie and retry issues), `brace-expansion` 5.0.9 (DoS), and `@eslint/config-array` 0.23.5 (minimatch-based DoS path).
 
+- **Pre-edit fallback quality gates**: Added optional `minRouteAccuracy` and `minOutcomeAccuracy` evaluation budget thresholds and applied them in `benchmarks/budgets/pre-edit.json` so unresolved `expectedRoute`/`expectedOutcome` assertions now gate CI.
+
 - **Cross-repo comparison latency**: Omit non-comparable CodeGraph CLI startup time from fair quality tables while retaining raw timing artifacts for diagnostics.
 - **Cross-repo comparison eligibility**: Exclude documentation-only symbols from the CodeGraph exact-definition cohort, keeping the comparator aligned to source-intent definition retrieval.
 - **Exact definition context**: Preserve exact definition lookup ordering when assembling `codebase_context` evidence, so diversification cannot displace the requested definition.

benchmarks/budgets/pre-edit.json

@@ -6,6 +6,8 @@
     "minHitAt5": 1.0,
     "minMrrAt10": 1.0,
     "minGraphNeighborRecall": 1.0,
+    "minRouteAccuracy": 1.0,
+    "minOutcomeAccuracy": 1.0,
     "maxContextResponseTokensAverage": 600,
     "maxContextResponseTokensP95": 800,
     "maxContextResponseTokensMax": 1000,

docs/evaluation.md

@@ -437,6 +437,8 @@ Example:
     "p95LatencyMaxAbsoluteMs": 4000,
     "minHitAt5": 0.4,
     "minMrrAt10": 0.25,
+    "minRouteAccuracy": 0.75,
+    "minOutcomeAccuracy": 0.75,
     "maxContextResponseTokensAverage": 1000,
     "maxContextResponseTokensP95": 1200,
     "maxContextResponseTokensMax": 1200,
@@ -454,5 +456,6 @@ Guidance:
 - Keep `p95LatencyMaxMultiplier` tolerant enough for CI variance.
 - Use absolute floor metrics (`minHitAt5`, `minMrrAt10`) to prevent silent quality drift.
 - Keep context response caps at or below the production default unless a dataset intentionally exercises a larger `tokenBudget`.
+- Add route/outcome gates (`minRouteAccuracy`, `minOutcomeAccuracy`) for datasets that assert `expectedRoute`/`expectedOutcome` so fallback behavior is enforced as an observable quality signal.
 - Track quality-per-token floors together with absolute quality so smaller responses do not pass by becoming less useful.
 - Duplicate-candidate gates measure retrieval waste before packing; the selected-file floor prevents evidence from concentrating in too few files.

docs/pre-edit-context-design.md

@@ -51,6 +51,8 @@ The gate runs in CI through `npm run eval:pre-edit:ci` with the deterministic mo
 | Hit@5 | 1.0000 | `minHitAt5` 1.0 |
 | MRR@10 | 1.0000 | `minMrrAt10` 1.0 |
 | Graph-neighbor recall | 1.0000 | `minGraphNeighborRecall` 1.0 |
+| Route accuracy | 1.0000 | `minRouteAccuracy` 1.0 |
+| Outcome accuracy | 1.0000 | `minOutcomeAccuracy` 1.0 |
 | p95 latency | 672 ms | `p95LatencyMaxAbsoluteMs` 5000 |
 | Response tokens average | 314.3 | `maxContextResponseTokensAverage` 600 |
 | Response tokens p95 | 394.9 | `maxContextResponseTokensP95` 800 |
@@ -60,7 +62,7 @@ The gate runs in CI through `npm run eval:pre-edit:ci` with the deterministic mo
 | Hit@5 per 1k response tokens | 3.181 | `minContextHitAt5Per1kResponseTokens` 1.0 |
 | MRR@10 per 1k response tokens | 3.181 | `minContextMrrAt10Per1kResponseTokens` 1.0 |
 
-The budget gate supports the `minGraphNeighborRecall` threshold for datasets whose queries assert graph neighbors; datasets without graph-neighbor expectations are unaffected because the metric is only compared when present.
+The budget gate also supports `minRouteAccuracy` and `minOutcomeAccuracy` so unresolved-query fallback behavior is enforced when queries assert `expectedRoute` and `expectedOutcome`.
 
 ## Explicit non-goals
 

src/eval/budget.ts

@@ -45,6 +45,26 @@ export function evaluateBudgetGate(
     });
   }
 
+  if (
+    thresholds.minRouteAccuracy !== undefined &&
+    summary.metrics.routeAccuracy < thresholds.minRouteAccuracy
+  ) {
+    violations.push({
+      metric: "minRouteAccuracy",
+      message: `Route accuracy ${summary.metrics.routeAccuracy.toFixed(4)} is below minimum ${thresholds.minRouteAccuracy.toFixed(4)}`,
+    });
+  }
+
+  if (
+    thresholds.minOutcomeAccuracy !== undefined &&
+    summary.metrics.outcomeAccuracy < thresholds.minOutcomeAccuracy
+  ) {
+    violations.push({
+      metric: "minOutcomeAccuracy",
+      message: `Outcome accuracy ${summary.metrics.outcomeAccuracy.toFixed(4)} is below minimum ${thresholds.minOutcomeAccuracy.toFixed(4)}`,
+    });
+  }
+
   if (comparison) {
     if (
       thresholds.hitAt5MaxDrop !== undefined &&

src/eval/schema.ts

@@ -471,6 +471,16 @@ export function parseBudget(raw: unknown, sourceLabel: string): EvalBudget {
         "minGraphNeighborRecall",
         sourceLabel
       ),
+      minRouteAccuracy: parseThresholdValue(
+        thresholds.minRouteAccuracy,
+        "minRouteAccuracy",
+        sourceLabel
+      ),
+      minOutcomeAccuracy: parseThresholdValue(
+        thresholds.minOutcomeAccuracy,
+        "minOutcomeAccuracy",
+        sourceLabel
+      ),
       maxContextResponseTokensAverage: parseThresholdValue(
         thresholds.maxContextResponseTokensAverage,
         "maxContextResponseTokensAverage",

src/eval/types.ts

@@ -82,6 +82,8 @@ export interface EvalBudget {
     minMrrAt10?: number;
     minRawDistinctTop3Ratio?: number;
     minGraphNeighborRecall?: number;
+    minRouteAccuracy?: number;
+    minOutcomeAccuracy?: number;
     maxContextResponseTokensAverage?: number;
     maxContextResponseTokensP95?: number;
     maxContextResponseTokensMax?: number;

tests/eval-budget.test.ts

@@ -27,6 +27,9 @@ function summary(p95: number): EvalSummary {
       ndcgAt10: 1,
       distinctTop3Ratio: 1,
       rawDistinctTop3Ratio: 1,
+      routeAccuracy: 1,
+      outcomeAccuracy: 1,
+      recoveryAccuracy: 0,
       latencyMs: {
         p50: p95,
         p95,
@@ -223,6 +226,71 @@ describe("eval budget gate", () => {
     expect(gate.violations).toHaveLength(0);
   });
 
+  it("fails when route accuracy falls below threshold", () => {
+    const budget: EvalBudget = {
+      name: "pre-edit",
+      failOnMissingBaseline: false,
+      thresholds: {
+        minRouteAccuracy: 1,
+      },
+    };
+
+    const gate = evaluateBudgetGate(
+      budget,
+      {
+        ...summary(5),
+        metrics: {
+          ...summary(5).metrics,
+          routeAccuracy: 0,
+        },
+      }
+    );
+
+    expect(gate.passed).toBe(false);
+    expect(gate.violations.some((v) => v.metric === "minRouteAccuracy")).toBe(true);
+  });
+
+  it("passes when route and outcome accuracy meet thresholds", () => {
+    const budget: EvalBudget = {
+      name: "pre-edit",
+      failOnMissingBaseline: false,
+      thresholds: {
+        minRouteAccuracy: 1,
+        minOutcomeAccuracy: 1,
+      },
+    };
+
+    const gate = evaluateBudgetGate(budget, summary(5));
+
+    expect(gate.passed).toBe(true);
+    expect(gate.violations.some((v) => v.metric === "minRouteAccuracy")).toBe(false);
+    expect(gate.violations.some((v) => v.metric === "minOutcomeAccuracy")).toBe(false);
+  });
+
+  it("fails when outcome accuracy falls below threshold", () => {
+    const budget: EvalBudget = {
+      name: "pre-edit",
+      failOnMissingBaseline: false,
+      thresholds: {
+        minOutcomeAccuracy: 1,
+      },
+    };
+
+    const gate = evaluateBudgetGate(
+      budget,
+      {
+        ...summary(5),
+        metrics: {
+          ...summary(5).metrics,
+          outcomeAccuracy: 0,
+        },
+      }
+    );
+
+    expect(gate.passed).toBe(false);
+    expect(gate.violations.some((v) => v.metric === "minOutcomeAccuracy")).toBe(true);
+  });
+
   it("skips graph-neighbor gate when metric is absent", () => {
     const budget: EvalBudget = {
       name: "search-only",

tests/eval-schema.test.ts

@@ -389,6 +389,8 @@ describe("eval schema", () => {
           rawDistinctTop3RatioMaxDrop: 0.1,
           p95LatencyMaxMultiplier: 1.5,
           minRawDistinctTop3Ratio: 0.7,
+          minRouteAccuracy: 0.95,
+          minOutcomeAccuracy: 0.85,
           maxContextResponseTokensAverage: 800,
           maxContextResponseTokensP95: 1200,
           maxContextResponseTokensMax: 1200,
@@ -404,6 +406,8 @@ describe("eval schema", () => {
     expect(budget.thresholds.hitAt5MaxDrop).toBe(0.05);
     expect(budget.thresholds.rawDistinctTop3RatioMaxDrop).toBe(0.1);
     expect(budget.thresholds.minRawDistinctTop3Ratio).toBe(0.7);
+    expect(budget.thresholds.minRouteAccuracy).toBe(0.95);
+    expect(budget.thresholds.minOutcomeAccuracy).toBe(0.85);
     expect(budget.thresholds.maxContextResponseTokensAverage).toBe(800);
     expect(budget.thresholds.maxContextResponseTokensP95).toBe(1200);
     expect(budget.thresholds.maxContextResponseTokensMax).toBe(1200);