diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c560760..54e75a1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -111,6 +111,9 @@ jobs: - name: Run eval harness smoke check (mock embeddings) run: npm run eval:smoke + - name: Run pre-edit context eval gate (mock embeddings) + run: npm run eval:pre-edit:ci + - name: Stop mock embeddings server if: always() run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index f5e4744..14b8989 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - **Context diagnostics**: `codebase_context` now supports an optional `diagnostic` flag across MCP, OpenCode, and Pi. It exposes bounded routing, retrieval, and evidence-pack traces for troubleshooting without changing normal text output. +- **Pre-edit evaluation gate**: `codebase_edit_context` now has a documented CI evaluation gate. The budget gate supports a `minGraphNeighborRecall` threshold, and `npm run eval:pre-edit:ci` runs the deterministic mock-embedding baseline (Hit@5 1.0, MRR@10 1.0, graph-neighbor recall 1.0) with thresholds in `benchmarks/budgets/pre-edit.json`. ### Fixed diff --git a/benchmarks/budgets/pre-edit.json b/benchmarks/budgets/pre-edit.json new file mode 100644 index 0000000..8418fcf --- /dev/null +++ b/benchmarks/budgets/pre-edit.json @@ -0,0 +1,17 @@ +{ + "name": "pre-edit-context-eval-budget", + "failOnMissingBaseline": false, + "thresholds": { + "p95LatencyMaxAbsoluteMs": 5000, + "minHitAt5": 1.0, + "minMrrAt10": 1.0, + "minGraphNeighborRecall": 1.0, + "maxContextResponseTokensAverage": 600, + "maxContextResponseTokensP95": 800, + "maxContextResponseTokensMax": 1000, + "maxContextDuplicateCandidateRatio": 0.5, + "minContextSelectedFileRatio": 0.5, + "minContextHitAt5Per1kResponseTokens": 1.0, + "minContextMrrAt10Per1kResponseTokens": 1.0 + } +} diff --git a/docs/pre-edit-context-design.md b/docs/pre-edit-context-design.md index cf22cbc..7af4b74 100644 --- a/docs/pre-edit-context-design.md +++ b/docs/pre-edit-context-design.md @@ -42,6 +42,26 @@ Add a `pre-edit-context` golden set with definition-led modification tasks. Each The acceptance gate is no regression in the existing `agent-context` and representative evaluation suites, plus a documented pre-edit baseline for Hit@5, MRR@10, graph-neighbor recall, p95 latency, and returned tokens. +## Baseline (mock embeddings, 2026-08-06) + +The gate runs in CI through `npm run eval:pre-edit:ci` with the deterministic mock-embedding provider and `benchmarks/budgets/pre-edit.json`. + +| Metric | Baseline | Budget threshold | +|---|---|---| +| Hit@5 | 1.0000 | `minHitAt5` 1.0 | +| MRR@10 | 1.0000 | `minMrrAt10` 1.0 | +| Graph-neighbor recall | 1.0000 | `minGraphNeighborRecall` 1.0 | +| p95 latency | 687 ms | `p95LatencyMaxAbsoluteMs` 5000 | +| Response tokens average | 358.5 | `maxContextResponseTokensAverage` 600 | +| Response tokens p95 | 399.5 | `maxContextResponseTokensP95` 800 | +| Response tokens max | 404 | `maxContextResponseTokensMax` 1000 | +| Duplicate candidate ratio | 0.0 | `maxContextDuplicateCandidateRatio` 0.5 | +| Selected-file ratio | 0.7857 | `minContextSelectedFileRatio` 0.5 | +| Hit@5 per 1k response tokens | 2.789 | `minContextHitAt5Per1kResponseTokens` 1.0 | +| MRR@10 per 1k response tokens | 2.789 | `minContextMrrAt10Per1kResponseTokens` 1.0 | + +The budget gate supports the `minGraphNeighborRecall` threshold for datasets whose queries assert graph neighbors; datasets without graph-neighbor expectations are unaffected because the metric is only compared when present. + ## Explicit non-goals - No new parser or call-resolution heuristic in the first release. diff --git a/package.json b/package.json index 9b2c3c8..df74172 100644 --- a/package.json +++ b/package.json @@ -48,6 +48,7 @@ "eval:pre-edit": "npx tsx src/cli.ts eval run --dataset benchmarks/golden/pre-edit-context.json", "eval:representative": "npx tsx src/cli.ts eval run --dataset benchmarks/golden/representative.json", "eval:agent:ci": "npx tsx src/cli.ts eval run --dataset benchmarks/golden/agent-context.json --reindex --ci --budget benchmarks/budgets/agent-context.json", + "eval:pre-edit:ci": "npx tsx src/cli.ts eval run --config .github/eval-config.json --reindex --dataset benchmarks/golden/pre-edit-context.json --ci --budget benchmarks/budgets/pre-edit.json", "eval:smoke": "npx tsx src/cli.ts eval run --config .github/eval-config.json --reindex --dataset benchmarks/golden/small.json", "eval:ci": "npx tsx src/cli.ts eval run --reindex --ci --budget benchmarks/budgets/default.json --against benchmarks/baselines/eval-baseline-summary.json", "eval:ci:ollama": "npx tsx src/cli.ts eval run --config .github/eval-ollama-config.json --reindex --ci --budget benchmarks/budgets/default.json --against benchmarks/baselines/eval-baseline-summary.json", diff --git a/src/eval/budget.ts b/src/eval/budget.ts index 2e5ad82..aa1e200 100644 --- a/src/eval/budget.ts +++ b/src/eval/budget.ts @@ -34,6 +34,17 @@ export function evaluateBudgetGate( }); } + if ( + thresholds.minGraphNeighborRecall !== undefined && + summary.metrics.graphNeighborRecall !== undefined && + summary.metrics.graphNeighborRecall < thresholds.minGraphNeighborRecall + ) { + violations.push({ + metric: "minGraphNeighborRecall", + message: `Graph-neighbor recall ${summary.metrics.graphNeighborRecall.toFixed(4)} is below minimum ${thresholds.minGraphNeighborRecall.toFixed(4)}`, + }); + } + if (comparison) { if ( thresholds.hitAt5MaxDrop !== undefined && diff --git a/src/eval/schema.ts b/src/eval/schema.ts index da9432f..d977fa2 100644 --- a/src/eval/schema.ts +++ b/src/eval/schema.ts @@ -466,6 +466,11 @@ export function parseBudget(raw: unknown, sourceLabel: string): EvalBudget { "minRawDistinctTop3Ratio", sourceLabel ), + minGraphNeighborRecall: parseThresholdValue( + thresholds.minGraphNeighborRecall, + "minGraphNeighborRecall", + sourceLabel + ), maxContextResponseTokensAverage: parseThresholdValue( thresholds.maxContextResponseTokensAverage, "maxContextResponseTokensAverage", diff --git a/src/eval/types.ts b/src/eval/types.ts index ada5e15..30299a7 100644 --- a/src/eval/types.ts +++ b/src/eval/types.ts @@ -81,6 +81,7 @@ export interface EvalBudget { minHitAt5?: number; minMrrAt10?: number; minRawDistinctTop3Ratio?: number; + minGraphNeighborRecall?: number; maxContextResponseTokensAverage?: number; maxContextResponseTokensP95?: number; maxContextResponseTokensMax?: number; diff --git a/tests/eval-budget.test.ts b/tests/eval-budget.test.ts index cdb04f9..ce44cef 100644 --- a/tests/eval-budget.test.ts +++ b/tests/eval-budget.test.ts @@ -177,6 +177,66 @@ describe("eval budget gate", () => { expect(gate.violations.some((v) => v.metric === "rawDistinctTop3RatioMaxDrop")).toBe(true); }); + it("fails when graph-neighbor recall drops below minimum", () => { + const budget: EvalBudget = { + name: "pre-edit", + failOnMissingBaseline: false, + thresholds: { + minGraphNeighborRecall: 1, + }, + }; + + const gate = evaluateBudgetGate( + budget, + { + ...summary(5), + metrics: { + ...summary(5).metrics, + graphNeighborRecall: 0.5, + }, + } + ); + expect(gate.passed).toBe(false); + expect(gate.violations.some((v) => v.metric === "minGraphNeighborRecall")).toBe(true); + }); + + it("passes when graph-neighbor recall meets the minimum", () => { + const budget: EvalBudget = { + name: "pre-edit", + failOnMissingBaseline: false, + thresholds: { + minGraphNeighborRecall: 0.5, + }, + }; + + const gate = evaluateBudgetGate( + budget, + { + ...summary(5), + metrics: { + ...summary(5).metrics, + graphNeighborRecall: 1, + }, + } + ); + expect(gate.passed).toBe(true); + expect(gate.violations).toHaveLength(0); + }); + + it("skips graph-neighbor gate when metric is absent", () => { + const budget: EvalBudget = { + name: "search-only", + failOnMissingBaseline: false, + thresholds: { + minGraphNeighborRecall: 1, + }, + }; + + const gate = evaluateBudgetGate(budget, summary(5)); + expect(gate.passed).toBe(true); + expect(gate.violations).toHaveLength(0); + }); + it("enforces context response, duplicate, and quality-per-token thresholds", () => { const budget: EvalBudget = { name: "context",