From 502672cb2ca0ddeeb67088dcd2102a617370d186 Mon Sep 17 00:00:00 2001 From: Rage Lopez Date: Mon, 13 Jul 2026 14:32:05 -0500 Subject: [PATCH] feat: add offline corpus validation harness --- docs/VALIDATE_CORPUS.md | 101 +++++ package.json | 6 +- src/cli.mjs | 51 ++- src/core/corpus-validation.mjs | 735 ++++++++++++++++++++++++++++++++ test/corpus-validation.test.mjs | 272 ++++++++++++ 5 files changed, 1161 insertions(+), 4 deletions(-) create mode 100644 docs/VALIDATE_CORPUS.md create mode 100644 src/core/corpus-validation.mjs create mode 100644 test/corpus-validation.test.mjs diff --git a/docs/VALIDATE_CORPUS.md b/docs/VALIDATE_CORPUS.md new file mode 100644 index 0000000..3a442ae --- /dev/null +++ b/docs/VALIDATE_CORPUS.md @@ -0,0 +1,101 @@ +# Offline Corpus Validation + +`pcf validate-corpus` measures PCF decisions against consented maintainer labels. +It is an offline evidence tool, not a training command, evaluator tuner, or +product-validity oracle. + +```bash +pcf validate-corpus consented.jsonl --format json +pcf validate-corpus consented.csv --format markdown +pcf validate-corpus - --input-format jsonl --format pretty < consented.jsonl +``` + +The command reads one file or standard input and writes only to standard output. +It makes no network request and does not modify PCF fixtures, benchmarks, +policies, feedback stores, or evaluator behavior. Use normal shell redirection +when a local receipt is desired. + +## JSONL contract + +Each non-empty line is one case: + +```json +{"id":"opaque-001","repository":"owner/repo","policyId":"policy-v1","consent":{"allowedForValidation":true,"reference":"consent-batch-a"},"pcf":{"lane":"review-now","score":91,"nextActor":"maintainer"},"ratings":[{"raterId":"maintainer-a","lane":"repair","nextActor":"reporter"},{"raterId":"maintainer-b","lane":"repair","nextActor":"reporter"}],"timing":{"baselineSeconds":120,"pcfSeconds":80},"provenance":{"dataset":"consented-alpha","caseRef":"alpha-001","policySnapshot":"policy-v1@abc123","collectedAt":"2026-07-13T12:00:00Z"}} +``` + +Required boundaries: + +- `id` is an opaque case identifier and must be unique. +- At least one of `repository` or `policyId` identifies the evaluation context. +- `consent.allowedForValidation` must be `true`, with a non-empty consent + reference. +- PCF and maintainer lanes are exactly `review-now`, `repair`, or `defer`. +- `pcf.score` is numeric from 0 through 100. +- Every case has at least one rating; `raterId` values must be unique within the + case. IDs are caller assertions and do not prove identity or independence. +- `provenance.dataset`, `caseRef`, and `policySnapshot` are required. +- `timing` is optional. When present, `baselineSeconds` must be positive and + `pcfSeconds` must be non-negative. +- Declared identifiers are single-line values capped at 256 characters. A case + may contain at most 50 ratings. + +## CSV contract + +CSV uses one row per rater. Repeated case metadata must be identical: + +```text +id,repository,policyId,dataset,caseRef,policySnapshot,consentReference,consentAllowed,pcfLane,pcfScore,pcfNextActor,raterId,raterLane,raterNextActor,baselineSeconds,pcfSeconds +``` + +`consentAllowed` must be the literal `true`. The header must contain exactly the +listed columns with no duplicates. Standard quoted CSV fields, embedded commas, +and escaped quotes are parsed locally; identifier values must remain single-line. + +## Privacy boundary + +The schema intentionally excludes raw third-party material. PCF fails closed if +any nested object includes fields named `title`, `body`, `text`, `content`, +`patch`, `diff`, `comment`, `comments`, `raw`, or `payload`. + +Receipts expose aggregate metrics plus opaque IDs, repository/policy identifiers, +approved `caseRef` values, and corpus provenance. They do not expose rater IDs, +consent references, raw issue or PR content, or local absolute paths. + +Local resource limits are 10 MiB per corpus, 10,000 cases per run, and 50 +ratings per case. These bounds protect an offline operator from accidental +oversized inputs; they are not statistical sufficiency claims. + +## Measurement semantics + +- A lane is scored only when its maintainer ratings have a strict majority. + Ties are reported as `NO_STRICT_MAJORITY` exclusions. +- The confusion matrix uses PCF lanes as rows and maintainer consensus as + columns. +- Precision and recall are reported for every lane. A missing denominator is + `null` in JSON and `n/a` in human receipts. +- False `review-now` means PCF selected `review-now` while maintainer consensus + selected `repair` or `defer`. +- Inter-rater output includes pairwise agreement and a nominal, multi-rater + kappa derived from aggregate lane prevalence. Variable rater counts are + allowed. +- Score calibration uses fixed 20-point bins and compares PCF `review-now` + frequency with consensus `review-now` frequency. It is descriptive, not a + probabilistic calibration claim. +- Paired timing reports seconds and percent saved. Negative savings remain + visible. +- The SHA-256 digest covers the exact input bytes. Replaying identical bytes + produces the same result object. + +## Formats and exit codes + +- `--format pretty`: concise terminal receipt. +- `--format json`: complete machine-readable result. +- `--format markdown`: aggregate human review receipt. +- Exit `0`: the declared corpus shape was valid and metrics were produced. +- Exit `1`: consent, privacy, schema, or integrity validation failed closed. +- Exit `2`: command usage or format was invalid. + +Every successful result retains decision status `INCONCLUSIVE`. Metrics require +human interpretation, an independently governed study, and a later VERITAS +decision before any accuracy, maintainer-endorsement, product-validity, or +release-readiness claim. diff --git a/package.json b/package.json index c0bfae7..ce25b33 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,7 @@ "src", "fixtures", "docs/MCP.md", + "docs/VALIDATE_CORPUS.md", "docs/WATCHLIST.md", "docs/SERIOUS_SCOUT.md", "docs/UPSTREAM_CONTRIBUTION_LEDGER.md", @@ -48,7 +49,7 @@ ], "scripts": { "start": "node src/server.mjs", - "check": "node --check scripts/run-pr-gate.mjs && node --check src/server.mjs && node --check src/cli.mjs && node --check src/config.mjs && node --check src/core/api.mjs && node --check src/core/adversary.mjs && node --check src/core/author-context.mjs && node --check src/core/ai-contribution-posture.mjs && node --check src/core/behavioral-signals.mjs && node --check src/core/benchmark.mjs && node --check src/core/calibration.mjs && node --check src/core/candidates.mjs && node --check src/core/contribution-drafts.mjs && node --check src/core/contributor-preflight.mjs && node --check src/core/diff-shape.mjs && node --check src/core/evaluator.mjs && node --check src/core/feedback.mjs && node --check src/core/history.mjs && node --check src/core/issue-form-validator.mjs && node --check src/core/lane-schema.mjs && node --check src/core/lane-status.mjs && node --check src/core/lane-store.mjs && node --check src/core/maintainer-stack.mjs && node --check src/core/pilot-proof.mjs && node --check src/core/policy.mjs && node --check src/core/policy-scan.mjs && node --check src/core/patch.mjs && node --check src/core/queue.mjs && node --check src/core/repository-context.mjs && node --check src/core/repro-gate.mjs && node --check src/core/mcp-submission.mjs && node --check src/core/scout.mjs && node --check src/core/semantic-duplicate-assist.mjs && node --check src/core/serious-scout.mjs && node --check src/core/setup.mjs && node --check src/core/setup-guide.mjs && node --check src/core/shielded-posture.mjs && node --check src/core/text-safety.mjs && node --check src/core/vouch-context.mjs && node --check src/core/watchlist.mjs && node --check src/github/client.mjs && node --check src/github/webhook.mjs && node --check src/github/templates.mjs && node --check src/mcp/core.mjs && node --check src/mcp/server.mjs && node --check scripts/mcp-smoke.mjs && node --check scripts/run-adversary.mjs && node --check scripts/run-benchmark.mjs && node --check scripts/run-maintainer-demo.mjs && node --check scripts/run-public-pilot.mjs && node --check scripts/run-large-bench.mjs && node --check scripts/run-serious-scout.mjs && node --check scripts/run-watchlist.mjs && node --check scripts/verify-ci-workflow.mjs && node --check scripts/verify-repo-hygiene.mjs && node --check public/app.js", + "check": "node --check scripts/run-pr-gate.mjs && node --check src/server.mjs && node --check src/cli.mjs && node --check src/config.mjs && node --check src/core/api.mjs && node --check src/core/adversary.mjs && node --check src/core/author-context.mjs && node --check src/core/ai-contribution-posture.mjs && node --check src/core/behavioral-signals.mjs && node --check src/core/benchmark.mjs && node --check src/core/calibration.mjs && node --check src/core/candidates.mjs && node --check src/core/contribution-drafts.mjs && node --check src/core/contributor-preflight.mjs && node --check src/core/corpus-validation.mjs && node --check src/core/diff-shape.mjs && node --check src/core/evaluator.mjs && node --check src/core/feedback.mjs && node --check src/core/history.mjs && node --check src/core/issue-form-validator.mjs && node --check src/core/lane-schema.mjs && node --check src/core/lane-status.mjs && node --check src/core/lane-store.mjs && node --check src/core/maintainer-stack.mjs && node --check src/core/pilot-proof.mjs && node --check src/core/policy.mjs && node --check src/core/policy-scan.mjs && node --check src/core/patch.mjs && node --check src/core/queue.mjs && node --check src/core/repository-context.mjs && node --check src/core/repro-gate.mjs && node --check src/core/mcp-submission.mjs && node --check src/core/scout.mjs && node --check src/core/semantic-duplicate-assist.mjs && node --check src/core/serious-scout.mjs && node --check src/core/setup.mjs && node --check src/core/setup-guide.mjs && node --check src/core/shielded-posture.mjs && node --check src/core/text-safety.mjs && node --check src/core/vouch-context.mjs && node --check src/core/watchlist.mjs && node --check src/github/client.mjs && node --check src/github/webhook.mjs && node --check src/github/templates.mjs && node --check src/mcp/core.mjs && node --check src/mcp/server.mjs && node --check scripts/mcp-smoke.mjs && node --check scripts/run-adversary.mjs && node --check scripts/run-benchmark.mjs && node --check scripts/run-maintainer-demo.mjs && node --check scripts/run-public-pilot.mjs && node --check scripts/run-large-bench.mjs && node --check scripts/run-serious-scout.mjs && node --check scripts/run-watchlist.mjs && node --check scripts/verify-ci-workflow.mjs && node --check scripts/verify-repo-hygiene.mjs && node --check public/app.js", "mcp": "node src/mcp/server.mjs", "mcp:smoke": "node scripts/mcp-smoke.mjs", "setup:pilot": "node src/cli.mjs setup", @@ -80,7 +81,8 @@ "demo:pr": "node src/cli.mjs evaluate fixtures/pr-unready.json", "demo:issue": "node src/cli.mjs evaluate fixtures/issue-unready.json", "demo:kernel": "node src/cli.mjs evaluate fixtures/pr-kernel-ready.json --profile kernel-grade", - "preflight": "node src/cli.mjs preflight" + "preflight": "node src/cli.mjs preflight", + "validate:corpus": "node src/cli.mjs validate-corpus" }, "engines": { "node": ">=22" diff --git a/src/cli.mjs b/src/cli.mjs index 03572ce..c9e25e0 100644 --- a/src/cli.mjs +++ b/src/cli.mjs @@ -1,10 +1,17 @@ #!/usr/bin/env node import { readFile } from "node:fs/promises"; +import { basename } from "node:path"; import { evaluateContribution, renderMarkdownReport } from "./core/evaluator.mjs"; import { parsePatchSubmission } from "./core/patch.mjs"; import { normalizeRepositoryFiles } from "./core/policy.mjs"; import { buildMaintainerQueue } from "./core/queue.mjs"; import { buildSetupGuide, renderSetupGuideMarkdown, renderSetupGuideText } from "./core/setup-guide.mjs"; +import { + CorpusValidationError, + renderCorpusValidationMarkdown, + renderCorpusValidationSummary, + validateCorpusText +} from "./core/corpus-validation.mjs"; import { loadConfig } from "./config.mjs"; const args = process.argv.slice(2); @@ -15,7 +22,7 @@ if (args.length === 0 || args.includes("--help") || args.includes("-h")) { } const command = args[0]; -if (!["evaluate", "evaluate-patch", "queue", "setup", "setup-pilot", "preflight"].includes(command)) { +if (!["evaluate", "evaluate-patch", "queue", "setup", "setup-pilot", "preflight", "validate-corpus"].includes(command)) { console.error(`Unknown command: ${command}`); printHelp(); process.exit(2); @@ -40,6 +47,44 @@ if (command === "setup" || command === "setup-pilot") { process.exit(0); } +if (command === "validate-corpus") { + const file = args[1]; + if (!file) { + console.error("Missing corpus file."); + printHelp(); + process.exit(2); + } + const format = readFlag(args, "--format") || "pretty"; + if (!["pretty", "json", "markdown"].includes(format)) { + console.error(`Unsupported format: ${format}. Use pretty, json, or markdown.`); + process.exit(2); + } + const inputFormat = readFlag(args, "--input-format"); + try { + const text = file === "-" ? await readStdin() : await readFile(file, "utf8"); + const result = validateCorpusText(text, { + inputFormat, + sourceName: file === "-" ? "stdin" : basename(file) + }); + if (format === "json") { + console.log(JSON.stringify(result, null, 2)); + } else if (format === "markdown") { + process.stdout.write(renderCorpusValidationMarkdown(result)); + } else { + process.stdout.write(renderCorpusValidationSummary(result)); + } + process.exit(0); + } catch (error) { + const message = error instanceof CorpusValidationError + ? error.message + : error?.code === "ENOENT" + ? `Cannot read corpus file '${basename(file)}'.` + : "Unexpected corpus validation error."; + console.error(`PCF corpus validation failed: ${message}`); + process.exit(1); + } +} + const file = args[1]; if (!file) { console.error("Missing input file."); @@ -215,7 +260,9 @@ function printHelp() { node src/cli.mjs evaluate [--format pretty|json|markdown] [--profile standard|kernel-grade] node src/cli.mjs evaluate-patch [--format pretty|json|markdown] [--profile kernel-grade] [--policy policy-files.json] node src/cli.mjs preflight [--allow-repair] [--format pretty|json|markdown] [--profile standard|kernel-grade] [--policy policy-files.json] + node src/cli.mjs validate-corpus [--input-format jsonl|csv] [--format pretty|json|markdown] cat queue-payload.json | node src/cli.mjs queue - --format json -Preflight exit codes: 0 = ready to submit, 1 = not ready, 2 = usage error.`); +Preflight exit codes: 0 = ready to submit, 1 = not ready, 2 = usage error. +Corpus validation exit codes: 0 = corpus measured, 1 = validation failed closed, 2 = usage error.`); } diff --git a/src/core/corpus-validation.mjs b/src/core/corpus-validation.mjs new file mode 100644 index 0000000..c9afc80 --- /dev/null +++ b/src/core/corpus-validation.mjs @@ -0,0 +1,735 @@ +import { createHash } from "node:crypto"; + +export const CORPUS_VALIDATION_VERSION = "2026.07.13"; +export const VALIDATION_LANES = Object.freeze(["review-now", "repair", "defer"]); + +const MAX_CORPUS_BYTES = 10 * 1024 * 1024; +const MAX_CASES = 10_000; +const MAX_RATINGS_PER_CASE = 50; +const MAX_IDENTIFIER_CHARS = 256; + +const FORBIDDEN_RAW_FIELDS = new Set([ + "body", + "comment", + "comments", + "content", + "diff", + "patch", + "payload", + "raw", + "text", + "title" +]); + +const CSV_COLUMNS = Object.freeze([ + "id", + "repository", + "policyId", + "dataset", + "caseRef", + "policySnapshot", + "consentReference", + "consentAllowed", + "pcfLane", + "pcfScore", + "pcfNextActor", + "raterId", + "raterLane", + "raterNextActor", + "baselineSeconds", + "pcfSeconds" +]); + +const CALIBRATION_BINS = Object.freeze([ + { id: "0-19", min: 0, max: 19 }, + { id: "20-39", min: 20, max: 39 }, + { id: "40-59", min: 40, max: 59 }, + { id: "60-79", min: 60, max: 79 }, + { id: "80-100", min: 80, max: 100 } +]); + +export class CorpusValidationError extends Error { + constructor(message) { + super(message); + this.name = "CorpusValidationError"; + } +} + +export function validateCorpusText(text, { + inputFormat = "", + sourceName = "" +} = {}) { + const rawText = String(text ?? ""); + if (!rawText.trim()) throw new CorpusValidationError("Corpus input is empty."); + if (Buffer.byteLength(rawText, "utf8") > MAX_CORPUS_BYTES) { + throw new CorpusValidationError("Corpus input exceeds the 10 MiB offline validation limit."); + } + const format = resolveInputFormat(rawText, inputFormat, sourceName); + const parsed = format === "csv" ? parseCsvCorpus(rawText) : parseJsonlCorpus(rawText); + const cases = normalizeCases(parsed); + const sha256 = createHash("sha256").update(rawText, "utf8").digest("hex"); + return buildValidationResult(cases, { + format, + sha256, + sourceName: safeSourceName(sourceName) + }); +} + +export function renderCorpusValidationSummary(result = {}) { + const corpus = result.corpus || {}; + const review = result.lanes?.["review-now"] || {}; + const agreement = result.agreement || {}; + const timing = result.timing || {}; + return [ + "Premature Contribution Firewall corpus validation", + `Corpus: ${corpus.totalCases || 0} case(s), ${corpus.scoredCases || 0} scored, ${corpus.ambiguousCases || 0} ambiguous`, + `Corpus SHA-256: ${corpus.sha256 || ""}`, + `Review-now precision/recall: ${formatMetric(review.precision)} / ${formatMetric(review.recall)}`, + `False review-now: ${result.falseReviewNow?.count || 0}`, + `Inter-rater agreement: ${formatMetric(agreement.pairwiseAgreement)} (nominal kappa ${formatMetric(agreement.nominalKappa)})`, + `Paired timing: ${timing.pairedCases || 0} case(s), median ${formatSeconds(timing.medianSecondsSaved)} saved`, + `Decision boundary: ${result.decision?.status || "INCONCLUSIVE"}`, + "Non-claim: measurement only; no benchmark mutation, model training, or maintainer endorsement.", + "" + ].join("\n"); +} + +export function renderCorpusValidationMarkdown(result = {}) { + const corpus = result.corpus || {}; + const provenance = result.provenance || {}; + const agreement = result.agreement || {}; + const timing = result.timing || {}; + const lines = [ + "# PCF Corpus Validation Receipt", + "", + `- Result: **${result.ok ? "PASS" : "FAIL"}** for deterministic corpus measurement`, + `- Decision boundary: **${result.decision?.status || "INCONCLUSIVE"}**`, + `- Cases: ${corpus.totalCases || 0} total; ${corpus.scoredCases || 0} scored; ${corpus.ambiguousCases || 0} excluded as ambiguous`, + `- Ratings: ${corpus.totalRatings || 0} across ${agreement.distinctRaters || 0} caller-asserted rater IDs`, + `- Corpus SHA-256: \`${corpus.sha256 || ""}\``, + `- Input: \`${escapeCode(corpus.sourceName || "unspecified")}\` (${corpus.inputFormat || "unknown"})`, + "", + "## Evidence provenance", + "", + `- Datasets: ${formatCodeList(provenance.datasets)}`, + `- Policy snapshots: ${formatCodeList(provenance.policySnapshots)}`, + `- Repositories represented: ${provenance.repositories || 0}`, + `- Consented cases: ${provenance.consentedCases || 0}/${corpus.totalCases || 0}`, + "", + "## Lane metrics", + "", + "| Lane | Precision | Recall | TP | FP | FN | Support |", + "| --- | ---: | ---: | ---: | ---: | ---: | ---: |" + ]; + for (const lane of VALIDATION_LANES) { + const metrics = result.lanes?.[lane] || {}; + lines.push(`| \`${lane}\` | ${formatMetric(metrics.precision)} | ${formatMetric(metrics.recall)} | ${metrics.truePositive || 0} | ${metrics.falsePositive || 0} | ${metrics.falseNegative || 0} | ${metrics.support || 0} |`); + } + + lines.push( + "", + "## Confusion matrix", + "", + "Rows are PCF lanes; columns are strict-majority maintainer consensus.", + "", + "| PCF \\ Maintainer | review-now | repair | defer |", + "| --- | ---: | ---: | ---: | ---: |" + ); + for (const predicted of VALIDATION_LANES) { + const row = result.confusionMatrix?.[predicted] || {}; + lines.push(`| \`${predicted}\` | ${row["review-now"] || 0} | ${row.repair || 0} | ${row.defer || 0} |`); + } + + lines.push( + "", + "## False `review-now`", + "", + `Count: **${result.falseReviewNow?.count || 0}**` + ); + if (result.falseReviewNow?.cases?.length) { + lines.push( + "", + "| Opaque case | Repository | Policy | Approved case reference | Consensus | PCF score |", + "| --- | --- | --- | --- | --- | ---: |" + ); + for (const item of result.falseReviewNow.cases) { + lines.push(`| \`${escapeCode(item.id)}\` | \`${escapeCode(item.repository || "-")}\` | \`${escapeCode(item.policyId || "-")}\` | \`${escapeCode(item.caseRef || "-")}\` | \`${escapeCode(item.consensusLane)}\` | ${item.pcfScore} |`); + } + } else { + lines.push("", "None in this corpus."); + } + + lines.push( + "", + "## Inter-rater agreement", + "", + `- Eligible multi-rater cases: ${agreement.eligibleCases || 0}`, + `- Rating pairs: ${agreement.totalPairs || 0}`, + `- Pairwise agreement: ${formatMetric(agreement.pairwiseAgreement)}`, + `- Nominal kappa: ${formatMetric(agreement.nominalKappa)}`, + `- Next-actor pairwise agreement: ${formatMetric(agreement.nextActorPairwiseAgreement)}`, + "", + "Rater IDs are caller-asserted identifiers. This receipt does not verify identity or independence.", + "", + "## Score calibration", + "", + "| PCF score | Cases | Mean score | Consensus review-now | PCF review-now | Absolute gap |", + "| --- | ---: | ---: | ---: | ---: | ---: |" + ); + for (const bin of result.calibration || []) { + lines.push(`| ${bin.id} | ${bin.count} | ${formatMetric(bin.meanScore)} | ${formatMetric(bin.observedReviewNowRate)} | ${formatMetric(bin.predictedReviewNowRate)} | ${formatMetric(bin.absoluteGap)} |`); + } + + lines.push( + "", + "## Paired triage time", + "", + `- Paired cases: ${timing.pairedCases || 0}`, + `- Mean seconds saved: ${formatSeconds(timing.meanSecondsSaved)}`, + `- Median seconds saved: ${formatSeconds(timing.medianSecondsSaved)}`, + `- Median percent saved: ${formatPercent(timing.medianPercentSaved)}`, + `- Faster / equal / slower: ${timing.fasterCases || 0} / ${timing.equalCases || 0} / ${timing.slowerCases || 0}`, + "", + "## Exclusions", + "" + ); + if (result.exclusions?.length) { + for (const item of result.exclusions) lines.push(`- \`${escapeCode(item.id)}\`: ${item.reason}`); + } else { + lines.push("None."); + } + + lines.push( + "", + "## Boundaries and non-claims", + "", + `- **${result.decision?.status || "INCONCLUSIVE"}:** ${result.decision?.reason || "Measurement requires human interpretation."}` + ); + for (const claim of result.nonClaims || []) lines.push(`- ${claim}`); + return `${lines.join("\n")}\n`; +} + +function buildValidationResult(cases, { format, sha256, sourceName }) { + const confusionMatrix = emptyConfusionMatrix(); + const exclusions = []; + const scored = []; + for (const item of cases) { + const consensusLane = strictMajority(item.ratings.map((rating) => rating.lane)); + const consensusNextActor = strictMajority(item.ratings.map((rating) => rating.nextActor).filter(Boolean)); + if (!consensusLane) { + exclusions.push({ id: item.id, reason: "NO_STRICT_MAJORITY" }); + continue; + } + confusionMatrix[item.pcf.lane][consensusLane] += 1; + scored.push({ ...item, consensusLane, consensusNextActor }); + } + + const falseReviewCases = scored + .filter((item) => item.pcf.lane === "review-now" && item.consensusLane !== "review-now") + .map((item) => ({ + id: item.id, + repository: item.repository, + policyId: item.policyId, + caseRef: item.provenance.caseRef, + consensusLane: item.consensusLane, + pcfScore: item.pcf.score + })); + + return { + ok: true, + artifact: "pcf-corpus-validation", + version: CORPUS_VALIDATION_VERSION, + corpus: { + sourceName, + inputFormat: format, + sha256, + totalCases: cases.length, + scoredCases: scored.length, + ambiguousCases: exclusions.length, + totalRatings: cases.reduce((sum, item) => sum + item.ratings.length, 0) + }, + provenance: buildProvenance(cases), + confusionMatrix, + lanes: buildLaneMetrics(confusionMatrix), + falseReviewNow: { + count: falseReviewCases.length, + rateAmongPredictedReviewNow: ratio( + falseReviewCases.length, + scored.filter((item) => item.pcf.lane === "review-now").length + ), + cases: falseReviewCases + }, + agreement: buildAgreement(cases), + calibration: buildCalibration(scored), + timing: buildTiming(cases), + nextActor: buildNextActorMetrics(scored), + exclusions, + decision: { + status: "INCONCLUSIVE", + reason: "This command measures a consented corpus but cannot establish product validity, maintainer endorsement, or release readiness by itself." + }, + nonClaims: [ + "No evaluator weights, policies, fixtures, or permanent benchmark expectations were changed.", + "No model was trained and no corpus case was promoted automatically.", + "No network access or GitHub write is required by the validation core.", + "Consent, rater identity, rater independence, and provenance are caller assertions; PCF validates their declared shape, not their external truth.", + "Raw issue bodies, titles, patches, diffs, comments, text, and payloads are rejected by the corpus contract." + ] + }; +} + +function normalizeCases(rawCases) { + if (!Array.isArray(rawCases) || !rawCases.length) { + throw new CorpusValidationError("Corpus must contain at least one case."); + } + if (rawCases.length > MAX_CASES) { + throw new CorpusValidationError(`Corpus exceeds the ${MAX_CASES} case validation limit.`); + } + const ids = new Set(); + return rawCases.map((rawCase, index) => { + assertNoForbiddenFields(rawCase, `case ${index + 1}`); + const item = normalizeCase(rawCase, index); + if (ids.has(item.id)) throw new CorpusValidationError(`Duplicate case id '${item.id}'.`); + ids.add(item.id); + return item; + }); +} + +function normalizeCase(rawCase, index) { + if (!rawCase || typeof rawCase !== "object" || Array.isArray(rawCase)) { + throw new CorpusValidationError(`Case ${index + 1} must be an object.`); + } + assertAllowedKeys(rawCase, ["id", "repository", "policyId", "consent", "pcf", "ratings", "timing", "provenance"], `Case ${index + 1}`); + const id = requiredString(rawCase.id, `Case ${index + 1} id`); + const repository = boundedOptionalString(rawCase.repository, `Case '${id}' repository`); + const policyId = boundedOptionalString(rawCase.policyId, `Case '${id}' policyId`); + if (!repository && !policyId) { + throw new CorpusValidationError(`Case '${id}' requires repository or policyId provenance.`); + } + const consent = rawCase.consent; + if (!consent || consent.allowedForValidation !== true) { + throw new CorpusValidationError(`Case '${id}' requires consent.allowedForValidation=true and a consent reference.`); + } + assertAllowedKeys(consent, ["allowedForValidation", "reference"], `Case '${id}' consent`); + if (!boundedOptionalString(consent.reference, `Case '${id}' consent reference`)) { + throw new CorpusValidationError(`Case '${id}' requires consent.allowedForValidation=true and a consent reference.`); + } + const pcf = rawCase.pcf || {}; + assertAllowedKeys(pcf, ["lane", "score", "nextActor"], `Case '${id}' pcf`); + const lane = validLane(pcf.lane, `Case '${id}' PCF lane`); + const score = boundedNumber(pcf.score, 0, 100, `Case '${id}' PCF score`); + const ratings = Array.isArray(rawCase.ratings) ? rawCase.ratings : []; + if (!ratings.length) throw new CorpusValidationError(`Case '${id}' requires at least one maintainer rating.`); + if (ratings.length > MAX_RATINGS_PER_CASE) { + throw new CorpusValidationError(`Case '${id}' exceeds the ${MAX_RATINGS_PER_CASE} rating limit.`); + } + const raterIds = new Set(); + const normalizedRatings = ratings.map((rating, ratingIndex) => { + assertAllowedKeys(rating, ["raterId", "lane", "nextActor"], `Case '${id}' rating ${ratingIndex + 1}`); + const raterId = requiredString(rating?.raterId, `Case '${id}' rating ${ratingIndex + 1} raterId`); + if (raterIds.has(raterId)) throw new CorpusValidationError(`Case '${id}' has duplicate rater '${raterId}'.`); + raterIds.add(raterId); + return { + raterId, + lane: validLane(rating?.lane, `Case '${id}' rating ${ratingIndex + 1} lane`), + nextActor: boundedOptionalString(rating?.nextActor, `Case '${id}' rating ${ratingIndex + 1} nextActor`) + }; + }); + const provenance = rawCase.provenance || {}; + assertAllowedKeys(provenance, ["dataset", "caseRef", "policySnapshot", "collectedAt"], `Case '${id}' provenance`); + const normalizedProvenance = { + dataset: requiredString(provenance.dataset, `Case '${id}' provenance.dataset`), + caseRef: requiredString(provenance.caseRef, `Case '${id}' provenance.caseRef`), + policySnapshot: requiredString(provenance.policySnapshot, `Case '${id}' provenance.policySnapshot`), + collectedAt: boundedOptionalString(provenance.collectedAt, `Case '${id}' provenance.collectedAt`) + }; + return { + id, + repository, + policyId, + pcf: { lane, score, nextActor: boundedOptionalString(pcf.nextActor, `Case '${id}' pcf.nextActor`) }, + ratings: normalizedRatings, + timing: normalizeTiming(rawCase.timing, id), + provenance: normalizedProvenance + }; +} + +function normalizeTiming(timing, id) { + if (timing === undefined || timing === null) return null; + assertAllowedKeys(timing, ["baselineSeconds", "pcfSeconds"], `Case '${id}' timing`); + const baselinePresent = timing.baselineSeconds !== "" && timing.baselineSeconds !== undefined && timing.baselineSeconds !== null; + const pcfPresent = timing.pcfSeconds !== "" && timing.pcfSeconds !== undefined && timing.pcfSeconds !== null; + if (!baselinePresent && !pcfPresent) return null; + const baselineSeconds = Number(timing.baselineSeconds); + const pcfSeconds = Number(timing.pcfSeconds); + if (!(baselineSeconds > 0) || !(pcfSeconds >= 0)) { + throw new CorpusValidationError(`Case '${id}' timing requires baselineSeconds > 0 and pcfSeconds >= 0.`); + } + return { baselineSeconds, pcfSeconds }; +} + +function parseJsonlCorpus(text) { + const cases = []; + for (const [index, line] of text.split(/\r?\n/).entries()) { + if (!line.trim()) continue; + try { + cases.push(JSON.parse(line)); + } catch (error) { + throw new CorpusValidationError(`Invalid JSONL on line ${index + 1}: ${error.message}`); + } + } + return cases; +} + +function parseCsvCorpus(text) { + const rows = parseCsvRows(text).filter((row) => row.some((value) => value !== "")); + if (rows.length < 2) throw new CorpusValidationError("CSV corpus requires a header and at least one data row."); + const headers = rows[0].map((value) => value.trim()); + headers[0] = headers[0].replace(/^\uFEFF/, ""); + const duplicateHeaders = headers.filter((header, index) => headers.indexOf(header) !== index); + if (duplicateHeaders.length) throw new CorpusValidationError(`CSV corpus has duplicate column(s): ${unique(duplicateHeaders).join(", ")}.`); + const missing = CSV_COLUMNS.filter((column) => !headers.includes(column)); + if (missing.length) throw new CorpusValidationError(`CSV corpus is missing required column(s): ${missing.join(", ")}.`); + const unsupported = headers.filter((column) => !CSV_COLUMNS.includes(column)); + if (unsupported.length) throw new CorpusValidationError(`CSV corpus has unsupported column(s): ${unsupported.join(", ")}.`); + const groups = new Map(); + for (let rowIndex = 1; rowIndex < rows.length; rowIndex += 1) { + const values = rows[rowIndex]; + if (values.length > headers.length) throw new CorpusValidationError(`CSV row ${rowIndex + 1} has too many columns.`); + const row = Object.fromEntries(headers.map((header, index) => [header, values[index] ?? ""])); + const id = requiredString(row.id, `CSV row ${rowIndex + 1} id`); + const metadata = csvMetadata(row); + let group = groups.get(id); + if (!group) { + group = { + signature: JSON.stringify(metadata), + item: { + id, + repository: row.repository, + policyId: row.policyId, + consent: { + allowedForValidation: row.consentAllowed.trim().toLowerCase() === "true", + reference: row.consentReference + }, + pcf: { + lane: row.pcfLane, + score: row.pcfScore, + nextActor: row.pcfNextActor + }, + ratings: [], + timing: row.baselineSeconds || row.pcfSeconds + ? { baselineSeconds: row.baselineSeconds, pcfSeconds: row.pcfSeconds } + : null, + provenance: { + dataset: row.dataset, + caseRef: row.caseRef, + policySnapshot: row.policySnapshot + } + } + }; + groups.set(id, group); + } else if (group.signature !== JSON.stringify(metadata)) { + throw new CorpusValidationError(`CSV case '${id}' has inconsistent metadata across rater rows.`); + } + group.item.ratings.push({ + raterId: row.raterId, + lane: row.raterLane, + nextActor: row.raterNextActor + }); + } + return [...groups.values()].map((group) => group.item); +} + +function csvMetadata(row) { + return Object.fromEntries(CSV_COLUMNS + .filter((column) => !["raterId", "raterLane", "raterNextActor"].includes(column)) + .map((column) => [column, row[column]])); +} + +function parseCsvRows(text) { + const rows = []; + let row = []; + let field = ""; + let quoted = false; + for (let index = 0; index < text.length; index += 1) { + const char = text[index]; + if (quoted) { + if (char === '"' && text[index + 1] === '"') { + field += '"'; + index += 1; + } else if (char === '"') { + quoted = false; + } else { + field += char; + } + } else if (char === '"' && field === "") { + quoted = true; + } else if (char === ",") { + row.push(field); + field = ""; + } else if (char === "\n") { + row.push(field.replace(/\r$/, "")); + rows.push(row); + row = []; + field = ""; + } else { + field += char; + } + } + if (quoted) throw new CorpusValidationError("CSV input ends inside a quoted field."); + if (field !== "" || row.length) { + row.push(field.replace(/\r$/, "")); + rows.push(row); + } + return rows; +} + +function buildLaneMetrics(matrix) { + return Object.fromEntries(VALIDATION_LANES.map((lane) => { + const truePositive = matrix[lane][lane]; + const predicted = VALIDATION_LANES.reduce((sum, actual) => sum + matrix[lane][actual], 0); + const support = VALIDATION_LANES.reduce((sum, prediction) => sum + matrix[prediction][lane], 0); + return [lane, { + truePositive, + falsePositive: predicted - truePositive, + falseNegative: support - truePositive, + precision: ratio(truePositive, predicted), + recall: ratio(truePositive, support), + support + }]; + })); +} + +function buildAgreement(cases) { + const eligible = cases.filter((item) => item.ratings.length >= 2); + let totalPairs = 0; + let agreeingPairs = 0; + let nextActorPairs = 0; + let nextActorAgreeing = 0; + const laneCounts = Object.fromEntries(VALIDATION_LANES.map((lane) => [lane, 0])); + const raters = new Set(); + for (const item of cases) { + for (const rating of item.ratings) raters.add(rating.raterId); + } + for (const item of eligible) { + for (const rating of item.ratings) laneCounts[rating.lane] += 1; + for (let left = 0; left < item.ratings.length; left += 1) { + for (let right = left + 1; right < item.ratings.length; right += 1) { + totalPairs += 1; + if (item.ratings[left].lane === item.ratings[right].lane) agreeingPairs += 1; + if (item.ratings[left].nextActor && item.ratings[right].nextActor) { + nextActorPairs += 1; + if (item.ratings[left].nextActor === item.ratings[right].nextActor) nextActorAgreeing += 1; + } + } + } + } + const totalRatings = Object.values(laneCounts).reduce((sum, count) => sum + count, 0); + const observed = ratio(agreeingPairs, totalPairs); + const expected = totalRatings + ? round(Object.values(laneCounts).reduce((sum, count) => sum + (count / totalRatings) ** 2, 0)) + : null; + const nominalKappa = observed === null || expected === null || expected === 1 + ? null + : round((observed - expected) / (1 - expected)); + return { + eligibleCases: eligible.length, + distinctRaters: raters.size, + totalPairs, + agreeingPairs, + pairwiseAgreement: observed, + expectedAgreement: expected, + nominalKappa, + nextActorPairs, + nextActorPairwiseAgreement: ratio(nextActorAgreeing, nextActorPairs) + }; +} + +function buildCalibration(scored) { + return CALIBRATION_BINS.map((bin) => { + const items = scored.filter((item) => item.pcf.score >= bin.min && item.pcf.score <= bin.max); + const observed = ratio(items.filter((item) => item.consensusLane === "review-now").length, items.length); + const predicted = ratio(items.filter((item) => item.pcf.lane === "review-now").length, items.length); + return { + id: bin.id, + count: items.length, + meanScore: items.length ? round(mean(items.map((item) => item.pcf.score))) : null, + observedReviewNowRate: observed, + predictedReviewNowRate: predicted, + absoluteGap: observed === null || predicted === null ? null : round(Math.abs(observed - predicted)) + }; + }); +} + +function buildTiming(cases) { + const pairs = cases.filter((item) => item.timing).map((item) => { + const secondsSaved = item.timing.baselineSeconds - item.timing.pcfSeconds; + return { + secondsSaved, + percentSaved: (secondsSaved / item.timing.baselineSeconds) * 100 + }; + }); + return { + pairedCases: pairs.length, + meanSecondsSaved: pairs.length ? round(mean(pairs.map((item) => item.secondsSaved))) : null, + medianSecondsSaved: pairs.length ? round(median(pairs.map((item) => item.secondsSaved))) : null, + meanPercentSaved: pairs.length ? round(mean(pairs.map((item) => item.percentSaved))) : null, + medianPercentSaved: pairs.length ? round(median(pairs.map((item) => item.percentSaved))) : null, + fasterCases: pairs.filter((item) => item.secondsSaved > 0).length, + equalCases: pairs.filter((item) => item.secondsSaved === 0).length, + slowerCases: pairs.filter((item) => item.secondsSaved < 0).length + }; +} + +function buildNextActorMetrics(scored) { + const eligible = scored.filter((item) => item.consensusNextActor && item.pcf.nextActor); + const matching = eligible.filter((item) => item.consensusNextActor === item.pcf.nextActor).length; + return { eligibleCases: eligible.length, matchingCases: matching, accuracy: ratio(matching, eligible.length) }; +} + +function buildProvenance(cases) { + return { + consentedCases: cases.length, + datasets: unique(cases.map((item) => item.provenance.dataset)).sort(), + policySnapshots: unique(cases.map((item) => item.provenance.policySnapshot)).sort(), + repositories: unique(cases.map((item) => item.repository).filter(Boolean)).length, + policyIds: unique(cases.map((item) => item.policyId).filter(Boolean)).length, + caseReferences: unique(cases.map((item) => item.provenance.caseRef)).length + }; +} + +function emptyConfusionMatrix() { + return Object.fromEntries(VALIDATION_LANES.map((predicted) => [ + predicted, + Object.fromEntries(VALIDATION_LANES.map((actual) => [actual, 0])) + ])); +} + +function strictMajority(values) { + if (!values.length) return ""; + const counts = new Map(); + for (const value of values) counts.set(value, (counts.get(value) || 0) + 1); + const ordered = [...counts.entries()].sort((left, right) => right[1] - left[1] || left[0].localeCompare(right[0])); + return ordered[0][1] > values.length / 2 ? ordered[0][0] : ""; +} + +function assertNoForbiddenFields(value, path) { + if (!value || typeof value !== "object") return; + if (Array.isArray(value)) { + value.forEach((item, index) => assertNoForbiddenFields(item, `${path}[${index}]`)); + return; + } + for (const [key, nested] of Object.entries(value)) { + const normalizedKey = key.toLowerCase().replace(/[^a-z]/g, ""); + if (FORBIDDEN_RAW_FIELDS.has(normalizedKey)) { + throw new CorpusValidationError(`${path} contains forbidden raw-content field '${key}'.`); + } + assertNoForbiddenFields(nested, `${path}.${key}`); + } +} + +function assertAllowedKeys(value, allowed, path) { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new CorpusValidationError(`${path} must be an object.`); + } + const unsupported = Object.keys(value).filter((key) => !allowed.includes(key)); + if (unsupported.length) { + throw new CorpusValidationError(`${path} has unsupported field(s): ${unsupported.join(", ")}.`); + } +} + +function resolveInputFormat(text, requested, sourceName) { + const format = optionalString(requested).toLowerCase(); + if (format) { + if (!new Set(["jsonl", "csv"]).has(format)) throw new CorpusValidationError(`Unsupported input format '${format}'. Use jsonl or csv.`); + return format; + } + if (/\.csv$/i.test(sourceName)) return "csv"; + if (/\.jsonl$/i.test(sourceName)) return "jsonl"; + return text.trimStart().startsWith("{") ? "jsonl" : "csv"; +} + +function validLane(value, label) { + const lane = requiredString(value, label); + if (!VALIDATION_LANES.includes(lane)) { + throw new CorpusValidationError(`${label} must be one of: ${VALIDATION_LANES.join(", ")}.`); + } + return lane; +} + +function boundedNumber(value, min, max, label) { + const number = Number(value); + if (!Number.isFinite(number) || number < min || number > max) { + throw new CorpusValidationError(`${label} must be a number from ${min} to ${max}.`); + } + return number; +} + +function requiredString(value, label) { + const normalized = boundedOptionalString(value, label); + if (!normalized) throw new CorpusValidationError(`${label} is required.`); + return normalized; +} + +function boundedOptionalString(value, label) { + const normalized = optionalString(value); + if (!normalized) return ""; + if (normalized.length > MAX_IDENTIFIER_CHARS) { + throw new CorpusValidationError(`${label} exceeds ${MAX_IDENTIFIER_CHARS} characters.`); + } + if (/[\u0000-\u001f\u007f]/u.test(normalized)) { + throw new CorpusValidationError(`${label} must be a single-line identifier without control characters.`); + } + return normalized; +} + +function optionalString(value) { + return value === undefined || value === null ? "" : String(value).trim(); +} + +function safeSourceName(value) { + const normalized = optionalString(value); + return normalized ? normalized.split(/[\\/]/).pop() : "unspecified"; +} + +function ratio(numerator, denominator) { + return denominator ? round(numerator / denominator) : null; +} + +function mean(values) { + return values.reduce((sum, value) => sum + value, 0) / values.length; +} + +function median(values) { + const sorted = [...values].sort((left, right) => left - right); + const middle = Math.floor(sorted.length / 2); + return sorted.length % 2 ? sorted[middle] : (sorted[middle - 1] + sorted[middle]) / 2; +} + +function round(value) { + return Number(Number(value).toFixed(4)); +} + +function unique(values) { + return [...new Set(values)]; +} + +function formatMetric(value) { + return value === null || value === undefined ? "n/a" : String(value); +} + +function formatSeconds(value) { + return value === null || value === undefined ? "n/a" : `${value}s`; +} + +function formatPercent(value) { + return value === null || value === undefined ? "n/a" : `${value}%`; +} + +function formatCodeList(values = []) { + return values.length ? values.map((value) => `\`${escapeCode(value)}\``).join(", ") : "none"; +} + +function escapeCode(value) { + return String(value ?? "").replaceAll("`", "").replaceAll("|", "\\|").replace(/\r?\n/g, " "); +} diff --git a/test/corpus-validation.test.mjs b/test/corpus-validation.test.mjs new file mode 100644 index 0000000..6aaaa53 --- /dev/null +++ b/test/corpus-validation.test.mjs @@ -0,0 +1,272 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { spawnSync } from "node:child_process"; +import { + CorpusValidationError, + renderCorpusValidationMarkdown, + renderCorpusValidationSummary, + validateCorpusText +} from "../src/core/corpus-validation.mjs"; + +test("JSONL validation measures lanes, false review-now, agreement, calibration, and timing", () => { + const result = validateCorpusText(validJsonl(), { + inputFormat: "jsonl", + sourceName: "consented.jsonl" + }); + + assert.equal(result.ok, true); + assert.equal(result.artifact, "pcf-corpus-validation"); + assert.equal(result.corpus.totalCases, 5); + assert.equal(result.corpus.scoredCases, 4); + assert.equal(result.corpus.ambiguousCases, 1); + assert.equal(result.corpus.totalRatings, 10); + assert.equal(result.confusionMatrix["review-now"]["review-now"], 1); + assert.equal(result.confusionMatrix["review-now"].repair, 1); + assert.equal(result.confusionMatrix.defer["review-now"], 1); + assert.equal(result.lanes["review-now"].precision, 0.5); + assert.equal(result.lanes["review-now"].recall, 0.5); + assert.equal(result.lanes.repair.precision, 1); + assert.equal(result.lanes.repair.recall, 0.5); + assert.equal(result.falseReviewNow.count, 1); + assert.deepEqual(result.falseReviewNow.cases, [{ + id: "case-2", + repository: "owner/repo", + policyId: "policy-v1", + caseRef: "alpha-2", + consensusLane: "repair", + pcfScore: 84 + }]); + assert.equal(result.agreement.eligibleCases, 5); + assert.equal(result.agreement.pairwiseAgreement, 0.8); + assert.equal(result.agreement.nominalKappa, 0.6); + assert.equal(result.calibration.find((bin) => bin.id === "80-100").count, 2); + assert.equal(result.timing.pairedCases, 2); + assert.equal(result.timing.meanSecondsSaved, 34); + assert.equal(result.timing.medianPercentSaved, 30); + assert.deepEqual(result.provenance.datasets, ["alpha"]); + assert.deepEqual(result.provenance.policySnapshots, ["policy-v1@abc123"]); + assert.match(result.corpus.sha256, /^[a-f0-9]{64}$/); + assert.equal(result.decision.status, "INCONCLUSIVE"); +}); + +test("CSV validation groups one row per rater into consented cases", () => { + const result = validateCorpusText(validCsv(), { + inputFormat: "csv", + sourceName: "consented.csv" + }); + + assert.equal(result.corpus.totalCases, 2); + assert.equal(result.corpus.totalRatings, 4); + assert.equal(result.corpus.scoredCases, 2); + assert.equal(result.falseReviewNow.count, 1); + assert.equal(result.agreement.pairwiseAgreement, 1); + assert.deepEqual(result.provenance.datasets, ["csv-alpha"]); +}); + +test("validation fails closed on consent, privacy, schema, timing, and duplicate raters", () => { + const missingConsent = caseRecord({ id: "missing-consent" }); + delete missingConsent.consent; + assert.throws( + () => validateCorpusText(`${JSON.stringify(missingConsent)}\n`, { inputFormat: "jsonl" }), + (error) => error instanceof CorpusValidationError && /consent/i.test(error.message) + ); + + const rawContent = caseRecord({ id: "raw-content", body: "private issue body" }); + assert.throws( + () => validateCorpusText(`${JSON.stringify(rawContent)}\n`, { inputFormat: "jsonl" }), + (error) => error instanceof CorpusValidationError && /forbidden raw-content field 'body'/i.test(error.message) + ); + + const unknownContent = caseRecord({ id: "unknown-content", description: "undeclared private material" }); + assert.throws( + () => validateCorpusText(`${JSON.stringify(unknownContent)}\n`, { inputFormat: "jsonl" }), + (error) => error instanceof CorpusValidationError && /unsupported field.*description/i.test(error.message) + ); + + const invalidTiming = caseRecord({ + id: "invalid-timing", + timing: { baselineSeconds: "unknown", pcfSeconds: "unknown" } + }); + assert.throws( + () => validateCorpusText(`${JSON.stringify(invalidTiming)}\n`, { inputFormat: "jsonl" }), + (error) => error instanceof CorpusValidationError && /timing requires/i.test(error.message) + ); + + const duplicateRater = caseRecord({ + id: "duplicate-rater", + ratings: [rating("maintainer-a", "repair"), rating("maintainer-a", "repair")] + }); + assert.throws( + () => validateCorpusText(`${JSON.stringify(duplicateRater)}\n`, { inputFormat: "jsonl" }), + (error) => error instanceof CorpusValidationError && /duplicate rater/i.test(error.message) + ); + + const tooManyRatings = caseRecord({ + id: "too-many-ratings", + ratings: Array.from({ length: 51 }, (_, index) => rating(`rater-${index}`, "repair")) + }); + assert.throws( + () => validateCorpusText(`${JSON.stringify(tooManyRatings)}\n`, { inputFormat: "jsonl" }), + (error) => error instanceof CorpusValidationError && /50 rating limit/i.test(error.message) + ); + + const multilineReference = caseRecord({ + id: "multiline-reference", + provenance: provenance("line-one\nline-two") + }); + assert.throws( + () => validateCorpusText(`${JSON.stringify(multilineReference)}\n`, { inputFormat: "jsonl" }), + (error) => error instanceof CorpusValidationError && /single-line identifier/i.test(error.message) + ); + + const csvWithExtraColumn = validCsv().replace("pcfSeconds\n", "pcfSeconds,body\n"); + assert.throws( + () => validateCorpusText(csvWithExtraColumn, { inputFormat: "csv" }), + (error) => error instanceof CorpusValidationError && /unsupported column.*body/i.test(error.message) + ); +}); + +test("strict-majority ties are excluded explicitly and identical bytes replay deterministically", () => { + const text = `${JSON.stringify(caseRecord({ + id: "tie-case", + ratings: [rating("maintainer-a", "review-now"), rating("maintainer-b", "repair")] + }))}\n`; + const first = validateCorpusText(text, { inputFormat: "jsonl", sourceName: "stdin" }); + const second = validateCorpusText(text, { inputFormat: "jsonl", sourceName: "stdin" }); + + assert.equal(first.corpus.scoredCases, 0); + assert.deepEqual(first.exclusions, [{ id: "tie-case", reason: "NO_STRICT_MAJORITY" }]); + assert.deepEqual(first, second); +}); + +test("receipts are concise, aggregate, and omit rater identities and local paths", () => { + const result = validateCorpusText(validJsonl(), { inputFormat: "jsonl", sourceName: "consented.jsonl" }); + const markdown = renderCorpusValidationMarkdown(result); + const summary = renderCorpusValidationSummary(result); + + assert.match(markdown, /False `review-now`/); + assert.match(markdown, /Inter-rater agreement/); + assert.match(markdown, /INCONCLUSIVE/); + assert.match(summary, /False review-now: 1/); + assert.equal(markdown.includes("maintainer-a"), false); + assert.equal(markdown.includes("private issue body"), false); + assert.equal(markdown.includes("/home/"), false); +}); + +test("CLI emits JSON and Markdown offline and returns a concise validation error", async () => { + const dir = await mkdtemp(join(tmpdir(), "pcf-corpus-validation-")); + const validPath = join(dir, "consented.jsonl"); + const invalidPath = join(dir, "unconsented.jsonl"); + await writeFile(validPath, validJsonl()); + const unconsented = caseRecord({ id: "unconsented" }); + delete unconsented.consent; + await writeFile(invalidPath, `${JSON.stringify(unconsented)}\n`); + + try { + const jsonRun = runCli(["validate-corpus", validPath, "--format", "json"]); + assert.equal(jsonRun.status, 0, jsonRun.stderr); + const parsed = JSON.parse(jsonRun.stdout); + assert.equal(parsed.corpus.totalCases, 5); + assert.equal(parsed.decision.status, "INCONCLUSIVE"); + + const markdownRun = runCli(["validate-corpus", validPath, "--format", "markdown"]); + assert.equal(markdownRun.status, 0, markdownRun.stderr); + assert.match(markdownRun.stdout, /# PCF Corpus Validation Receipt/); + + const invalidRun = runCli(["validate-corpus", invalidPath]); + assert.equal(invalidRun.status, 1); + assert.match(invalidRun.stderr, /^PCF corpus validation failed:/); + assert.equal(invalidRun.stderr.includes(" at "), false); + + const missingRun = runCli(["validate-corpus", join(dir, "missing.jsonl")]); + assert.equal(missingRun.status, 1); + assert.match(missingRun.stderr, /^PCF corpus validation failed: Cannot read corpus file 'missing.jsonl'\./); + assert.equal(missingRun.stderr.includes("ENOENT"), false); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +function validJsonl() { + return [ + caseRecord({ + id: "case-1", + pcf: { lane: "review-now", score: 92, nextActor: "maintainer" }, + ratings: [rating("maintainer-a", "review-now"), rating("maintainer-b", "review-now")], + timing: { baselineSeconds: 120, pcfSeconds: 72 }, + provenance: provenance("alpha-1") + }), + caseRecord({ + id: "case-2", + pcf: { lane: "review-now", score: 84, nextActor: "maintainer" }, + ratings: [rating("maintainer-a", "repair"), rating("maintainer-b", "repair")], + timing: { baselineSeconds: 100, pcfSeconds: 80 }, + provenance: provenance("alpha-2") + }), + caseRecord({ + id: "case-3", + pcf: { lane: "repair", score: 65, nextActor: "reporter" }, + ratings: [rating("maintainer-a", "repair"), rating("maintainer-b", "repair")], + provenance: provenance("alpha-3") + }), + caseRecord({ + id: "case-4", + pcf: { lane: "defer", score: 40, nextActor: "maintainer" }, + ratings: [rating("maintainer-a", "review-now"), rating("maintainer-b", "review-now")], + provenance: provenance("alpha-4") + }), + caseRecord({ + id: "case-5", + pcf: { lane: "repair", score: 58, nextActor: "reporter" }, + ratings: [rating("maintainer-a", "review-now"), rating("maintainer-b", "repair")], + provenance: provenance("alpha-5") + }) + ].map((item) => JSON.stringify(item)).join("\n") + "\n"; +} + +function validCsv() { + return [ + "id,repository,policyId,dataset,caseRef,policySnapshot,consentReference,consentAllowed,pcfLane,pcfScore,pcfNextActor,raterId,raterLane,raterNextActor,baselineSeconds,pcfSeconds", + "csv-1,owner/repo,policy-v1,csv-alpha,csv-1,policy-v1@abc,consent-csv,true,review-now,90,maintainer,maintainer-a,review-now,maintainer,100,70", + "csv-1,owner/repo,policy-v1,csv-alpha,csv-1,policy-v1@abc,consent-csv,true,review-now,90,maintainer,maintainer-b,review-now,maintainer,100,70", + "csv-2,owner/repo,policy-v1,csv-alpha,csv-2,policy-v1@abc,consent-csv,true,review-now,82,maintainer,maintainer-a,repair,reporter,,", + "csv-2,owner/repo,policy-v1,csv-alpha,csv-2,policy-v1@abc,consent-csv,true,review-now,82,maintainer,maintainer-b,repair,reporter,," + ].join("\n") + "\n"; +} + +function caseRecord(overrides = {}) { + return { + id: "case", + repository: "owner/repo", + policyId: "policy-v1", + consent: { allowedForValidation: true, reference: "consent-alpha" }, + pcf: { lane: "repair", score: 60, nextActor: "reporter" }, + ratings: [rating("maintainer-a", "repair")], + provenance: provenance("alpha-case"), + ...overrides + }; +} + +function rating(raterId, lane, nextActor = lane === "repair" ? "reporter" : "maintainer") { + return { raterId, lane, nextActor }; +} + +function provenance(caseRef) { + return { + dataset: "alpha", + caseRef, + policySnapshot: "policy-v1@abc123", + collectedAt: "2026-07-13T12:00:00Z" + }; +} + +function runCli(args) { + return spawnSync(process.execPath, ["src/cli.mjs", ...args], { + cwd: process.cwd(), + encoding: "utf8", + env: { ...process.env, http_proxy: "http://127.0.0.1:9", https_proxy: "http://127.0.0.1:9" } + }); +}