diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f1c585ff..98f25a2c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,7 +68,7 @@ jobs: run: python scripts/check_oss_boundary.py --root . --allowlist config/oss_boundary_allowlist.json - name: Lint committed BENCHMARK.md fixtures for leak patterns if: ${{ needs.classify-changes.outputs.docs_only != 'true' }} - run: python scripts/ci/check_public_benchmarks.py --require-files tests/golden + run: uv run python scripts/ci/check_public_benchmarks.py --require-files tests/golden - name: Lint if: ${{ needs.classify-changes.outputs.docs_only != 'true' }} run: uv run ruff check . @@ -302,6 +302,7 @@ jobs: tests/test_cli.py tests/test_harbor_output_provenance.py tests/test_harbor_runtime_skill_isolation.py + tests/test_publication_identity.py tests/test_harbor_secure_copy.py::test_checked_windows_fallback_accepts_crt_descriptor_identity tests/test_harbor_secure_copy.py::test_checked_windows_fallback_verifies_portable_chmod_semantics tests/test_harbor_secure_copy.py::test_native_windows_fallback_copies_tree_and_file diff --git a/CHANGELOG.md b/CHANGELOG.md index 0fc1c927..8799b690 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -107,6 +107,46 @@ All notable changes to SkillEvaluator are documented in this file. conversion limit, preserves nonzero Wilson interval widths and paired-effect directions at large case counts, and documents exact-rational omission markers. +- `BENCHMARK.md` publication verdicts now require completed Tier 1 and Tier 2 + execution evidence by default, conservatively resolve conflicting peer + policy and Tier 3 result metadata, bind every tier and policy claim to one + versioned source-tree digest, preserve the Tier 3 run ID, persist an explicit + publication status in JSON and HTML, and reject + publication `PASS` cards whose decision evidence is missing, incomplete, + linked, or hidden in raw HTML. Tier 3 evidence now cross-checks complete + per-condition attempt and error records, and the public-card gate rejects + ambiguous structural headings, browser-rendered decision-text aliases, and + literal or entity-encoded Unicode control/format characters. Custom Tier 3 + result roots inside the skill are rejected; use the canonical `evals/results` + path or an external results root. +- Clean and not-applicable License, Code Risk, Secrets, and Dependency Audit + runs now record structured execution evidence, so legitimate built-in Tier 1 + results can satisfy publication without relying on findings or errors. +- Advisory skip reasons are bounded, flattened, control-free, and escaped for + terminal and Markdown output, preventing oversized metadata, Rich markup, + terminal escape sequences, or multiline text from corrupting reports. +- Malformed SkillSpector issue paths containing NUL bytes or lone surrogates + now make the scan incomplete instead of crashing filesystem alias checks. +- Publication source identity now seals forward, reverse, and final source reads, + normalizes filesystem case aliases, aligns Tier 1/Tier 2 and Tier 3 runtime + projections with the v2 generated-artifact exclusions, and carries a bounded + source-change marker through rerendered Tier 3 evidence. Tier 3 runs with any + staged repository projection or external reference/workspace skills remain + runnable but cannot certify publication until those inputs have their own + versioned identity. +- Report output is required outside the publication target. A default + in-target `reports/` location relocates to an authenticated sibling + `-reports` (or `-reports`), refuses unowned collisions, and is + excluded from Tier 3 runtime staging. Report roots inside the target repository + or an included-skill root are excluded from linked or full-repository context + only after they are reserved or authenticated as generated output, preventing + an authored subtree from being silently omitted. +- Public benchmark provenance now rejects placeholder identities, malformed or + future calendar dates, missing duplicated run IDs, hostile benchmark-policy + metadata, and environment-label substitutions in required proof fields. +- Hardened Tier 3 HTML and Markdown reporting against recursive or malformed + display metadata, Markdown structure injection, and inline-JavaScript + injection through untrusted agent or skill names. - Tier 3 now decodes bounded native Codex `exec` wrappers into their static tool calls. It preserves call order and outer-call provenance, maps an outer observation only when its rendered inner call is known, keeps ambiguous diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 790b89b9..66dd7530 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -10,3 +10,46 @@ records the exact resolved dependency set used for this release. | LLM | Anthropic (MIT), Boto3 (Apache-2.0), LiteLLM (MIT), OpenAI (Apache-2.0) | | Tier 3 | Harbor (Apache-2.0) | | Security | Bandit (Apache-2.0), pip-audit (Apache-2.0) | + +## Unicode security data + +`src/skillevaluator/publication_text.py` contains Unicode 15.1 general-category +ranges and a generated subset of the Unicode 17.0.0 `confusables.txt` data used +by Unicode Technical Standard #39. + +UNICODE LICENSE V3 + +COPYRIGHT AND PERMISSION NOTICE + +Copyright © 1991-2026 Unicode, Inc. + +NOTICE TO USER: Carefully read the following legal agreement. BY DOWNLOADING, +INSTALLING, COPYING OR OTHERWISE USING DATA FILES, AND/OR SOFTWARE, YOU +UNEQUIVOCALLY ACCEPT, AND AGREE TO BE BOUND BY, ALL OF THE TERMS AND CONDITIONS +OF THIS AGREEMENT. IF YOU DO NOT AGREE, DO NOT DOWNLOAD, INSTALL, COPY, +DISTRIBUTE OR USE THE DATA FILES OR SOFTWARE. + +Permission is hereby granted, free of charge, to any person obtaining a copy of +data files and any associated documentation (the "Data Files") or software and +any associated documentation (the "Software") to deal in the Data Files or +Software without restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, and/or sell copies of the Data Files +or Software, and to permit persons to whom the Data Files or Software are +furnished to do so, provided that either (a) this copyright and permission +notice appear with all copies of the Data Files or Software, or (b) this +copyright and permission notice appear in associated Documentation. + +THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD +PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS +NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL +DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, +WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING +OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THE DATA FILES OR +SOFTWARE. + +Except as contained in this notice, the name of a copyright holder shall not be +used in advertising or otherwise to promote the sale, use or other dealings in +these Data Files or Software without prior written authorization of the +copyright holder. diff --git a/docs/benchmark-rollout.mdx b/docs/benchmark-rollout.mdx index bba00384..75a9b7de 100644 --- a/docs/benchmark-rollout.mdx +++ b/docs/benchmark-rollout.mdx @@ -56,13 +56,43 @@ skillevaluator validate ./skills/example-skill \ --output-dir ./benchmark-backfill/example-skill ``` -Tier 3 is publication-required by default. Omitting `--agent-eval` therefore -produces an `INCOMPLETE` card with Tier 3 marked `NOT RUN`, and never a -publication recommendation. If a catalog intentionally makes Tier 3 optional, -its orchestration must persist `benchmark_policy.tier3_required = false`; the -card then discloses that policy. Do not reuse a prior live score while changing -only its label or date. If a baseline was not run, keep the explicit “uplift -unavailable” state. +Tier 1 is always publication-required. Tier 2 and Tier 3 are +publication-required by default. Omitting Tier 2 with `--no-dedup` or +`--tiers 1,3`, or omitting Tier 3 by leaving out `--agent-eval`, therefore +produces an `INCOMPLETE` card with the omitted tier marked `NOT RUN` and never a +publication recommendation. `--no-block-on-dedup` changes the command exit +gate; it is not a publication waiver. + +If a catalog intentionally makes Tier 2 or Tier 3 optional, its orchestration +must persist `benchmark_policy.tier2_required = false` or +`benchmark_policy.tier3_required = false`. The card then discloses that policy. +An optional tier that produced incomplete evidence still makes the card +`INCOMPLETE`; only absent or cleanly skipped evidence can use the waiver. +Do not reuse a prior live score while changing only its label or date. If a +baseline was not run, keep the explicit “uplift unavailable” state. + +When an orchestrator combines tiers from separate jobs, merge the actual result +objects, including each result's `publication_target` and the built-in Tier 1 +or Tier 2 `publication_evidence` producer marker, and preserve the explicit +`benchmark_policy` in the combined artifact. Every contributing result must +carry the same exact NFC filesystem-entry name and +`skill-evaluator-source-tree/2` digest. Completed Tier 3 evidence +must also preserve its run-owned `run_id` in the payload and summary. Anonymous +legacy results, a changed source tree, or inconsistent run identity make the +combined publication status `INCOMPLETE`. +Do not infer a waiver from an unselected tier or a non-blocking job. A missing +result or policy key defaults to required, so an incomplete split-tier merge +fails closed. Peer result objects have equal precedence: conflicting required +flags resolve to `true` independent of merge order. Agent-evaluation payload +metadata and its summary are duplicated higher-precedence policy claims that +must agree. Those claims are used only when the containing result is bound to +the same exact source digest; a foreign or anonymous payload cannot create a +waiver. + +Use the combined JSON report's top-level `publication_status` or +`publication.status` for publication automation. Do not substitute +`overall_status`, which records the process gate and can remain `passed` when +publication evidence is incomplete. The candidate card is: @@ -87,7 +117,7 @@ Expected presentation changes include: - removal of the `Num` column; - explicit `NOT RUN`, `SKIPPED`, and `INCOMPLETE` tier states; - collapsible methodology and non-blocking observations; -- evaluator version, dataset digest, task composition, Tier 3 requirement, +- evaluator version, dataset digest, task composition, Tier 2 and Tier 3 requirements, isolation wording, and freshness copy. Investigate any numerical change. The canonical mapping is Security=`security`, @@ -105,7 +135,7 @@ and does not override that gate. Scan all candidates before copying them into skills: ```bash -python scripts/ci/check_public_benchmarks.py \ +uv run python scripts/ci/check_public_benchmarks.py \ --require-files \ ./benchmark-backfill ``` @@ -113,7 +143,14 @@ python scripts/ci/check_public_benchmarks.py \ The linter rejects configured known leak patterns such as retired product identities, any validation-profile metadata line, common absolute home paths, ambiguous legacy uplift cells, the old `Num` column, missing metadata/decision -sections, and a publication `PASS` without completed required Tier 3 evidence. +sections, and a publication `PASS` without completed required Tier 1, Tier 2, +or Tier 3 evidence. +Structural H1/H2 headings must be undecorated, root-level Markdown; raw HTML, +formatted, quoted, and list-nested headings are rejected as ambiguous. Decision +text is checked using its rendered form, including line breaks and image alt +text. Unicode category C, Zl, and Zp characters are rejected whether literal or +encoded as HTML character references, and confusable aliases of required +headings, metadata labels, tier-table structure, and decision text fail closed. A clean result means none of those configured patterns matched; it is a fixture regression guard, not proof that a card is safe to publish. Keep human review and the repository's broader boundary and security checks in the promotion @@ -128,7 +165,7 @@ catalog: cp ./benchmark-backfill/example-skill/BENCHMARK.md \ ./skills/example-skill/BENCHMARK.md -python scripts/ci/check_public_benchmarks.py --require-files ./skills +uv run python scripts/ci/check_public_benchmarks.py --require-files ./skills git diff --check git diff -- ./skills/example-skill/BENCHMARK.md ``` @@ -149,6 +186,8 @@ Regenerate a card when any of these inputs changes: - attempt count/pass threshold; - execution environment or isolation mode. -The generated evaluation date must come from the live run artifact. Older -artifacts without an unambiguous timestamp remain “not recorded”; never replace -that state with the report-generation date. +The generated evaluation date must come from the live run artifact as a +timezone-aware ISO timestamp. Date-only, timezone-naive, malformed, or +materially future-dated values are not publication evidence. Older artifacts +without an unambiguous timestamp remain “not recorded”; never replace that state +with the report-generation date. diff --git a/docs/ci-integration.mdx b/docs/ci-integration.mdx index 99ca3d16..4bf6e9d0 100644 --- a/docs/ci-integration.mdx +++ b/docs/ci-integration.mdx @@ -233,11 +233,22 @@ When the exit code is not enough — dashboards, custom thresholds, badge genera | --- | --- | --- | | `overall_passed` | boolean | `true` when every check passed | | `overall_status` | string | `passed`, `failed`, or `incomplete` (a required scanner produced no evidence) | +| `publication_status` | string | `pass`, `neutral`, `fail`, or `incomplete`; use this field for publication eligibility | +| `publication` | object | Publication decision, eligibility, reasons, and Tier 3 evidence status | +| `benchmark_policy` | object | Persisted `tier2_required` and `tier3_required` evidence policy used for the publication decision | +| `results[].publication_target` | object | Exact NFC filesystem-entry name, versioned source-tree digest, and digest algorithm used to bind split-tier evidence | +| `results[].publication_evidence` | object | Validated built-in Tier 1 or Tier 2 producer contract: schema version, producer, tier, and canonical check ID | +| `tier3.run_id` | string | Run-owned Tier 3 identity; duplicated in the Tier 3 summary for consistency checks | | `severity_counts` | object | Totals for `critical`, `high`, `medium`, `low` | | `total_errors`, `total_warnings` | number | Aggregate counts across all validators | | `skills` | array | Per-skill `{ name, passed, issue_count }` | | `quality_summary` | array | Quality details with `overall_score` (0–100) and `grade` (A–F); a folder target reports the collection average plus a `skill_count` | +`overall_status` and `overall_passed` describe the configured command gate. They +can be green when an advisory tier did not run, so do not use them as a +publication signal. Require `publication_status == "pass"` (or +`publication.eligible == true`) before publishing a benchmark. + A step that enforces a stricter score than the built-in gate: ```bash title="Gate on quality score with jq" diff --git a/docs/environment-variables.mdx b/docs/environment-variables.mdx index d981158a..747527a3 100644 --- a/docs/environment-variables.mdx +++ b/docs/environment-variables.mdx @@ -101,7 +101,7 @@ execution environments are covered in | Variable | Default | Effect | | --- | --- | --- | -| `SKILLEVALUATOR_RESULTS_DIR` | `/evals/results` | External root for run results. Precedence for writes: the `--results-dir` flag, then this variable, then the legacy in-skill location. Read commands (`view`, `compare`) honor the same order and also fall back to the legacy location so older runs stay visible. | +| `SKILLEVALUATOR_RESULTS_DIR` | `/evals/results` | External root for run results. Precedence for writes: the `--results-dir` flag, then this variable, then the canonical in-skill location. Configured roots must remain outside the skill; `evals/results` is the only supported in-skill results root so generated output cannot change the skill's publication identity. Read commands (`view`, `compare`) honor the same order and also fall back to the legacy location so older runs stay visible. | | `SKILLEVALUATOR_LOCAL_SANDBOX` | `require` | Local-mode sandbox policy. `require` fails closed when no OS sandbox backend (Bubblewrap on Linux, Seatbelt on macOS) is usable; `prefer` degrades to advisory-only guardrails with a loud warning; `off` skips sandbox probing entirely — for skills you fully trust. No value enables native Windows: local mode fails closed there before anything runs — use WSL2 or `--env-mode docker`. | | `SKILLEVALUATOR_LOCAL_ALLOW_NET` | `true` | Network egress for local-mode trials. Set to `0` to airgap a skill that must not reach the network. Incompatible with the `nv_build` provider — NVIDIA Build local agents require network access, so airgapped runs are rejected up front. | | `SKILLEVALUATOR_LOCAL_STRICT_READS` | `false` | Tightens the sandbox's read-only view of the host system to a stricter path set. | diff --git a/docs/reports.mdx b/docs/reports.mdx index c1d3993a..36d77935 100644 --- a/docs/reports.mdx +++ b/docs/reports.mdx @@ -24,7 +24,26 @@ The flag is repeatable and accepts comma- or space-separated values the terminal shows a compact summary and the run writes `html` and `json` report files. An explicit `-r` is honored exactly, including `-r cli`, which prints the full terminal report and writes no files. Files land in `reports/` -unless you override it with `-o`. +unless you override it with `-o`. Report output must be outside the source tree +whose publication identity it describes. When the default `reports/` would be +inside the target (for example, after `cd my-skill`), SkillEvaluator safely +reserves the sibling directory `my-skill-reports/` instead. A catalog uses the +same rule and relocates to `-reports/`. An explicit in-target `-o` is +rejected; choose an external directory instead. Existing nonempty sibling +directories are reused only when their SkillEvaluator ownership marker is +valid, preventing the default from overwriting an unrelated directory. +For Tier 3, an output directory inside the target repository or an included +skill root must likewise be new, empty, or already carry a valid ownership +marker. This applies to linked-context staging as well as `--copy-repo`. A +nonempty unowned directory is rejected so selecting an authored subtree cannot +silently remove that subtree from the agent's runtime context. + +For compatibility, commands launched from a parent directory keep the familiar +layout when the default output is safe to claim: `skillevaluator validate +./my-skill` writes to `./reports/` when that directory is absent, empty, or +already authenticated as SkillEvaluator output. If `./reports/` is nonempty +and unowned, the default relocates to `./my-skill-reports/` instead of writing +into unrelated content. For `validate`, that compact default is a pipeline view — a per-check ticker plus a summary. Pass `--verbose` to get the full per-check detail stream in @@ -77,16 +96,87 @@ dataset provenance, agent/model details, and a dimension-by-dimension profile names and absolute host paths. The card carries one overall verdict — `PASS`, `NEUTRAL`, `FAIL`, or -`INCOMPLETE`. It -fails closed: if a required scanner produced no trustworthy evidence, the -verdict is `INCOMPLETE` and the card explicitly says not to use it to recommend -publication. A Publication Recommendation section appears only on a clean -`PASS`; `NEUTRAL` means the evidence is complete but at least one required -dimension remains below the pass band. Tier 3 evidence is publication-required -by default, so a missing, skipped, or incomplete live evaluation produces -`INCOMPLETE` rather than a publication recommendation. A catalog may -explicitly persist `benchmark_policy.tier3_required = false`; the card -discloses that exception instead of implying that Tier 3 ran. +`INCOMPLETE`. It fails closed: if a required scanner produced no trustworthy +evidence, the verdict is `INCOMPLETE` and the card explicitly says not to use +it to recommend publication. A Publication Recommendation section appears only +on a clean `PASS`; `NEUTRAL` means the evidence is complete but at least one +required dimension remains below the pass band. + +Tier 1 is always required for publication. Tier 2 and Tier 3 are also required +by default, so a missing, skipped, or incomplete required tier produces +`INCOMPLETE` rather than a publication recommendation. Catalog orchestration +may explicitly persist `benchmark_policy.tier2_required = false` or +`benchmark_policy.tier3_required = false`; the card discloses each exception +instead of implying that the tier ran. An optional tier that is present but +incomplete still makes the card `INCOMPLETE` because malformed evidence cannot +certify publication. A default-passed result object is not sufficient. Tier 1 +and Tier 2 evidence must carry a valid `publication_evidence` marker written by +a built-in command wrapper, and every recognized result must record a success, +finding, or positive check count that proves it executed. Legacy and custom +results remain visible, and their failures still block publication, but they +cannot satisfy a required tier or create a waiver. Generic per-result `optional` +metadata is not a publication waiver; only the resolved `benchmark_policy` from +recognized evidence can make Tier 2 or Tier 3 optional. An unrecognized `true` +claim can conservatively require a tier, but an unrecognized `false` claim +cannot waive one. + +Agent-evaluation payload metadata and its summary are duplicated claims at the +same precedence level and must agree. A conflict between valid booleans at that +level resolves to required evidence. Invalid entries do not count as policy +values; when the level contains no valid boolean, resolution falls through to +the next lower-precedence source, and no valid value anywhere means required. +The consistent payload/summary claim takes precedence over recognized peer result +metadata. Conflicting booleans among peer results likewise resolve to required +evidence so aggregation order cannot create a publication waiver. +Every Tier 1, Tier 2, and Tier 3 result used for publication carries a +`publication_target` with the exact NFC filesystem-entry name, a SHA-256 source digest, +and the versioned `skill-evaluator-source-tree/2` recipe. The recipe binds +author-owned relative paths, node kinds, executable bits, and file contents +while excluding only known generated artifacts. In particular, authored +`evals/` inputs are included. The v2 exclusions are any-depth `.git`, `.venv`, +`__pycache__`, and `node_modules`; root `.evals`, `.results`, `.versions`, +`results`, and `versions`; `evals/results`; and the generated root files +`BENCHMARK.md`, `skill-card.md`, and `skill.oms.sig`. Filesystem case aliases +are treated according to the host filesystem, while a case-distinct authored +name on a case-sensitive filesystem remains covered. A same-named file below +another authored directory is still included. Tier 1, Tier 2, Tier 3 runtime +projection, and full `--copy-repo` staging omit the same generated inputs so +excluded generated bytes inside the target cannot affect evaluation evidence. +Split-job results must match this identity exactly. Tier 3 runs that stage any +linked/full repository context or external reference/workspace skills remain +runnable but are marked `INCOMPLETE`, because that projection is not fully +determined by this target-only publication identity. Legacy anonymous results +remain readable, but their publication status is `INCOMPLETE`. + +Built-in Tier 1 and Tier 2 results also carry a versioned +`publication_evidence` object with exact `schema_version`, `producer`, `tier`, +and canonical `check_id` fields. Reporters accept only the documented built-in +producer/check combinations. This marker is provenance inside trusted local or +CI artifacts; it is not a cryptographic signature for accepting result files +from an untrusted party. + +Publication model, evaluator, environment, and skill identities reject reserved +placeholder text after Unicode confusable-skeleton comparison. This prevents a +visually spoofed placeholder such as one containing a Cyrillic or Greek letter +from becoming publication evidence while preserving ordinary multilingual +identities. + +Payload and summary policy claims are accepted only when their containing +evidence matches that exact publication target; evidence from another source +cannot waive a required tier. A skipped Tier 3 payload must satisfy the same +target binding before its policy can waive evidence. A completed Tier 3 payload +also duplicates its run-owned `run_id` in the payload and summary; missing or +contradictory run identity is incomplete publication evidence. +When multiple complete Tier 3 payloads disagree, reporters select the most +conservative effective verdict (`FAIL`, then `NEUTRAL`, then `PASS`) before +applying deterministic tie-breakers. + +Execution controls do not rewrite that publication policy. `--no-dedup` and +`--tiers 1,3` select which validators run, while `--no-block-on-dedup` changes +whether Tier 2 affects the command exit code. None of those flags persists a +publication waiver. Without explicit policy metadata, a run that omits Tier 2 +therefore exits according to its CLI gate but still produces an `INCOMPLETE` +publication card. Keep the file with the skill and refresh it whenever the skill, eval dataset, agent/model, evaluator version, scoring policy, attempt policy, or execution @@ -113,7 +203,10 @@ of: With an external root (the first two options), each skill gets its own subdirectory — runs land under `//`, not directly under -``. +``. Configured roots must be outside the skill. The canonical +`/evals/results/` path is the only supported in-skill result root; this +keeps generated runs outside the versioned source-tree digest used to combine +publication evidence. Each run creates a collision-safe directory named by its run ID (`YYYYMMDD_HHMMSS__<12-hex-nonce>`). Where symlinks are supported, a @@ -317,9 +410,17 @@ For CI and tooling, two JSON entry points matter: - **`validate --tier3` runs** — the combined `skillevaluator-output-.json` report embeds the Tier 3 payload alongside the Tier 1 and Tier 2 results, so one file covers all three - tiers. Each result carries finalized gating metadata. Tier 3 is advisory by - default and becomes blocking with `--block-on-agent-eval`; Tier 2 is blocking - by default and becomes advisory with `--no-block-on-dedup`. + tiers. Each built-in Tier 1 or Tier 2 result carries its validated + `publication_evidence` producer marker alongside finalized gating metadata, + and the top-level + `benchmark_policy` records whether Tier 2 and Tier 3 evidence is required for + publication. `publication_status` is the compact machine-readable verdict; + `publication` carries eligibility, reasons, and the Tier 3 evidence status. + This is intentionally separate from `overall_status`, which continues to + describe the command/process gate. Tier 3 is advisory for the CLI exit code by default and becomes + blocking with `--block-on-agent-eval`; Tier 2 is blocking for the exit code + by default and becomes advisory with `--no-block-on-dedup`. Those exit-code + controls are separate from `benchmark_policy`. Both paths speak the same dialect: standalone `tier3 evaluate` and `validate --tier3` share one HTML renderer for `report.html`, and both embed diff --git a/docs/superpowers/specs/2026-08-24-required-tier-publication-evidence-design.md b/docs/superpowers/specs/2026-08-24-required-tier-publication-evidence-design.md new file mode 100644 index 00000000..0654773a --- /dev/null +++ b/docs/superpowers/specs/2026-08-24-required-tier-publication-evidence-design.md @@ -0,0 +1,139 @@ +# Required-Tier Publication Evidence Design + +## Problem + +`BenchmarkReporter` can recommend publication when its Tier Status table says +that Tier 1 or Tier 2 was not run. The overall verdict checks the results it was +given and has a special completeness rule for Tier 3, but it does not model the +absence of other publication-required tiers. The public benchmark linter repeats +the gap by validating only Tier 3 consistency. + +## Publication contract + +- Tier 1 is always required. A card without non-skipped Tier 1 evidence is + `INCOMPLETE`. +- Tier 2 is required by default. Only an explicitly persisted + `benchmark_policy.tier2_required = false` makes absent or cleanly skipped Tier + 2 evidence optional. +- Tier 3 keeps its current default-required policy and + `benchmark_policy.tier3_required = false` escape hatch. +- Run-selection flags such as `--no-dedup` and `--tiers 1,3` are not publication + waivers. Split-tier CI uses the same selectors, so inferring a waiver from + them would preserve the original bug. +- `--no-block-on-dedup` changes the process exit gate only. It does not make + Tier 2 optional for publication. +- Invalid policy values cannot waive required evidence. Each policy key is + resolved independently by source precedence: the duplicated agent-evaluation + payload and summary claims first, then peer result metadata. The duplicated + claims must agree when both are valid booleans; a conflict resolves to + required evidence. Invalid entries do not count as values; a level with no + valid boolean falls through to the next lower-precedence source, and no valid + value anywhere means required. Peer results share one precedence level, so + conflicting booleans resolve to required evidence independent of aggregation + order. +- Agent-evaluation policy claims are target-bound. A payload or summary for a + different skill cannot waive evidence for the current report, including when + that foreign Tier 3 result is a clean advisory skip. +- Every publication-contributing result is bound to one exact source snapshot + through a versioned `publication_target`. The digest covers normalized + author-owned paths, node kinds, executable bits, and file contents, including + authored `evals/` inputs. The versioned recipe omits only its enumerated + generated paths: any-depth `.git`, `.venv`, `__pycache__`, and `node_modules`; + root generated state/result/version directories; `evals/results`; and the + three generated root publication files. Filesystem aliases follow actual + filesystem identity, and a same-named nested authored file remains covered. + Producer scans and Tier 3 agent-visible projections share this exclusion + contract. A Tier 3 projection that actually adds linked/full repository + context cannot certify publication until runtime context has its own + versioned identity; full-mode selection can depend on excluded Git/operator + state even when the repository root is the target. External reference or + workspace skills have the same restriction. The run remains usable but + carries a source-identity conflict and is `INCOMPLETE`. A missing, malformed, + or different digest makes + publication `INCOMPLETE` even when the visible target names match. +- Completed Tier 3 evidence carries its run-owned `run_id` in both the payload + and summary. The runner persists the target identity at the execution + boundary; reporters never backfill it from the current live path. +- Generic per-result `optional` metadata cannot waive a publication-required + validator. Only the resolved `benchmark_policy` makes Tier 2 or Tier 3 + optional; Tier 1 is always required. +- Optional evidence that is present but malformed or incomplete cannot certify + publication. + +## Components and data flow + +1. A shared publication assessment resolves `tier2_required` and + `tier3_required`, validates Tier 1/Tier 2 execution evidence, classifies Tier + 3 completeness, and keeps publication status separate from process status. + `BenchmarkReporter` renders that assessment and both requirements in + Evaluation Metadata. +2. Tier classification uses one shared validator-name classifier so the + benchmark and HTML reporters agree that names such as `Similarity Check` and + `Context Optimization` belong to Tier 2. +3. `JSONReporter` preserves the resolved publication policy plus explicit + `publication_status` and structured `publication` details so an external + aggregator does not mistake a successful process gate for publication + completeness. Per-result source identities remain available for split-job + aggregation. +4. `check_public_benchmarks.py` parses all three tier rows. For PASS cards it + requires completed Tier 1, completed Tier 2 unless explicitly optional, and + completed Tier 3 unless explicitly optional. Missing Tier 2 policy metadata + defaults to required so older contradictory cards fail closed. +5. Tier 1 and Tier 2 producers compare source snapshots before and after their + checks. Tier 3 persists its source identity and run ID in `result.json` and + propagates them through payload, summary, JSON, HTML, and BENCHMARK metadata. + A source-change conflict propagates as a bounded generic marker instead of + leaking raw path/digest details or disappearing during rerender. +6. Documentation and the changelog explain that execution selection, CLI exit + gating, and publication completeness are separate contracts. + +The downstream CI aggregator that produced the public cards is outside this +repository. This change gives that aggregator a fail-closed source/run binding; +deployment-specific artifact transport remains a separate integration proof. + +## Alternatives considered + +### Always require Tier 2 with no override + +This is simple but removes the existing pattern of persisted publication-policy +exceptions and gives catalog owners no explicit migration path. + +### Infer Tier 2 optionality from CLI selection + +This preserves current partial-run PASS cards, but it also treats a split-tier +job as publication-complete. That is the failure described by issue #73. + +### Explicit persisted required-tier policy + +This is the selected design. It separates what ran, what affects the process +exit code, and what evidence publication requires. + +## Error handling and compatibility + +- Existing programmatic Tier 1 + Tier 3 calls change from PASS to INCOMPLETE + unless they explicitly persist `tier2_required = false`. +- Existing cards without a Tier 2 policy line are interpreted as requiring Tier + 2. Good cards with completed Tier 2 remain valid; contradictory PASS cards do + not. +- Existing `tier3_required` policy behavior remains unchanged. +- Legacy results without source identity and completed Tier 3 results without a + run ID remain readable but cannot certify publication. +- A first-class CLI or YAML publication-policy option is intentionally out of + scope. Metadata injection remains the orchestration escape hatch. + +## Verification + +- Prove the reporter regression red before production changes and green after. +- Cover missing Tier 1, required and optional Tier 2, skipped/incomplete Tier 2, + invalid policy values, and Tier 2 validator-name classification. +- Cover the publication linter against required, optional, spoofed, and legacy + cards. +- Exercise `validate` with `--no-dedup`, `--tiers 1,3`, and + `--no-block-on-dedup` to show that CLI execution and exit semantics are + preserved while the publication card fails closed. +- Verify generated BENCHMARK, JSON, and HTML outputs from the same result set. +- Merge split-tier results from one unchanged source as a positive control, then + change an author-owned file between jobs and verify publication becomes + `INCOMPLETE`. +- Run the focused suites, committed-card gate, Ruff, package build, and the full + default test suite. diff --git a/docs/tier3-live-evaluation.mdx b/docs/tier3-live-evaluation.mdx index 924ab2d5..549b08ba 100644 --- a/docs/tier3-live-evaluation.mdx +++ b/docs/tier3-live-evaluation.mdx @@ -110,11 +110,17 @@ native-task identities. The snapshot is removed when the run returns and requires temporary space only for the selected compatibility scope. Tier 3 does not install evaluator-owned directories from the target or any -nested, reference, or workspace skill into agent-visible copies. `--copy-repo` -uses the same filtering and excludes generated result roots. Authenticated -historical result roots remain excluded after the configured output location is -changed; a present marker that is copied, stale, or cannot be authenticated -fails staging closed instead of copying that tree. Custom result roots created +nested, reference, or workspace skill into agent-visible copies. Runtime skill +projection and `--copy-repo` also omit every generated path excluded by the +versioned publication-source digest. The active report directory and the whole +catalog report root are explicitly excluded from full repo context, preventing +a repeated run from exposing prior reports or Tier 3 evidence to the agent. An +in-repository report directory is excluded only after SkillEvaluator reserves +or authenticates it as generated output; a nonempty unowned directory is +rejected instead of silently removing authored files from `--copy-repo`. +Authenticated historical result roots remain excluded after the configured +output location is changed; a present marker that is copied, stale, or cannot +be authenticated fails staging closed instead of copying that tree. Custom result roots created outside `evals/` by a pre-marker version cannot be identified safely after the configured location changes. Move or delete that old content before `--copy-repo` or another full-context evaluation, then rerun with the current @@ -127,6 +133,16 @@ dataset/configuration files, graders, developer eval guidance, benchmark conversion reports, native Harbor tasks, custom environments, evaluator tests, or generated results. +Publication identity covers the evaluated skill tree itself. A Tier 3 run that +actually stages linked or full-repository context, or separately configured +external reference/workspace skills, still executes and remains available for +diagnosis, but its publication evidence fails closed as `INCOMPLETE`. +Full-repository selection also depends on Git/operator state excluded from the +skill digest, even when the repository root is the skill directory. Keep +publication-certifying runtime inputs inside the evaluated skill tree without +repository-context staging. Binding arbitrary repository projections under a +separate, versioned identity is not currently supported. + MCP declarations are read only from a bounded, stable, single-link regular `evals/environment/mcp_servers.toml` through a descriptor-anchored evaluator root. Linked configuration files or parents, hardlinks, reparse points, and @@ -392,7 +408,7 @@ The high-signal flags, with defaults: | `--progress` | `auto` | Progress rendering: `auto`, `rich` (live TTY view with secret redaction), `plain`, or `off` | | `--autopilot` | off | When no evaluation source exists, generate exactly one eval case — with the configured provider, falling back to a deterministic keyless template — never overwriting an existing source | | `--grading-mode` | `default` | `default`, `default_plus_custom`, or `custom_only` — see [Custom Graders & Tasks](custom-graders.mdx) | -| `--results-dir` | `evals/results`, or `SKILLEVALUATOR_RESULTS_DIR` when set | Write results under an external root instead of the skill directory | +| `--results-dir` | `evals/results`, or `SKILLEVALUATOR_RESULTS_DIR` when set | Write results under an external root instead of the skill directory; custom in-skill roots are rejected | | `--harbor-keep-jobs` | off | Retain Harbor job directories for inspection | | `--timeout-multiplier` | `1.0` | Scale Harbor step timeouts | | `--copy-repo` | off | Copy the surrounding repository into the task environment | diff --git a/scripts/ci/check_public_benchmarks.py b/scripts/ci/check_public_benchmarks.py index eda94073..642a6592 100644 --- a/scripts/ci/check_public_benchmarks.py +++ b/scripts/ci/check_public_benchmarks.py @@ -14,24 +14,42 @@ import argparse import re import sys +import unicodedata from dataclasses import dataclass +from datetime import UTC, date, datetime, timedelta +from functools import lru_cache +from html import unescape +from html.parser import HTMLParser from pathlib import Path +from typing import TYPE_CHECKING +from urllib.parse import unquote as url_unquote + +from markdown_it import MarkdownIt + +from skillevaluator.publication_text import publication_confusable_skeleton, publication_identity_present + +if TYPE_CHECKING: + from markdown_it.token import Token REQUIRED_MARKERS = ( - "# Skill Benchmark:", - "Overall verdict:", - "## Evaluation Metadata", "- Evaluation date:", "- Evaluator version:", "- Agents:", "- Tasks:", + "- Source digest:", "- Dataset digest:", + "- Tier 3 run ID:", "- Attempts per task:", "- Environment:", "- Tier 3 evidence:", - "## Results at a Glance", - "## Tier Status", - "## Freshness", +) + +_REQUIRED_HEADINGS = ( + (1, "Skill Benchmark:", True, "# Skill Benchmark:"), + (2, "Evaluation Metadata", False, "## Evaluation Metadata"), + (2, "Results at a Glance", False, "## Results at a Glance"), + (2, "Tier Status", False, "## Tier Status"), + (2, "Freshness", False, "## Freshness"), ) LINE_RULES = ( @@ -42,7 +60,7 @@ ( "internal environment identity", re.compile( - r"(?:^\s*-\s*Environment:\s*`?astra`?\s*$|\bastra[\s_-]+sandbox\b)", + r"(?:^\s*-\s*Environment:\s*`?astra(?:[^A-Za-z0-9]|$)|\bastra[\s_-]+sandbox\b)", flags=re.IGNORECASE, ), ), @@ -76,15 +94,37 @@ ) _AGENT_MODEL_STATE = re.compile( - r"^[^,]+ \((?:`[^`,]+`|model not recorded)\)$", + r"^(?P[^,]+) \((?P`[^`]+`|model not recorded)\)$", + flags=re.IGNORECASE, +) +_OVERALL_VERDICT_FIELD = re.compile( + r"^\s*(?:(?:✅|❌|⚠\ufe0f?)\s*)?Overall verdict:\s*(?P.*)$", flags=re.IGNORECASE, ) -_RECORDED_AGENT_MODEL_STATE = re.compile(r"^[^,]+ \(`[^`,]+`\)$") -_OVERALL_PASS = re.compile( - r"^\s*>\s*.*Overall verdict:\s*PASS\b", - flags=re.IGNORECASE | re.MULTILINE, +_OVERALL_VERDICT_SKELETON_FIELD = re.compile( + r"^\s*(?:(?:✅|❌|⚠\ufe0f?)\s*)?overall verdict:\s*(?P.*)$", +) +_INVISIBLE_IDENTITY_CHARACTERS = frozenset( + { + "\u115f", + "\u1160", + "\u2800", + "\u3164", + "\uffa0", + "\U00013441", + "\U00013442", + "\U0001d159", + } ) +_SECURITY_CONFUSABLES = {"\u0406": "I", "\u0456": "i"} _METADATA_FIELD_RULES = ( + ( + "Source digest", + re.compile( + r"(?:`sha256:[0-9a-f]{64}`\s+\(skill-evaluator-source-tree/2\)|not recorded\b.*)", + flags=re.IGNORECASE, + ), + ), ( "Evaluation date", re.compile(r"(?:\d{4}-\d{2}-\d{2}|not recorded\b.*)", flags=re.IGNORECASE), @@ -101,6 +141,10 @@ "Dataset digest", re.compile(r"(?:`[^`\s][^`]*`(?:\s+\([^)]*\))?|not recorded\b.*)", flags=re.IGNORECASE), ), + ( + "Tier 3 run ID", + re.compile(r"(?:`[A-Za-z0-9][A-Za-z0-9._-]{0,159}`|not recorded\b.*)"), + ), ( "Attempts per task", re.compile(r"(?:[1-9]\d*|not recorded\b.*)", flags=re.IGNORECASE), @@ -111,7 +155,18 @@ ), ( "Tier 3 evidence", - re.compile(r"(?:required for publication|optional by policy)", flags=re.IGNORECASE), + re.compile(r"(?:required for publication|optional by policy)"), + ), +) +_METADATA_FIELD_NAMES = ( + *(field for field, _pattern in _METADATA_FIELD_RULES), + "Tier 2 evidence", + "Agents", +) +_PASS_SOURCE_METADATA_FIELD_RULES = ( + ( + "Source digest", + re.compile(r"`sha256:[0-9a-f]{64}`\s+\(skill-evaluator-source-tree/2\)"), ), ) _PASS_METADATA_FIELD_RULES = ( @@ -125,9 +180,381 @@ flags=re.IGNORECASE, ), ), + ("Tier 3 run ID", re.compile(r"`[A-Za-z0-9][A-Za-z0-9._-]{0,159}`")), ("Attempts per task", re.compile(r"[1-9]\d*")), ("Environment", re.compile(r"`[^`\s][^`]*`")), ) +_TIER_COMPLETION_STATUSES = { + 1: frozenset({"PASSED", "PASSED WITH OBSERVATIONS"}), + 2: frozenset({"PASSED", "PASSED WITH OBSERVATIONS"}), + 3: frozenset({"PASS"}), +} +_OPTIONAL_TIER_ABSENCE_STATUSES = frozenset({"NOT RUN", "SKIPPED (ADVISORY)"}) +_TIER_POLICY_VALUES = frozenset({"required for publication", "optional by policy"}) +_OVERALL_VERDICT_STATUSES = frozenset({"PASS", "FAIL", "NEUTRAL", "INCOMPLETE"}) +_KNOWN_TIER_STATUSES = { + 1: frozenset({"PASSED", "PASSED WITH OBSERVATIONS", "FAILED", "INCOMPLETE", "NOT RUN", "SKIPPED (ADVISORY)"}), + 2: frozenset({"PASSED", "PASSED WITH OBSERVATIONS", "FAILED", "INCOMPLETE", "NOT RUN", "SKIPPED (ADVISORY)"}), + 3: frozenset({"PASS", "FAIL", "FAILED", "NEUTRAL", "INCOMPLETE", "NOT RUN", "SKIPPED (ADVISORY)"}), +} +_MAX_BENCHMARK_BYTES = 128 * 1024 +_MAX_BENCHMARK_LINE_CHARACTERS = 32 * 1024 +_MARKDOWN = MarkdownIt("commonmark").enable("table") +_NON_RENDERED_HTML_TAGS = frozenset({"noscript", "script", "style", "template"}) +_STRUCTURAL_HTML_HEADING_TAGS = frozenset({"h1", "h2"}) +_VOID_HTML_TAGS = frozenset( + {"area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "param", "source", "track", "wbr"} +) +_HTML_TEXT_SEPARATOR_TAGS = frozenset( + { + "address", + "article", + "aside", + "blockquote", + "dd", + "details", + "div", + "dl", + "dt", + "fieldset", + "figcaption", + "figure", + "footer", + "form", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "header", + "hgroup", + "hr", + "li", + "main", + "nav", + "ol", + "p", + "pre", + "section", + "summary", + "table", + "tbody", + "td", + "tfoot", + "th", + "thead", + "tr", + "ul", + } +) +_AMBIGUOUS_HTML_RENDERING_TAGS = _NON_RENDERED_HTML_TAGS | frozenset( + { + "audio", + "base", + "bdi", + "bdo", + "button", + "canvas", + "dialog", + "embed", + "form", + "iframe", + "input", + "link", + "math", + "meta", + "meter", + "noembed", + "noframes", + "object", + "option", + "optgroup", + "output", + "picture", + "plaintext", + "progress", + "select", + "source", + "svg", + "table", + "textarea", + "video", + "xmp", + } +) +_AMBIGUOUS_HTML_RENDERING_ATTRIBUTES = frozenset( + { + "align", + "bgcolor", + "class", + "color", + "dir", + "face", + "height", + "hidden", + "id", + "inert", + "size", + "style", + "width", + } +) + + +def _html_element_is_visually_hidden(tag: str, attrs: list[tuple[str, str | None]]) -> bool: + """Return whether an HTML element is removed from visual rendering.""" + if tag.casefold() in _NON_RENDERED_HTML_TAGS: + return True + normalized_attrs = {name.casefold(): value for name, value in attrs} + if "hidden" in normalized_attrs: + return True + style = normalized_attrs.get("style") + if style is None: + return False + uncommented_style = re.sub(r"/\*.*?\*/", "", style, flags=re.DOTALL) + declarations: dict[str, tuple[str, bool]] = {} + for declaration in uncommented_style.split(";"): + property_name, separator, value = declaration.partition(":") + if not separator: + continue + normalized_property = property_name.strip().casefold() + if normalized_property not in {"display", "visibility"}: + continue + important = re.search(r"!\s*important\s*$", value, flags=re.IGNORECASE) is not None + normalized_value = re.sub(r"!\s*important\s*$", "", value, flags=re.IGNORECASE).strip().casefold() + previous = declarations.get(normalized_property) + if previous is not None and previous[1] and not important: + continue + declarations[normalized_property] = (normalized_value, important) + display = declarations.get("display", ("", False))[0] + visibility = declarations.get("visibility", ("", False))[0] + return display == "none" or visibility in {"hidden", "collapse"} + + +def _close_html_element_stack(stack: list[tuple[str, bool]], tag: str) -> int | None: + """Pop one exactly nested element; ambiguous markup stays fail-closed.""" + if not stack or stack[-1][0] != tag: + return None + _name, is_hidden = stack.pop() + return int(is_hidden) + + +class _VisibleHTMLParser(HTMLParser): + """Collect rendered text while suppressing comments and control elements.""" + + def __init__(self, *, suppress_visually_hidden: bool = True) -> None: + super().__init__(convert_charrefs=True) + self._suppress_visually_hidden = suppress_visually_hidden + self._element_stack: list[tuple[str, bool]] = [] + self._hidden_depth = 0 + self._line_parts: dict[int, list[str]] = {} + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + normalized_tag = tag.casefold() + is_hidden = normalized_tag in _NON_RENDERED_HTML_TAGS or ( + self._suppress_visually_hidden and _html_element_is_visually_hidden(normalized_tag, attrs) + ) + if normalized_tag not in _VOID_HTML_TAGS: + self._element_stack.append((normalized_tag, is_hidden)) + if is_hidden: + if normalized_tag not in _VOID_HTML_TAGS: + self._hidden_depth += 1 + return + if self._hidden_depth: + return + if normalized_tag == "br" or normalized_tag in _HTML_TEXT_SEPARATOR_TAGS: + self._append_visible(" ") + elif normalized_tag == "img": + alt = next((value for name, value in attrs if name.casefold() == "alt"), None) + if alt is not None: + self._append_visible(f" {alt} ") + + def handle_startendtag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + # HTML slash syntax does not self-close non-void elements in browsers. + self.handle_starttag(tag, attrs) + + def handle_endtag(self, tag: str) -> None: + normalized_tag = tag.casefold() + if normalized_tag == "br": + if not self._hidden_depth: + # HTML5 treats the otherwise-invalid
token as
. + self._append_visible(" ") + return + hidden_before = self._hidden_depth + hidden_count = _close_html_element_stack(self._element_stack, normalized_tag) + if hidden_count is None: + return + self._hidden_depth = max(0, self._hidden_depth - hidden_count) + if not hidden_before and normalized_tag in _HTML_TEXT_SEPARATOR_TAGS: + self._append_visible(" ") + + def handle_data(self, data: str) -> None: + if self._hidden_depth: + return + self._append_visible(data) + + def _append_visible(self, data: str) -> None: + start_line, _column = self.getpos() + for offset, part in enumerate(data.split("\n")): + self._line_parts.setdefault(start_line + offset, []).append(part) + + def visible_lines(self) -> tuple[tuple[int, str], ...]: + return tuple( + (line, normalized) + for line, parts in sorted(self._line_parts.items()) + if (normalized := " ".join("".join(parts).split())) + ) + + +class _HTMLAttributeParser(HTMLParser): + """Collect decoded HTML attribute values for leak-pattern scanning.""" + + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.values: list[str] = [] + self.image_alts: list[str] = [] + self.events: list[tuple[str, str, bool, str | None, int]] = [] + self.ambiguous_rendering_lines: list[int] = [] + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + normalized_tag = tag.casefold() + self.values.extend(value for _name, value in attrs if value is not None) + alt = next((value for name, value in attrs if name.casefold() == "alt"), None) + line = self.getpos()[0] + self.events.append(("start", normalized_tag, _html_element_is_visually_hidden(normalized_tag, attrs), alt, line)) + attribute_names = {name.casefold() for name, _value in attrs} + if ( + normalized_tag in _AMBIGUOUS_HTML_RENDERING_TAGS + or attribute_names & _AMBIGUOUS_HTML_RENDERING_ATTRIBUTES + or any(name.startswith("on") for name in attribute_names) + ): + self.ambiguous_rendering_lines.append(line) + if normalized_tag == "img": + self.image_alts.extend(value for name, value in attrs if name.casefold() == "alt" and value is not None) + + def handle_startendtag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + self.handle_starttag(tag, attrs) + + def handle_endtag(self, tag: str) -> None: + self.events.append(("end", tag.casefold(), False, None, self.getpos()[0])) + + +class _RawStructuralHeadingParser(HTMLParser): + """Locate raw HTML headings that cannot be trusted as card structure.""" + + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.lines: list[int] = [] + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + del attrs + if tag.casefold() in _STRUCTURAL_HTML_HEADING_TAGS: + self.lines.append(self.getpos()[0]) + + def handle_startendtag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + # A slash does not self-close h1/h2 in text/html, but the start still + # creates ambiguous browser-visible structure and must be rejected. + self.handle_starttag(tag, attrs) + + +class _HTMLStructureParser(HTMLParser): + """Collect real tag events/headings without regex-parsing quoted markup.""" + + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.events: list[tuple[str, str]] = [] + self.headings: list[tuple[int, int, str]] = [] + self._heading_level: int | None = None + self._heading_line = 1 + self._heading_parts: list[str] = [] + self._element_stack: list[tuple[str, bool]] = [] + self._hidden_depth = 0 + self._in_noscript = False + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + normalized_tag = tag.casefold() + if self._in_noscript: + return + self.events.append(("start", normalized_tag)) + is_hidden = _html_element_is_visually_hidden(normalized_tag, attrs) + if normalized_tag not in _VOID_HTML_TAGS: + self._element_stack.append((normalized_tag, is_hidden)) + if normalized_tag == "noscript": + self._in_noscript = True + if is_hidden: + if normalized_tag not in _VOID_HTML_TAGS: + self._hidden_depth += 1 + return + if self._hidden_depth: + return + if normalized_tag in _STRUCTURAL_HTML_HEADING_TAGS and self._heading_level is None: + self._heading_level = int(normalized_tag[1]) + self._heading_line = self.getpos()[0] + self._heading_parts = [] + elif self._heading_level is not None and ( + normalized_tag == "br" or normalized_tag in _HTML_TEXT_SEPARATOR_TAGS + ): + self._heading_parts.append(" ") + elif self._heading_level is not None and normalized_tag == "img": + alt = next((value for name, value in attrs if name.casefold() == "alt"), None) + if alt is not None: + self._heading_parts.extend((" ", alt, " ")) + + def handle_startendtag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + # HTML slash syntax does not self-close non-void elements in browsers. + self.handle_starttag(tag, attrs) + + def handle_endtag(self, tag: str) -> None: + normalized_tag = tag.casefold() + if normalized_tag == "br": + self.events.append(("end", normalized_tag)) + if not self._in_noscript and not self._hidden_depth and self._heading_level is not None: + self._heading_parts.append(" ") + return + if self._in_noscript: + if normalized_tag != "noscript": + return + self._in_noscript = False + self.events.append(("end", normalized_tag)) + hidden_before = self._hidden_depth + hidden_count = _close_html_element_stack(self._element_stack, normalized_tag) + if hidden_count: + self._hidden_depth = max(0, self._hidden_depth - hidden_count) + if self._hidden_depth: + return + if ( + not hidden_before + and self._heading_level is not None + and normalized_tag in _HTML_TEXT_SEPARATOR_TAGS + and normalized_tag != f"h{self._heading_level}" + ): + self._heading_parts.append(" ") + if self._heading_level is not None and normalized_tag == f"h{self._heading_level}": + title = " ".join("".join(self._heading_parts).split()) + self.headings.append((self._heading_level, self._heading_line, title)) + self._heading_level = None + self._heading_parts = [] + + def handle_data(self, data: str) -> None: + if not self._in_noscript and not self._hidden_depth and self._heading_level is not None: + self._heading_parts.append(data) + + +@lru_cache(maxsize=256) +def _html_structure( + content: str, +) -> tuple[ + tuple[tuple[str, str], ...], + tuple[tuple[int, int, str], ...], +]: + parser = _HTMLStructureParser() + try: + parser.feed(content) + parser.close() + except (AssertionError, ValueError): + return (), () + return tuple(parser.events), tuple(parser.headings) @dataclass(frozen=True) @@ -156,53 +583,237 @@ def benchmark_files(roots: list[Path]) -> list[Path]: def scan_file(path: Path) -> list[Offender]: try: + if path.stat().st_size > _MAX_BENCHMARK_BYTES: + return [Offender(path, 1, f"BENCHMARK.md exceeds {_MAX_BENCHMARK_BYTES:,} bytes")] text = path.read_text(encoding="utf-8") except (OSError, UnicodeError) as error: return [Offender(path, 1, f"unreadable file ({type(error).__name__})")] offenders: list[Offender] = [] + for line_number, line in enumerate(text.splitlines(keepends=True), 1): + # Character references can render the same bidirectional and format + # controls as literal Unicode. Reject both representations before any + # semantic normalization removes them. + encoded_surface = unescape(line.replace("\n", "").replace("\r", "").replace("\t", "")) + if _contains_disallowed_category_c(line, allow_layout_whitespace=True) or _contains_disallowed_category_c( + encoded_surface, + allow_layout_whitespace=False, + ): + offenders.append(Offender(path, line_number, "benchmark contains disallowed control character")) + break + + for line_number, line in enumerate(text.splitlines(), 1): + if len(line) > _MAX_BENCHMARK_LINE_CHARACTERS: + return [ + Offender( + path, + line_number, + f"benchmark line exceeds {_MAX_BENCHMARK_LINE_CHARACTERS:,} characters", + ) + ] for marker in REQUIRED_MARKERS: if marker not in text: offenders.append(Offender(path, 1, f"missing required section: {marker}")) + literal_code_lines = { + line_number + for token in _markdown_tokens(text) + if token.type in {"fence", "code_block"} and token.map is not None + for line_number in range(token.map[0] + 1, token.map[1] + 1) + } for line_number, line in enumerate(text.splitlines(), 1): + # Decode references in rendered text while preserving literal bytes in + # code spans, matching CommonMark rather than raw HTML unescaping. + rendered_surfaces = () if line_number in literal_code_lines else _rendered_inline_surfaces(line) + semantic_lines = tuple(_semantic_text(surface) for surface in (line, *rendered_surfaces)) for reason, pattern in LINE_RULES: - if pattern.search(line): + candidate_lines = semantic_lines + if reason in {"retired product identity", "internal environment identity"}: + candidate_lines = ( + *candidate_lines, + *(publication_confusable_skeleton(semantic_line) for semantic_line in semantic_lines), + ) + if reason == "internal environment identity": + for semantic_line in semantic_lines: + environment_field = re.match( + r"^\s*-\s*Environment:\s*(?P.*)$", + semantic_line, + flags=re.IGNORECASE, + ) + if environment_field is not None: + candidate_lines = ( + *candidate_lines, + "- Environment: " + + publication_confusable_skeleton(environment_field.group("value")), + ) + if any(pattern.search(candidate_line) for candidate_line in candidate_lines): offenders.append(Offender(path, line_number, reason)) + semantic_document = _semantic_text(text) + for reason, pattern in LINE_RULES: + if reason not in {"retired product identity", "internal environment identity"}: + continue + if ( + pattern.search(semantic_document) + or pattern.search(publication_confusable_skeleton(semantic_document)) + ) and not any(offender.reason == reason for offender in offenders): + offenders.append(Offender(path, 1, reason)) + + _check_required_headings(path, text, offenders) _check_metadata_semantics(path, text, offenders) _check_verdict_tier_consistency(path, text, offenders) + _check_untrusted_structural_headings(path, text, offenders) + _check_html_rendering_safety(path, text, offenders) return offenders +def _check_required_headings(path: Path, text: str, offenders: list[Offender]) -> None: + """Require canonical headings in rendered Markdown structure.""" + headings = _heading_entries(text) + for _index, level, line, title, _trusted in headings: + structural_title = title.partition(":")[0] + ":" if level == 1 and ":" in title else title + if level in {1, 2} and not structural_title.isascii(): + offenders.append(Offender(path, line, "non-ASCII structural heading")) + for required_level, required_title, is_prefix, marker in _REQUIRED_HEADINGS: + required_visual_key = _visual_text_key(required_title) + required_canonical_key = _canonical_text_key(required_title) + matching_headings = [ + (line, title, trusted) + for _index, level, line, title, trusted in headings + if level == required_level + and ( + _visual_text_key(title).startswith(required_visual_key) + if is_prefix + else _visual_text_key(title) == required_visual_key + ) + ] + trusted_headings = [ + heading + for heading in matching_headings + if heading[2] + and ( + _canonical_text_key(heading[1]).startswith(required_canonical_key) + if is_prefix + else _canonical_text_key(heading[1]) == required_canonical_key + ) + ] + trusted_prefix_identity = ( + trusted_headings[0][1][len(required_title) :] + if trusted_headings and trusted_headings[0][1].casefold().startswith(required_title.casefold()) + else "" + ) + if not trusted_headings or (is_prefix and not _identity_present(trusted_prefix_identity)): + offenders.append(Offender(path, 1, f"missing required section: {marker}")) + if len(matching_headings) > 1 and required_title not in {"Evaluation Metadata", "Tier Status"}: + offenders.append(Offender(path, matching_headings[1][0], f"duplicate required section: {marker}")) + + +def _check_untrusted_structural_headings(path: Path, text: str, offenders: list[Offender]) -> None: + """Require root h1/h2 structure to use unambiguous plain Markdown text.""" + duplicate_lines = {offender.line for offender in offenders if offender.reason.startswith("duplicate ")} + for line_number in _untrusted_structural_heading_lines(text): + if line_number not in duplicate_lines: + offenders.append(Offender(path, line_number, "ambiguous structural heading markup")) + + +def _check_html_rendering_safety(path: Path, text: str, offenders: list[Offender]) -> None: + """Reject raw HTML whose browser presentation cannot be parsed safely.""" + stack: list[tuple[str, int]] = [] + ambiguous_lines: set[int] = set() + + def inspect_fragment(content: str, base_line: int) -> None: + parser = _HTMLAttributeParser() + try: + parser.feed(content) + parser.close() + except (AssertionError, ValueError): + ambiguous_lines.add(base_line) + return + ambiguous_lines.update(base_line + relative_line - 1 for relative_line in parser.ambiguous_rendering_lines) + for event, tag, _visually_hidden, _alt, relative_line in parser.events: + line = base_line + relative_line - 1 + if event == "start": + if tag not in _VOID_HTML_TAGS: + stack.append((tag, line)) + continue + if tag == "br": + # HTML5 defines
as a line break rather than a close. + continue + if not stack or stack[-1][0] != tag: + ambiguous_lines.add(line) + continue + stack.pop() + + for token in _markdown_tokens(text): + if token.map is None: + continue + if token.type == "html_block": + inspect_fragment(token.content, token.map[0] + 1) + elif token.type == "inline": + for child in token.children or []: + if child.type == "html_inline": + inspect_fragment(child.content, token.map[0] + 1) + + ambiguous_lines.update(line for _tag, line in stack) + for line in sorted(ambiguous_lines): + offenders.append(Offender(path, line, "ambiguous HTML rendering semantics")) + + def _check_metadata_semantics(path: Path, text: str, offenders: list[Offender]) -> None: - metadata_lines = _metadata_section_lines(text) - if not metadata_lines: + metadata_heading_lines = _section_heading_lines(text, "Evaluation Metadata") + metadata_sections = _section_occurrences(text, "Evaluation Metadata") + if len(metadata_heading_lines) > 1: + offenders.append(Offender(path, metadata_heading_lines[1], "duplicate Evaluation Metadata section")) + if not metadata_sections: return + fallback_line, section_tokens = metadata_sections[0] + metadata_lines = _metadata_list_items(section_tokens) + + for line_number, source in metadata_lines: + rendered = _rendered_inline_fragment(source) + rendered_match = re.match(r"^\s*-\s*(?P