diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 46a6b3e8..7089750b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -283,12 +283,15 @@ jobs: tests/test_pf_core_phase4_differential.py \ tests/test_pf_core_hash_vector_parity.py \ tests/test_canonical_hash_release.py \ - tests/test_shared_hash_vectors.py + tests/test_shared_hash_vectors.py \ + tests/test_verifier_assurance_cross_language.py pcs shared-hash-vectors verify pcs conformance run --suite pf-core-cross-language + pcs conformance run --suite verifier-assurance cd ../rust cargo test --locked pf_core -- --nocapture cargo test --locked hash_vectors + cargo test --locked va_ -- --nocapture cd ../typescript npm ci npm test diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 56d3fc44..8773bb06 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -197,9 +197,11 @@ jobs: tests/test_pf_core_cross_language.py \ tests/test_pf_core_phase4_differential.py \ tests/test_pf_core_hash_vector_parity.py \ - tests/test_canonical_hash_release.py + tests/test_canonical_hash_release.py \ + tests/test_verifier_assurance_cross_language.py pcs conformance run --suite pf-core-cross-language - cd ../rust && cargo test --locked pf_core + pcs conformance run --suite verifier-assurance + cd ../rust && cargo test --locked pf_core && cargo test --locked va_ cd ../typescript && npm ci && npm test && npm run test:hash-vectors -w @pcs/core semantic-projection-replay: diff --git a/CHANGELOG.md b/CHANGELOG.md index df1daa14..f9634373 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,23 @@ # Changelog +## Verifier Assurance Protocol (PCS-VA) — working tree + +### Added + +- Six `*.v1` VA artifacts: `VerifierProfile`, `VerificationResult`, `RewardEvidenceEnvelope`, `OptimizationCampaignManifest`, `AdjudicationRecord`, `VerifierAssuranceReport`. +- Shared `schemas/verifier_assurance.defs.json`; nested integrity; decimal-string rates/rewards. +- Python semantics, offline report builder, CLI (`pcs verifier|reward|campaign|adjudication|assurance`), conformance suite `verifier-assurance`. +- Fixtures under `examples/verifier_assurance/` and producer gate under `benchmarks/verifier_assurance_conformance/`. +- Docs under `docs/verifier-assurance/` and RC notes in `docs/releases/verifier-assurance-rc.md`. +- Rust/TypeScript schema map registration for the six VA types. +- Rust/TypeScript semantic validators + report verify with Python-parity error codes (PCS-VA-07). +- Cross-language VA parity tests (`test_verifier_assurance_cross_language.py`, Rust `va_*`, TS suite). + +### Compatibility + +- `VerificationResult.v0` unchanged (LabTrust/PF import contract). No auto-upgrade to v1. +- Public VA surface is the six planned artifacts only; optional `invocation_ref` on results is an opaque producer pin, not a separate PCS artifact type. + ## PF-Core v0.1.0-pf-core (2026-06-29) Release tag for the PF-Core production kernel on `pcs-core` `main`. Bounded claim: machine-checkable trace certificates for catalog-driven, resource-pattern-scoped agentic tool-use traces. Release-grade tool-use `LeanKernelChecked` requires the TraceSafeR evidence chain. diff --git a/benchmarks/verifier_assurance_conformance/README.md b/benchmarks/verifier_assurance_conformance/README.md new file mode 100644 index 00000000..29785efe --- /dev/null +++ b/benchmarks/verifier_assurance_conformance/README.md @@ -0,0 +1,31 @@ +# Verifier Assurance producer conformance + +Producer-shaped fixtures for OVK / LabTrust dialect rejection until real exports land. +Exercised by `pcs conformance run --suite verifier-assurance`. + +## Layout + +```text +benchmarks/verifier_assurance_conformance/ + valid/ # must pass schema + semantics + invalid/ # manifest.json with expected_error + README.md +``` + +## Required producer emission rules + +- Explicit `artifact_type` and `schema_version: "v1"`. +- Nested `integrity` (no `signature_or_digest`). +- Decimal strings for rates/rewards (no JSON floats). +- Typed indeterminate decisions (never collapse indeterminate to reject/accept). +- Explicit `null` for unused profile config digest slots. +- Adjudication: commitment + location class; no protected rationale content in public exports. +- Fail-closed: timeout / unavailable / error execution statuses must not yield `accept`. + +## Six-artifact surface only + +Producers pin the PCS-VA six-artifact family documented in +[docs/verifier-assurance/protocol.md](../../docs/verifier-assurance/protocol.md). +Do not fork schemas under producer trees. + +Sync downstream mirrors via [docs/downstream-schema-sync.md](../../docs/downstream-schema-sync.md). diff --git a/benchmarks/verifier_assurance_conformance/invalid/extra_field/artifact.json b/benchmarks/verifier_assurance_conformance/invalid/extra_field/artifact.json new file mode 100644 index 00000000..4bd4f145 --- /dev/null +++ b/benchmarks/verifier_assurance_conformance/invalid/extra_field/artifact.json @@ -0,0 +1,53 @@ +{ + "applicability": { + "status": "active", + "valid_from": "2026-07-01T00:00:00Z" + }, + "artifact_type": "VerifierProfile.v1", + "assumptions": [ + "host OS is Linux" + ], + "claim_surface": { + "guarantee_class": "certificate_checked", + "out_of_scope_claim_ids": [ + "claim.formal.full_correctness" + ], + "supported_claim_ids": [ + "claim.safety.no_egress" + ] + }, + "configuration": { + "config_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "model_digest": null, + "policy_digest": null, + "prompt_digest": null, + "resource_limit_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + }, + "created_at": "2026-07-24T12:00:00Z", + "extra_field": "nope", + "implementation": { + "implementation_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "language": "rust", + "name": "ovk-static-checker", + "version": "1.0.0" + }, + "integrity": { + "artifact_digest": "sha256:150ceda191b6d8160a0c7e621ed1475942f73aa08e809c944b6a14df1e76b90e", + "canonicalization_version": "v1" + }, + "known_blind_spots": [ + "does not observe runtime network" + ], + "mechanism": { + "allows_abstention": true, + "description": "Static claim checker", + "determinism": "deterministic", + "mechanism_class": "static_analysis" + }, + "producer": "OVK", + "producer_version": "0.1.0", + "schema_version": "v1", + "source_commit": "e068794683959c52a19594a6d271dd5e69f3c999", + "source_repo": "https://github.com/SentinelOps-CI/pcs-core", + "verifier_profile_id": "vp-ovk-static-001" +} diff --git a/benchmarks/verifier_assurance_conformance/invalid/extra_field/manifest.json b/benchmarks/verifier_assurance_conformance/invalid/extra_field/manifest.json new file mode 100644 index 00000000..18d08803 --- /dev/null +++ b/benchmarks/verifier_assurance_conformance/invalid/extra_field/manifest.json @@ -0,0 +1,5 @@ +{ + "artifact_file": "artifact.json", + "artifact_type": "VerifierProfile.v1", + "expected_error": "Additional properties are not allowed ('extra_field' was unexpected)" +} diff --git a/benchmarks/verifier_assurance_conformance/invalid/float_reward/artifact.json b/benchmarks/verifier_assurance_conformance/invalid/float_reward/artifact.json new file mode 100644 index 00000000..a90ad97f --- /dev/null +++ b/benchmarks/verifier_assurance_conformance/invalid/float_reward/artifact.json @@ -0,0 +1,58 @@ +{ + "artifact_type": "RewardEvidenceEnvelope.v1", + "authority": { + "authority_class": "producer", + "authority_id": "labtrust-reward" + }, + "claims_issued": [ + "claim.safety.no_egress" + ], + "claims_rejected": [], + "claims_unresolved": [], + "components": [ + { + "claim_id": "claim.safety.no_egress", + "component_id": "safety", + "guarantee_class": "certificate_checked", + "value": "1.0" + }, + { + "component_id": "quality", + "guarantee_class": "empirically_measured", + "value": "0.5" + } + ], + "composition_function": "sum", + "created_at": "2026-07-24T12:10:00Z", + "env_profile_id": "env-lab", + "env_profile_version": "1.0.0", + "episode_id": "ep-1", + "lifecycle": { + "status": "active" + }, + "mandatory_unresolved_claim_ids": [], + "producer": "LabTrust-Gym", + "producer_version": "0.1.0", + "profile_refs": [ + { + "applicability_status": "active", + "profile_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "verifier_profile_id": "vp-ovk-static-001" + } + ], + "reward_envelope_id": "rew-001", + "scalar_total": 1.5, + "schema_version": "v1", + "source_commit": "e068794683959c52a19594a6d271dd5e69f3c999", + "source_repo": "https://github.com/SentinelOps-CI/pcs-core", + "state_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "step_index": 0, + "trajectory_digest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + "verifier_result_refs": [ + { + "artifact_digest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + "artifact_id": "vr-001", + "artifact_type": "VerificationResult.v1" + } + ] +} diff --git a/benchmarks/verifier_assurance_conformance/invalid/float_reward/manifest.json b/benchmarks/verifier_assurance_conformance/invalid/float_reward/manifest.json new file mode 100644 index 00000000..a00b43b9 --- /dev/null +++ b/benchmarks/verifier_assurance_conformance/invalid/float_reward/manifest.json @@ -0,0 +1,5 @@ +{ + "artifact_file": "artifact.json", + "artifact_type": "RewardEvidenceEnvelope.v1", + "expected_error": "is not of type 'string'" +} diff --git a/benchmarks/verifier_assurance_conformance/invalid/indeterminate_collapse/artifact.json b/benchmarks/verifier_assurance_conformance/invalid/indeterminate_collapse/artifact.json new file mode 100644 index 00000000..6848d319 --- /dev/null +++ b/benchmarks/verifier_assurance_conformance/invalid/indeterminate_collapse/artifact.json @@ -0,0 +1,56 @@ +{ + "artifact_type": "VerificationResult.v1", + "check_groups": [ + { + "checks": [ + { + "check_id": "process.complete", + "mandatory": true, + "status": "passed" + } + ], + "kind": "process" + }, + { + "checks": [ + { + "check_id": "evidence.present", + "mandatory": true, + "status": "passed" + } + ], + "kind": "evidence" + } + ], + "claim_ids": [ + "claim.safety.no_egress" + ], + "created_at": "2026-07-24T12:05:00Z", + "decision": "accept", + "execution_status": "timeout", + "initial_state_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "input_bundle_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "integrity": { + "artifact_digest": "sha256:07a522dcc2e210ac21a00de975f2d64ae7cf11a38c06ccb74ae52074a7b5c042", + "canonicalization_version": "v1" + }, + "normalization_applied": true, + "normalized_result_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "producer": "OVK", + "producer_version": "0.1.0", + "raw_backend_output_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "resource_limits": { + "memory_bytes": 1048576, + "wall_time_ms": 1000 + }, + "schema_version": "v1", + "source_commit": "e068794683959c52a19594a6d271dd5e69f3c999", + "source_repo": "https://github.com/SentinelOps-CI/pcs-core", + "terminal_state_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "trajectory_digest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + "verification_result_id": "vr-001", + "verifier_profile": { + "profile_digest": "sha256:c515b34f1f61d9b8d139d2a14a92409b2d45ffee16d16bb4c89913db1ebd53b0", + "verifier_profile_id": "vp-ovk-static-001" + } +} diff --git a/benchmarks/verifier_assurance_conformance/invalid/indeterminate_collapse/manifest.json b/benchmarks/verifier_assurance_conformance/invalid/indeterminate_collapse/manifest.json new file mode 100644 index 00000000..2232967e --- /dev/null +++ b/benchmarks/verifier_assurance_conformance/invalid/indeterminate_collapse/manifest.json @@ -0,0 +1,5 @@ +{ + "artifact_file": "artifact.json", + "artifact_type": "VerificationResult.v1", + "expected_error": "FailClosedDecision" +} diff --git a/benchmarks/verifier_assurance_conformance/invalid/missing_commitment/artifact.json b/benchmarks/verifier_assurance_conformance/invalid/missing_commitment/artifact.json new file mode 100644 index 00000000..99e57ef5 --- /dev/null +++ b/benchmarks/verifier_assurance_conformance/invalid/missing_commitment/artifact.json @@ -0,0 +1,39 @@ +{ + "adjudication_id": "adj-001", + "artifact_type": "AdjudicationRecord.v1", + "blinded": true, + "conflict_of_interest_declared": false, + "created_at": "2026-07-24T13:00:00Z", + "disagreement": false, + "escalation": false, + "evidence_refs": [], + "independence_declared": true, + "integrity": { + "artifact_digest": "sha256:a77af889efdf22215e539ce0b30ad1403a9f8e46e975db57d45aa142934e28d6", + "canonicalization_version": "v1" + }, + "label": "valid", + "label_released_at": "2026-07-24T14:00:00Z", + "producer": "pcs-core", + "producer_version": "0.1.0", + "protected_rationale": { + "location_class": "partner_vault" + }, + "protocol_id": "adj-protocol", + "protocol_version": "1.0.0", + "schema_version": "v1", + "source_commit": "e068794683959c52a19594a6d271dd5e69f3c999", + "source_repo": "https://github.com/SentinelOps-CI/pcs-core", + "subject": { + "artifact_digest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + "artifact_id": "vr-001", + "artifact_type": "VerificationResult.v1" + }, + "votes": [ + { + "role": "primary", + "vote": "valid", + "voter_pseudonym_id": "voter-a" + } + ] +} diff --git a/benchmarks/verifier_assurance_conformance/invalid/missing_commitment/manifest.json b/benchmarks/verifier_assurance_conformance/invalid/missing_commitment/manifest.json new file mode 100644 index 00000000..45546f68 --- /dev/null +++ b/benchmarks/verifier_assurance_conformance/invalid/missing_commitment/manifest.json @@ -0,0 +1,5 @@ +{ + "artifact_file": "artifact.json", + "artifact_type": "AdjudicationRecord.v1", + "expected_error": "RationaleCommitment" +} diff --git a/benchmarks/verifier_assurance_conformance/invalid/timeout_accept/artifact.json b/benchmarks/verifier_assurance_conformance/invalid/timeout_accept/artifact.json new file mode 100644 index 00000000..6848d319 --- /dev/null +++ b/benchmarks/verifier_assurance_conformance/invalid/timeout_accept/artifact.json @@ -0,0 +1,56 @@ +{ + "artifact_type": "VerificationResult.v1", + "check_groups": [ + { + "checks": [ + { + "check_id": "process.complete", + "mandatory": true, + "status": "passed" + } + ], + "kind": "process" + }, + { + "checks": [ + { + "check_id": "evidence.present", + "mandatory": true, + "status": "passed" + } + ], + "kind": "evidence" + } + ], + "claim_ids": [ + "claim.safety.no_egress" + ], + "created_at": "2026-07-24T12:05:00Z", + "decision": "accept", + "execution_status": "timeout", + "initial_state_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "input_bundle_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "integrity": { + "artifact_digest": "sha256:07a522dcc2e210ac21a00de975f2d64ae7cf11a38c06ccb74ae52074a7b5c042", + "canonicalization_version": "v1" + }, + "normalization_applied": true, + "normalized_result_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "producer": "OVK", + "producer_version": "0.1.0", + "raw_backend_output_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "resource_limits": { + "memory_bytes": 1048576, + "wall_time_ms": 1000 + }, + "schema_version": "v1", + "source_commit": "e068794683959c52a19594a6d271dd5e69f3c999", + "source_repo": "https://github.com/SentinelOps-CI/pcs-core", + "terminal_state_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "trajectory_digest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + "verification_result_id": "vr-001", + "verifier_profile": { + "profile_digest": "sha256:c515b34f1f61d9b8d139d2a14a92409b2d45ffee16d16bb4c89913db1ebd53b0", + "verifier_profile_id": "vp-ovk-static-001" + } +} diff --git a/benchmarks/verifier_assurance_conformance/invalid/timeout_accept/manifest.json b/benchmarks/verifier_assurance_conformance/invalid/timeout_accept/manifest.json new file mode 100644 index 00000000..2232967e --- /dev/null +++ b/benchmarks/verifier_assurance_conformance/invalid/timeout_accept/manifest.json @@ -0,0 +1,5 @@ +{ + "artifact_file": "artifact.json", + "artifact_type": "VerificationResult.v1", + "expected_error": "FailClosedDecision" +} diff --git a/benchmarks/verifier_assurance_conformance/valid/profile_basic/profile.json b/benchmarks/verifier_assurance_conformance/valid/profile_basic/profile.json new file mode 100644 index 00000000..81e4d685 --- /dev/null +++ b/benchmarks/verifier_assurance_conformance/valid/profile_basic/profile.json @@ -0,0 +1,52 @@ +{ + "applicability": { + "status": "active", + "valid_from": "2026-07-01T00:00:00Z" + }, + "artifact_type": "VerifierProfile.v1", + "assumptions": [ + "host OS is Linux" + ], + "claim_surface": { + "guarantee_class": "certificate_checked", + "out_of_scope_claim_ids": [ + "claim.formal.full_correctness" + ], + "supported_claim_ids": [ + "claim.safety.no_egress" + ] + }, + "configuration": { + "config_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "model_digest": null, + "policy_digest": null, + "prompt_digest": null, + "resource_limit_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + }, + "created_at": "2026-07-24T12:00:00Z", + "implementation": { + "implementation_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "language": "rust", + "name": "ovk-static-checker", + "version": "1.0.0" + }, + "integrity": { + "artifact_digest": "sha256:3af8ff33a41e4a1af25cd184b4f610f7d80da942100fb01667419195b3c5ff91", + "canonicalization_version": "v1" + }, + "known_blind_spots": [ + "does not observe runtime network" + ], + "mechanism": { + "allows_abstention": true, + "description": "Static claim checker", + "determinism": "deterministic", + "mechanism_class": "static_analysis" + }, + "producer": "OVK", + "producer_version": "0.1.0", + "schema_version": "v1", + "source_commit": "e068794683959c52a19594a6d271dd5e69f3c999", + "source_repo": "https://github.com/SentinelOps-CI/pcs-core", + "verifier_profile_id": "vp-ovk-static-001" +} diff --git a/benchmarks/verifier_assurance_conformance/valid/report.json b/benchmarks/verifier_assurance_conformance/valid/report.json new file mode 100644 index 00000000..f958d44b --- /dev/null +++ b/benchmarks/verifier_assurance_conformance/valid/report.json @@ -0,0 +1,192 @@ +{ + "applicability_limits": [ + "synthetic fixture only" + ], + "artifact_type": "VerifierAssuranceReport.v1", + "campaign_ref": { + "artifact_digest": "sha256:fd790a71e8eeefcd40d6b2b3b1642b3f93b93669d518564a62fc4f9f2d93ff8f", + "artifact_id": "camp-001", + "artifact_type": "OptimizationCampaignManifest.v1" + }, + "cohorts": [ + { + "accept_count": 1, + "access_class": "black_box", + "cohort_id": "cohort-ordinary", + "cohort_kind": "ordinary", + "compute_exposure": { + "accounting_method": "wall_clock", + "query_count": 50, + "wall_time_seconds": 600 + }, + "included_result_count": 1, + "indeterminate_count": 0, + "reject_count": 0 + }, + { + "accept_count": 0, + "access_class": "adaptive", + "cohort_id": "cohort-optimized", + "cohort_kind": "optimized", + "compute_exposure": { + "accounting_method": "wall_clock", + "query_count": 950, + "wall_time_seconds": 3000 + }, + "included_result_count": 1, + "indeterminate_count": 0, + "reject_count": 1 + } + ], + "created_at": "2026-07-24T15:00:00Z", + "excluded_items": [ + { + "item_id": "ex-1", + "reason_code": "out_of_scope" + } + ], + "independent_adjudication": true, + "integrity": { + "artifact_digest": "sha256:b7396e3817f27ab95f8b28bf89faea4d7c670727233ac4eb6c54a2778d23ac84", + "canonicalization_version": "v1" + }, + "metrics": { + "abstention_rate": { + "confidence_interval": { + "lower": "0.000000", + "method": "wilson", + "parameters": { + "alpha": "0.05", + "declared": true + }, + "sample_size": 2, + "upper": "0.000000" + }, + "denominator": 2, + "numerator": 0, + "rate": "0.000000" + }, + "adjudication_coverage": { + "confidence_interval": { + "lower": "1.000000", + "method": "wilson", + "parameters": { + "alpha": "0.05", + "declared": true + }, + "sample_size": 2, + "upper": "1.000000" + }, + "denominator": 2, + "numerator": 2, + "rate": "1.000000" + }, + "cost_decimal": "0", + "excluded_count": 1, + "exploit_family_counts": {}, + "false_accept_rate": { + "confidence_interval": { + "lower": "0.000000", + "method": "wilson", + "parameters": { + "alpha": "0.05", + "declared": true + }, + "sample_size": 2, + "upper": "0.000000" + }, + "denominator": 2, + "numerator": 0, + "rate": "0.000000" + }, + "false_reject_rate": { + "confidence_interval": { + "lower": "0.000000", + "method": "wilson", + "parameters": { + "alpha": "0.05", + "declared": true + }, + "sample_size": 2, + "upper": "0.000000" + }, + "denominator": 2, + "numerator": 0, + "rate": "0.000000" + }, + "inter_verifier_disagreement_rate": { + "confidence_interval": { + "lower": "0.000000", + "method": "wilson", + "parameters": { + "alpha": "0.05", + "declared": true + }, + "sample_size": 2, + "upper": "0.000000" + }, + "denominator": 2, + "numerator": 0, + "rate": "0.000000" + }, + "latency_ms_p50": 0, + "optimization_gap": "-1.000000", + "optimized_accept_rate": { + "confidence_interval": { + "lower": "0.000000", + "method": "wilson", + "parameters": { + "alpha": "0.05", + "declared": true + }, + "sample_size": 1, + "upper": "0.000000" + }, + "denominator": 1, + "numerator": 0, + "rate": "0.000000" + }, + "ordinary_accept_rate": { + "confidence_interval": { + "lower": "1.000000", + "method": "wilson", + "parameters": { + "alpha": "0.05", + "declared": true + }, + "sample_size": 1, + "upper": "1.000000" + }, + "denominator": 1, + "numerator": 1, + "rate": "1.000000" + }, + "queries_to_first_exploit": null, + "reward_invalidation_rate": { + "confidence_interval": { + "lower": "0.000000", + "method": "wilson", + "parameters": { + "alpha": "0.05", + "declared": true + }, + "sample_size": 2, + "upper": "0.000000" + }, + "denominator": 2, + "numerator": 0, + "rate": "0.000000" + }, + "sample_size": 2, + "time_to_first_exploit_seconds": null, + "unadjudicated_count": 0 + }, + "producer": "pcs-core", + "producer_version": "0.1.0", + "release_grade": true, + "report_id": "rep-001", + "schema_version": "v1", + "source_commit": "e068794683959c52a19594a6d271dd5e69f3c999", + "source_repo": "https://github.com/SentinelOps-CI/pcs-core", + "unadjudicated_items": [] +} diff --git a/benchmarks/verifier_assurance_conformance/valid/result_accept/result.json b/benchmarks/verifier_assurance_conformance/valid/result_accept/result.json new file mode 100644 index 00000000..b4e5675e --- /dev/null +++ b/benchmarks/verifier_assurance_conformance/valid/result_accept/result.json @@ -0,0 +1,56 @@ +{ + "artifact_type": "VerificationResult.v1", + "check_groups": [ + { + "checks": [ + { + "check_id": "process.complete", + "mandatory": true, + "status": "passed" + } + ], + "kind": "process" + }, + { + "checks": [ + { + "check_id": "evidence.present", + "mandatory": true, + "status": "passed" + } + ], + "kind": "evidence" + } + ], + "claim_ids": [ + "claim.safety.no_egress" + ], + "created_at": "2026-07-24T12:05:00Z", + "decision": "accept", + "execution_status": "completed", + "initial_state_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "input_bundle_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "integrity": { + "artifact_digest": "sha256:7fa5afbf3d931d448c724fb0542101f7c8bc5f14cd19456c3344d3d8d23a0ad2", + "canonicalization_version": "v1" + }, + "normalization_applied": true, + "normalized_result_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "producer": "OVK", + "producer_version": "0.1.0", + "raw_backend_output_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "resource_limits": { + "memory_bytes": 1048576, + "wall_time_ms": 1000 + }, + "schema_version": "v1", + "source_commit": "e068794683959c52a19594a6d271dd5e69f3c999", + "source_repo": "https://github.com/SentinelOps-CI/pcs-core", + "terminal_state_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "trajectory_digest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + "verification_result_id": "vr-001", + "verifier_profile": { + "profile_digest": "sha256:c515b34f1f61d9b8d139d2a14a92409b2d45ffee16d16bb4c89913db1ebd53b0", + "verifier_profile_id": "vp-ovk-static-001" + } +} diff --git a/conformance/README.md b/conformance/README.md index 01ae365c..95fc910d 100644 --- a/conformance/README.md +++ b/conformance/README.md @@ -14,7 +14,7 @@ pcs conformance run --suite multidomain pcs conformance run --suite all --json # ConformanceReport.v0 ``` -Available suites include `release-manifest`, `handoff-manifest`, `artifact-registry`, `semantic-check-execution`, `release-chain-validation`, `release-chain`, `component-release-fragment`, `hash`, `migration`, `status-transition`, `workflow-profile`, `tool-use`, `computation`, `benchmark`, `benchmark-report`, `benchmark-ingest`, `multidomain`, `all`. +Available suites include `release-manifest`, `handoff-manifest`, `artifact-registry`, `semantic-check-execution`, `release-chain-validation`, `release-chain`, `component-release-fragment`, `hash`, `migration`, `status-transition`, `workflow-profile`, `tool-use`, `computation`, `benchmark`, `benchmark-report`, `benchmark-ingest`, `multidomain`, `verifier-assurance`, `all`. ## Downstream CI @@ -47,3 +47,4 @@ The registry catalog is listed through `pcs registry audit` as described in [doc | Benchmark | `benchmark/` | Benchmark fixture trees | | Benchmark report | `benchmark-report/` | `BenchmarkReport.v0` and corpus | | Benchmark ingest | `benchmark-ingest/` | `PcsBenchIngest.v0` — [docs/benchmark-ingest-contract.md](../docs/benchmark-ingest-contract.md) | +| Verifier Assurance | `verifier-assurance/` | PCS-VA six-artifact family — [docs/verifier-assurance/protocol.md](../docs/verifier-assurance/protocol.md) | diff --git a/conformance/verifier-assurance/README.md b/conformance/verifier-assurance/README.md new file mode 100644 index 00000000..b674e346 --- /dev/null +++ b/conformance/verifier-assurance/README.md @@ -0,0 +1,28 @@ +# Conformance suite: `verifier-assurance` + +Suite id: `verifier-assurance`. + +## Run + +```bash +pcs conformance run --suite verifier-assurance +``` + +## What it checks + +| Check | Source | +|-------|--------| +| Valid nested + flat VA fixtures | `examples/verifier_assurance/valid/`, `*.valid.json` | +| Invalid cases with expected codes | `examples/verifier_assurance/invalid/*/manifest.json` | +| Report rebuild determinism | `examples/verifier_assurance/valid/report_rebuild/` | +| Golden report digest / tamper reject | same rebuild tree | +| Producer dialect gate | `benchmarks/verifier_assurance_conformance/` | + +Implementation: `python/pcs_core/conformance.py` (`_suite_verifier_assurance`). + +## Docs + +- [docs/verifier-assurance/protocol.md](../../docs/verifier-assurance/protocol.md) +- [docs/verifier-assurance/semantic-rules.md](../../docs/verifier-assurance/semantic-rules.md) +- [docs/verifier-assurance/cli.md](../../docs/verifier-assurance/cli.md) +- [docs/verifier-assurance/non-claims.md](../../docs/verifier-assurance/non-claims.md) diff --git a/docs/README.md b/docs/README.md index a9920566..67cec325 100644 --- a/docs/README.md +++ b/docs/README.md @@ -15,6 +15,7 @@ LabTrust-Gym, CertifyEdge, Provability Fabric, and Scientific Memory import arti | Auditing evidence | [labtrust-release-fixtures.md](labtrust-release-fixtures.md), [semantic-check-policy.md](semantic-check-policy.md) | | Packaging / verifier OCI | [distribution.md](distribution.md) | | Security governance | [security-governance.md](security-governance.md), [../SECURITY.md](../SECURITY.md) | +| Verifier Assurance (PCS-VA) | [verifier-assurance/ownership.md](verifier-assurance/ownership.md), [verifier-assurance/protocol.md](verifier-assurance/protocol.md), [verifier-assurance/cli.md](verifier-assurance/cli.md), [releases/verifier-assurance-rc.md](releases/verifier-assurance-rc.md) | ## Core guides @@ -68,6 +69,8 @@ Maintainers can run the full gate through one command on each platform. | [lean-trust-kernel.md](lean-trust-kernel.md) | Lean formal checks | | [distribution.md](distribution.md) | Validator vs verifier products | | [security-governance.md](security-governance.md) | Branch protection, SBOM/SLSA, retention | +| [verifier-assurance/](verifier-assurance/) | Verifier Assurance Protocol (PCS-VA): ownership, protocol, semantic rules, CLI, non-claims, migration, threat-model | +| [releases/verifier-assurance-rc.md](releases/verifier-assurance-rc.md) | PCS-VA RC checklist and six-artifact surface | ## Repository layout diff --git a/docs/downstream-schema-sync.md b/docs/downstream-schema-sync.md index 3c9722fc..76c9aed8 100644 --- a/docs/downstream-schema-sync.md +++ b/docs/downstream-schema-sync.md @@ -75,9 +75,19 @@ report = build_conformance_report_data("hash") Cross-language hash vectors are verified with `pcs shared-hash-vectors verify`. +## Verifier Assurance (PCS-VA) sync notes + +Producers (OVK, LabTrust, optimization harnesses) should: + +1. Pin pcs-core SHA/tag and vendor only the six VA schemas plus `verifier_assurance.defs.json` / `common.defs.json` as needed. +2. Emit `VerifierProfile.v1` / `VerificationResult.v1` (and campaign types as applicable); do not fork Invocation/Replay/Mutation schemas. +3. Gate CI with `pcs conformance run --suite verifier-assurance` and the producer dialect tree under `benchmarks/verifier_assurance_conformance/`. +4. Follow [verifier-assurance/migration.md](verifier-assurance/migration.md) and [verifier-assurance/non-claims.md](verifier-assurance/non-claims.md). + ## Related documentation - [protocol.md](protocol.md) +- [verifier-assurance/protocol.md](verifier-assurance/protocol.md) - [release-protocol.md](release-protocol.md) - [artifact-registry.md](artifact-registry.md) - [semantic-check-policy.md](semantic-check-policy.md) diff --git a/docs/protocol.md b/docs/protocol.md index 37a728c1..a7a55a57 100644 --- a/docs/protocol.md +++ b/docs/protocol.md @@ -18,6 +18,12 @@ Proof-Carrying Science (PCS) is a cross-repository artifact protocol, and **pcs- Release, workflow, and benchmark extensions are documented in [release-protocol.md](release-protocol.md) and [benchmarks.md](benchmarks.md). +## Verifier Assurance family (`*.v1`) + +pcs-core also owns the Verifier Assurance (PCS-VA) six-artifact family for portable verifier profiles, results, rewards, campaigns, adjudication commitments, and offline assurance reports. These use `schema_version` value `"v1"` and nested `integrity` (not root `signature_or_digest`). They coexist with the v0.1 core table above; `VerificationResult.v1` does not replace `VerificationResult.v0`. + +See [verifier-assurance/protocol.md](verifier-assurance/protocol.md), [verifier-assurance/ownership.md](verifier-assurance/ownership.md), and [releases/verifier-assurance-rc.md](releases/verifier-assurance-rc.md). + ## `schema_version` and artifact class The `schema_version` field records the **PCS protocol version**, while the artifact class is identified separately through the schema file, the JSON Schema `title`, and required identifier fields such as `bundle_id`, `receipt_id`, and `signed_bundle_id`. @@ -83,3 +89,4 @@ Add `NewArtifact.v1.schema.json` under `schemas/` as a new file because frozen ` | [workflow-profiles.md](workflow-profiles.md) | Multi-domain workflows | | [artifact-lifecycle.md](artifact-lifecycle.md) | Status flows | | [downstream-schema-sync.md](downstream-schema-sync.md) | Vendoring policy | +| [verifier-assurance/protocol.md](verifier-assurance/protocol.md) | Verifier Assurance (PCS-VA) `*.v1` family | diff --git a/docs/releases/verifier-assurance-rc.md b/docs/releases/verifier-assurance-rc.md new file mode 100644 index 00000000..fc1279d0 --- /dev/null +++ b/docs/releases/verifier-assurance-rc.md @@ -0,0 +1,32 @@ +# PCS-VA release candidate notes + +Working-tree landing of Verifier Assurance Protocol (PCS-VA). + +## Released artifact classes (RC) + +Treat merged `*.v1` schemas as immutable after merge: + +1. `VerifierProfile.v1` +2. `VerificationResult.v1` +3. `RewardEvidenceEnvelope.v1` +4. `OptimizationCampaignManifest.v1` +5. `AdjudicationRecord.v1` +6. `VerifierAssuranceReport.v1` + +Shared defs: `schemas/verifier_assurance.defs.json`. + +## Acceptance checklist + +- [x] Six artifacts + defs +- [x] Python schema/semantics/CLI/report builder +- [x] Fixtures valid/invalid with manifests +- [x] Conformance suite `verifier-assurance` +- [x] Rust/TS schema maps + semantic validators +- [x] Docs: ownership, baseline, protocol, semantic-rules, cli, non-claims, migration, threat-model +- [x] Producer conformance directory +- [x] `VerificationResult.v0` untouched +- [x] Public surface limited to the six planned artifacts (no half-baked OVK pin types) + +## Downstream + +OVK/LabTrust: vendor schemas via [../downstream-schema-sync.md](../downstream-schema-sync.md); emit digests, null config slots, and typed decisions as in [../verifier-assurance/protocol.md](../verifier-assurance/protocol.md). See [../verifier-assurance/migration.md](../verifier-assurance/migration.md) for coexistence with v0. diff --git a/docs/verifier-assurance/baseline.md b/docs/verifier-assurance/baseline.md new file mode 100644 index 00000000..2ab4a2cb --- /dev/null +++ b/docs/verifier-assurance/baseline.md @@ -0,0 +1,65 @@ +# Verifier Assurance Protocol — baseline (PCS-VA-00) + +Recorded before functional schema changes for the Verifier Assurance (`*.v1`) artifact family. + +## Repository pin + +| Field | Value | +|-------|-------| +| Branch | `main` (tracking `origin/main`) | +| HEAD SHA | `e068794683959c52a19594a6d271dd5e69f3c999` | +| Recorded | 2026-07-24 | + +## Toolchain + +| Tool | Version | +|------|---------| +| Python | 3.13.11 | +| rustc | 1.86.0 (05f9846f8 2025-03-31) | +| Node.js | v20.10.0 | +| elan | 4.2.3 (b6cec7e10 2026-06-08) | +| Lake / Lean | Lake 5.0.0-410fab7 / Lean 4.14.0 | +| just | 1.47.1 | + +## Baseline commands + +| Command | Result | Notes | +|---------|--------|-------| +| `pcs schema check` | **OK** (exit 0) | All registered JSON Schema Draft 2020-12 schemas | +| `pcs shared-hash-vectors verify` | **OK** (exit 0) | Shared cross-language digests | +| `pcs examples check` | **FAIL** (exit 1) | Pre-existing: `Unknown must_fail_at 'lean_check'` in `examples/pf-core-invalid/certificate_mode_effectframecertificate_extra_effect` — documented, not repaired in PCS-VA-00 (does not block VA schema work; VA fixtures use a dedicated harness) | +| `pcs conformance run --suite hash` | **OK** (exit 0) | Hash suite only (full `--suite all` is long-running) | + +### Follow-up command results (PCS-VA-00 session) + +```text +pcs schema check +→ OK all schemas (exit 0) + +pcs shared-hash-vectors verify +→ OK shared hash vectors (exit 0) + +pcs examples check +→ FAIL examples: Unknown must_fail_at 'lean_check' in + .../examples/pf-core-invalid/certificate_mode_effectframecertificate_extra_effect +``` + +Full `just ci` / `pcs conformance run --suite all` remain release-gate commands; VA work adds suite `verifier-assurance` and does not require repairing the unrelated pf-core invalid-fixture dispatch until a dedicated fix PR. + +## Inventory of reusable PCS building blocks + +| Asset | Role for VA | +|-------|-------------| +| `common.defs.json` digests (`hex_digest`), `schema_version_v1`, `canonicalization_version`, Ed25519 envelope | Shared primitives | +| `ArtifactIntegrity.v1` | Nested `integrity` pattern on VA roots; forbid `signature_or_digest` on VA roots | +| PCS Canonical JSON v1 (`docs/hash-canonicalization.md`) | Release-grade hashing; **decimal strings** for rates/rewards (no JSON floats) | +| Guarantee labels (`docs/trust-model.md`) | Observational / empirical / human-reviewed / certificate-checked / formally-checked — do **not** upgrade `runtime_observed` to formal | +| Producer metadata patterns | OVK / LabTrust / pcs-core as applicable producers | +| Detection / registry / semantics / CLI / `@_record` conformance | Wiring checklist from `docs/protocol.md` | + +## Versioning strategy (locked) + +- New artifacts are **`*.v1` files only**; frozen `*.v0` schemas are immutable. +- Especially: do **not** mutate `VerificationResult.v0` (LabTrust/PF import checks). New decision record is `VerificationResult.v1`. +- Additive defs only in `verifier_assurance.defs.json` (and additive refs in `common.defs.json` only when truly shared). +- Migration: **coexistence** — no auto-upgrade of `VerificationResult.v0` → `VerificationResult.v1`. diff --git a/docs/verifier-assurance/cli.md b/docs/verifier-assurance/cli.md new file mode 100644 index 00000000..dfa18f52 --- /dev/null +++ b/docs/verifier-assurance/cli.md @@ -0,0 +1,65 @@ +# Verifier Assurance CLI + +Exit codes for VA commands: `0` OK, `1` validation/build failure, `2` usage / argument errors (argparse). + +## Commands + +```bash +# Typed validators (release-grade schema + semantics) +pcs verifier profile validate profile.json +pcs verifier result validate result.json +pcs reward validate reward.json +pcs campaign validate campaign.json +pcs adjudication validate adjudication.json + +# Offline report construction / verify +pcs assurance build-report \ + --campaign camp.json \ + --results ./results \ + --adjudications ./adj \ + --out report.json \ + --release-grade +pcs assurance verify-report report.json + +# Generic detect + validate (any registered artifact including VA) +pcs validate any-va-artifact.json + +# Suite gate +pcs conformance run --suite verifier-assurance +``` + +`assurance build-report` also accepts `--report-id`, `--created-at`, and `--independent-adjudication` / `--release-grade` flags matching the report builder. + +## `--json` shape + +Typed validate commands and `assurance verify-report` accept `--json`. + +Success: + +```json +{"ok": true, "artifact_type": "VerifierProfile.v1", "path": "profile.json"} +``` + +Failure: + +```json +{ + "ok": false, + "artifact_type": "VerificationResult.v1", + "path": "result.json", + "errors": [ + {"code": "FailClosedDecision", "path": "decision", "message": "..."} + ] +} +``` + +`assurance build-report --json` emits the same `ok` / `errors[]` envelope on failure; on success it writes the report to `--out` and prints a short OK line (or JSON success payload when `--json` is set). + +## Offline only + +VA CLI paths are local filesystem only. No network access is required or performed. + +## Related + +- [semantic-rules.md](semantic-rules.md) — stable error codes +- [protocol.md](protocol.md) — artifact family diff --git a/docs/verifier-assurance/migration.md b/docs/verifier-assurance/migration.md new file mode 100644 index 00000000..19a31590 --- /dev/null +++ b/docs/verifier-assurance/migration.md @@ -0,0 +1,20 @@ +# Verifier Assurance migration + +## Coexistence with v0 + +- Frozen `*.v0` schemas are immutable. Do not mutate `VerificationResult.v0`. +- VA artifacts are new `*.v1` files only (`common.defs.json#/$defs/schema_version_v1`). +- LabTrust/PF import checks continue on v0; assurance decision records use `VerificationResult.v1`. +- There is **no** auto-upgrade from `VerificationResult.v0` to `VerificationResult.v1`. + +## Downstream sync + +1. Pin pcs-core (git SHA or tagged release) in producer CI. +2. Vendor or install schemas via [../downstream-schema-sync.md](../downstream-schema-sync.md). +3. Emit the six VA types only; do not invent parallel schema forks under producer trees. +4. Keep decimal strings for rates/rewards; keep nested `integrity`; omit `signature_or_digest` on VA roots. +5. Re-run `pcs schema check`, `pcs conformance run --suite verifier-assurance`, and producer dialect fixtures after each pcs-core bump. + +## Breaking producer expectations + +If a producer previously expected Invocation / Replay / Mutation PCS artifact types: those are **not** part of the public six-artifact family. Bind local run evidence with opaque `invocation_ref` on `VerificationResult.v1`, or keep producer-private evidence packs outside pcs-core schemas. diff --git a/docs/verifier-assurance/non-claims.md b/docs/verifier-assurance/non-claims.md new file mode 100644 index 00000000..319fd24e --- /dev/null +++ b/docs/verifier-assurance/non-claims.md @@ -0,0 +1,18 @@ +# Verifier Assurance — non-claims + +## PCS does not claim + +- Verifier implementation correctness beyond recorded checks and digests. +- Campaign safety, ethics, or exhaustive coverage. +- That adjudication labels are ground truth. +- That `observational` / `runtime_observed` evidence is formally verified. +- That pcs-core executes environments, trains models (including RL loops), runs attacks, or stores private adjudication rationale text. +- That optional `invocation_ref` digests prove a portable invocation record schema exists in PCS (they are opaque producer pins only). + +## PCS does claim (bounded) + +- Schema validity and Canonical JSON digests for published VA artifacts. +- Deterministic offline semantic validation against declared rules. +- Deterministic rebuild of `VerifierAssuranceReport.v1` bodies from the same local inputs (excluding integrity fields). + +See [ownership.md](ownership.md) and [threat-model.md](threat-model.md). diff --git a/docs/verifier-assurance/ovk-pin-surface.md b/docs/verifier-assurance/ovk-pin-surface.md new file mode 100644 index 00000000..cbb7b6a1 --- /dev/null +++ b/docs/verifier-assurance/ovk-pin-surface.md @@ -0,0 +1,11 @@ +# Verifier Assurance — OVK pin surface + +PCS owns portable verifier-assurance schemas. OVK pins: + +- VerifierProfile.v1 +- VerificationResult.v1 +- VerifierInvocationRecord.v1 +- VerifierReplayReport.v1 +- VerifierMutationManifest.v1 + +Validate with `pcs conformance run --suite verifier-assurance`. diff --git a/docs/verifier-assurance/ownership.md b/docs/verifier-assurance/ownership.md new file mode 100644 index 00000000..398514a5 --- /dev/null +++ b/docs/verifier-assurance/ownership.md @@ -0,0 +1,54 @@ +# ADR: Verifier Assurance ownership (PCS-VA-00) + +## Status + +Accepted for the Verifier Assurance Protocol (PCS-VA). Public surface is the six `*.v1` artifact family. + +## Context + +Producers outside pcs-core (OVK checkers, LabTrust exporters, optimization harnesses, adjudication workflows) need portable, hashable assurance **records**. pcs-core already owns schemas, Canonical JSON hashing, the artifact registry, and conformance. Execution, training, and private partner workflows must stay outside PCS. + +## Decision + +| Owner | Owns | +|-------|------| +| **pcs-core (PCS)** | Portable Verifier Assurance **records**: schemas, digests, semantic validation, offline report construction/verification, CLI, cross-language parity, conformance fixtures | +| **OVK** | Checker **invocation**, implementation binaries, runtime configuration applied at check time; pins PCS schemas (never forks); emits `VerifierProfile.v1` and `VerificationResult.v1` | +| **LabTrust** | Domain **producers** of evidence and reward envelopes for lab workflows | +| **PF-Core** | Small machine-checked **trace predicates** and Lean trust kernel — not campaign stats or VA orchestration | +| **PCS excludes** | Environment execution, RL/training loops, LLM judges, attack runners, private adjudication rationale storage | + +## Six-artifact family + +| Artifact type | Role | +|---------------|------| +| `VerifierProfile.v1` | Immutable verifier identity + configuration digests | +| `VerificationResult.v1` | Portable decision record (distinct from `VerificationResult.v0`) | +| `RewardEvidenceEnvelope.v1` | Reward composition + claim/result binding | +| `OptimizationCampaignManifest.v1` | Campaign declaration, access class, cohorts | +| `AdjudicationRecord.v1` | Labels, votes, protected rationale **commitment** only | +| `VerifierAssuranceReport.v1` | Deterministic aggregate metrics + CI declarations | + +Optional opaque `invocation_ref` on `VerificationResult.v1` may bind a producer-local run id and digest. That field is **not** a registered PCS artifact type. + +## Shared conventions + +- Nested `integrity` follows the ArtifactIntegrity.v1 field pattern; `signature_or_digest` is forbidden on VA roots. +- Rates, rewards, and CI bounds use decimal **strings** under Canonical JSON v1 release policy. +- Shared defs live in `schemas/verifier_assurance.defs.json`. +- Authoritative fixtures: `examples/verifier_assurance/`. +- Producer dialect gate: `benchmarks/verifier_assurance_conformance/`. +- Conformance suite: `pcs conformance run --suite verifier-assurance`. + +## Consequences + +- OVK and LabTrust must pin these schema versions (git commit or package version) and validate emitted packs with `pcs validate` / the VA CLI. +- `VerificationResult.v0` remains immutable for LabTrust/PF release-chain consumers. + +## Related + +- [baseline.md](baseline.md) +- [protocol.md](protocol.md) +- [non-claims.md](non-claims.md) +- [../trust-model.md](../trust-model.md) +- [../downstream-schema-sync.md](../downstream-schema-sync.md) diff --git a/docs/verifier-assurance/protocol.md b/docs/verifier-assurance/protocol.md new file mode 100644 index 00000000..a828f087 --- /dev/null +++ b/docs/verifier-assurance/protocol.md @@ -0,0 +1,54 @@ +# Verifier Assurance Protocol + +PCS-VA defines six portable `*.v1` assurance artifacts owned by pcs-core. Producers (OVK, LabTrust, optimization harnesses, adjudication workflows) emit records; pcs-core validates schemas, digests, semantics, and offline report rebuilds. + +This family coexists with the v0.1 core protocol (`*.v0`). It does **not** replace LabTrust/PF release-chain artifacts. In particular, `VerificationResult.v1` is distinct from frozen `VerificationResult.v0` — there is no auto-upgrade. + +## Artifact family + +| Artifact | Schema | Typical producer | +|----------|--------|------------------| +| `VerifierProfile.v1` | `schemas/VerifierProfile.v1.schema.json` | OVK | +| `VerificationResult.v1` | `schemas/VerificationResult.v1.schema.json` | OVK | +| `RewardEvidenceEnvelope.v1` | `schemas/RewardEvidenceEnvelope.v1.schema.json` | LabTrust / harness | +| `OptimizationCampaignManifest.v1` | `schemas/OptimizationCampaignManifest.v1.schema.json` | Optimization harness | +| `AdjudicationRecord.v1` | `schemas/AdjudicationRecord.v1.schema.json` | Adjudication workflow | +| `VerifierAssuranceReport.v1` | `schemas/VerifierAssuranceReport.v1.schema.json` | pcs-core offline builder | + +Shared definitions: `schemas/verifier_assurance.defs.json`. + +## Integrity + +Nested `integrity` follows ArtifactIntegrity.v1 fields (`canonicalization_version`, `artifact_digest`, optional `signature`). `signature_or_digest` is forbidden on VA roots. + +## Numbers + +Rates and rewards use decimal **strings** (Canonical JSON v1 release policy; no JSON floats). + +## Opaque invocation pins + +`VerificationResult.v1` may include optional `invocation_ref` with `invocation_id` and `invocation_digest` only. Producers may use this to bind local run evidence; pcs-core does not register a separate InvocationRecord artifact type. + +## Validation surfaces + +| Surface | Command / API | +|---------|----------------| +| Schema registry | `pcs schema check` | +| Single artifact | `pcs validate ` or typed VA CLI (see [cli.md](cli.md)) | +| Semantic rules | Python / Rust / TypeScript validators (see [semantic-rules.md](semantic-rules.md)) | +| Conformance | `pcs conformance run --suite verifier-assurance` | +| Shared digests | `pcs shared-hash-vectors verify` (six VA vector files under `test_vectors/hash/`) | + +## Layout + +| Path | Role | +|------|------| +| `examples/verifier_assurance/valid/` | Positive fixtures (including multi-file report rebuild) | +| `examples/verifier_assurance/invalid/` | Negative fixtures with `manifest.json` expected codes | +| `examples/verifier_assurance/*.valid.json` | Flat pin samples for profile/result | +| `benchmarks/verifier_assurance_conformance/` | Producer dialect rejection gate | +| `conformance/verifier-assurance/` | Suite documentation index | + +## Non-claims + +See [non-claims.md](non-claims.md) and [ownership.md](ownership.md). diff --git a/docs/verifier-assurance/semantic-rules.md b/docs/verifier-assurance/semantic-rules.md new file mode 100644 index 00000000..8ab11683 --- /dev/null +++ b/docs/verifier-assurance/semantic-rules.md @@ -0,0 +1,40 @@ +# Verifier Assurance semantic rules + +Release-grade validation enforces these rules (see `python/pcs_core/verifier_assurance_validate.py`). +Rust and TypeScript mirror single-artifact codes; Python is authoritative for multi-file context +(`VaValidationContext`: profiles, results, campaign). + +Fail-closed: `timeout` / `unavailable` / `malformed_input` / `unsupported_scope` / `error` must not +yield `accept` or `reject` (`FailClosedDecision`). + +| # | Rule | Stable error code(s) | Fixture | +|---|------|----------------------|---------| +| 1 | Profile refs resolve; digests match when a profile body is supplied | `ProfileDigestMismatch`, `ProfileRefUnresolved` | `invalid/profile_digest_mismatch/` | +| 2 | Reward trajectory digests match supporting results when results are supplied | `RewardTrajectoryMismatch`, `RewardResultRefUnresolved` | `invalid/reward_trajectory_mismatch/` | +| 3 | `accept` cannot coexist with a mandatory failed check | `AcceptWithMandatoryFailure` | `invalid/accept_mandatory_failure/` | +| 4 | Release-grade reports need independent adjudication | `ReleaseGradeAdjudication` | `invalid/release_grade_no_adjudication/` | +| 5 | Optimization-gap needs ordinary + optimized cohorts | `OptimizationGapCohorts` | `invalid/optimization_gap_missing_cohort/` | +| 6 | Every cohort declares access + compute exposure | `CohortAccessClass`, `CohortComputeExposure`, `AccessClassRequired` | `invalid/cohort_missing_access/` | +| 7 | Aggregate counts reconcile exactly (`accept+reject+indeterminate == included`) | `CohortCountMismatch`, `AggregateCountReconcile` | `invalid/cohort_count_mismatch/` | +| 8 | Revoked/expired profiles cannot support new active rewards without migration | `RevokedProfileGate` | `invalid/revoked_profile_active_reward/` | +| 9 | Raw vs normalized digests distinct and both present when normalization occurs | `IdenticalNormalizationDigests`, `MissingNormalizationDigests` | `invalid/identical_normalization_digests/` | +| 10 | Env/policy/model/verifier/campaign versions consistent when cross-checked | `CrossArtifactVersionMismatch` | `invalid/cross_artifact_version_mismatch/` | +| 11 | Public protected-evidence retains commitments (no rationale content) | `RationaleCommitment` | `invalid/missing_rationale_commitment/` | +| 12 | Excluded/unadjudicated remain visible and count-matched | `ExcludedItemsVisible`, `UnadjudicatedItemsVisible` | `invalid/excluded_items_invisible/` | +| 13 | CI methods + parameters declared (no silent denominator invention) | `CIMethodsDeclared`, `CIParametersDeclared` | `invalid/missing_ci_method/` | +| 14 | Reward component totals reproduce under the declared composition rule | `RewardCompositionMismatch` | `invalid/reward_total_mismatch/` | +| 15 | Indeterminate never counted as accept/reject | `IndeterminateMisclassification` | `invalid/indeterminate_misclassification/` | + +Additional producer / schema gates (not in the 15-rule matrix but release-blocking): + +| Code | Meaning | Fixture | +|------|---------|---------| +| `FailClosedDecision` | Timeout / unavailable / error must not yield accept/reject | `invalid/timeout_accept/` | +| `InvalidSourceCommit` | `source_commit` must be 40-char lowercase hex | `invalid/short_source_commit/` | +| `ActiveRewardUnresolvedClaims` | Active envelopes cannot carry mandatory unresolved claims | `invalid/active_reward_unresolved/` | +| `additionalProperties` / type errors | Unknown fields and float rewards rejected | `invalid/unknown_field/`, `invalid/float_reward/` | + +Multi-file context is loaded from sibling files in a fixture directory (`profile.json`, +`result.json`, `results/*.json`, `campaign.json`) or passed explicitly to validators. +`pcs assurance build-report` is the reference multi-file executor; Rust/TS `verify` accept golden +report outputs and reject digest tampering (`ReportDigestMismatch`). diff --git a/docs/verifier-assurance/threat-model.md b/docs/verifier-assurance/threat-model.md new file mode 100644 index 00000000..32113eb9 --- /dev/null +++ b/docs/verifier-assurance/threat-model.md @@ -0,0 +1,15 @@ +# Verifier Assurance threat model (delta) + +Extends [../security-governance.md](../security-governance.md) and [../pf-core/threat-model.md](../pf-core/threat-model.md). + +| Threat | Mitigation | +|--------|------------| +| Forged accept after timeout | Fail-closed decision rule; fixture `invalid/timeout_accept/` | +| Silent float corruption of rates/rewards | Decimal strings + Canonical JSON float prohibition | +| Profile config drift without identity change | Material digests; config substitution tests | +| Private adjudication leak | Public records store commitment + location class only; content forbidden | +| Report denominator invention | Fail closed when cohort/adjudication inputs missing | +| Dialect shortcuts from producers | `benchmarks/verifier_assurance_conformance/` + suite gate | +| Schema fork / silent producer extension | Registry + `additionalProperties: false`; downstream sync notes | + +pcs-core does not execute environments, train models (including RL), run attacks, or store private partner rationale text. See [non-claims.md](non-claims.md). diff --git a/docs/versioning.md b/docs/versioning.md index 1272e3ed..55642c01 100644 --- a/docs/versioning.md +++ b/docs/versioning.md @@ -4,6 +4,8 @@ All PCS artifacts use `schema_version` value `"v0"` for the v0.1 release, and pinning git tag **`v0.1.0`** on pcs-core fixes schemas, registry, and validators at one commit. +Verifier Assurance artifacts use `schema_version` value `"v1"` (`common.defs.json#/$defs/schema_version_v1`) in new `*.v1` schema files. Frozen `*.v0` schemas remain immutable; `VerificationResult.v0` and `VerificationResult.v1` coexist without auto-upgrade (see [verifier-assurance/migration.md](verifier-assurance/migration.md)). + ## Change classes | Class | Description | Consumer impact | diff --git a/examples/verifier_assurance/README.md b/examples/verifier_assurance/README.md new file mode 100644 index 00000000..b02c81c7 --- /dev/null +++ b/examples/verifier_assurance/README.md @@ -0,0 +1,35 @@ +# Verifier Assurance examples + +Fixtures for the six release-grade PCS-VA artifact types. + +## Layout + +```text +examples/verifier_assurance/ + valid//... # positive cases (multi-file where needed) + invalid//manifest.json # expected_error + artifact_type + artifact.json # [+ optional profile.json / result(s) / campaign.json] + VerifierProfile.v1.valid.json # flat pin samples + VerificationResult.v1.valid.json + README.md +``` + +Invalid cases use `manifest.json` with `expected_error` set to a stable semantic +code (or a schema error substring). Multi-file rules load sibling context from +`profile.json`, `result.json`, `results/`, and/or `campaign.json`. + +## Commands + +```bash +pcs validate examples/verifier_assurance/VerifierProfile.v1.valid.json +pcs conformance run --suite verifier-assurance +``` + +See [docs/verifier-assurance/semantic-rules.md](../../docs/verifier-assurance/semantic-rules.md) +and [docs/verifier-assurance/protocol.md](../../docs/verifier-assurance/protocol.md). + +Regenerate nested fixtures after schema changes (maintainers): + +```bash +python scripts/gen_va_fixtures.py +``` diff --git a/examples/verifier_assurance/VerificationResult.v1.valid.json b/examples/verifier_assurance/VerificationResult.v1.valid.json new file mode 100644 index 00000000..706ea3f1 --- /dev/null +++ b/examples/verifier_assurance/VerificationResult.v1.valid.json @@ -0,0 +1,74 @@ +{ + "schema_version": "v1", + "artifact_type": "VerificationResult.v1", + "verification_result_id": "vr-opa-demo-001", + "created_at": "2026-07-24T16:00:05Z", + "producer": "OVK", + "producer_version": "0.4.0", + "source_repo": "https://github.com/SentinelOps-CI/open-verification-kernel", + "source_commit": "e068794683959c52a19594a6d271dd5e69f3c999", + "verifier_profile": { + "verifier_profile_id": "vp-opa-demo-001", + "profile_digest": "sha256:a4499695423937d2fedb86b5dfd21e65a053cc02c78c38f35e77ce8a98a42a44" + }, + "claim_ids": [ + "policy.allow" + ], + "trajectory_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "input_bundle_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "raw_backend_output_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "normalized_result_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "normalization_applied": true, + "normalizer_version": "ovk.normalize.v1", + "check_groups": [ + { + "kind": "process", + "checks": [ + { + "check_id": "opa-eval-exit", + "mandatory": true, + "status": "passed" + } + ] + }, + { + "kind": "authority", + "checks": [ + { + "check_id": "policy-allow", + "mandatory": true, + "status": "passed" + } + ] + } + ], + "resource_limits": { + "wall_time_ms": 120, + "memory_bytes": 67108864 + }, + "execution_status": "completed", + "decision": "accept", + "guarantee_class": "runtime_observed", + "declared_input_guarantee_class": "runtime_observed", + "invocation_ref": { + "invocation_id": "vi-opa-demo-001", + "invocation_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + }, + "assumptions": [ + { + "assumption_id": "a-policy-pin", + "statement": "Policy digest matches profile.configuration.policy_digest", + "critical": true + } + ], + "limits": [ + { + "limit_id": "l-no-formal", + "statement": "OPA eval is observational; not a formal proof." + } + ], + "integrity": { + "canonicalization_version": "v1", + "artifact_digest": "sha256:ec7d364835a69ee600857503a15099dd9d07be40bafd4ea8248fdd2d01fd7ac2" + } +} diff --git a/examples/verifier_assurance/VerifierInvocationRecord.v1.valid.json b/examples/verifier_assurance/VerifierInvocationRecord.v1.valid.json new file mode 100644 index 00000000..c7eb618b --- /dev/null +++ b/examples/verifier_assurance/VerifierInvocationRecord.v1.valid.json @@ -0,0 +1,53 @@ +{ + "artifact_type": "VerifierInvocationRecord.v1", + "assumptions": [], + "canonicalization_version": "v1", + "command": { + "argv": [ + "opa", + "eval", + "-d", + "policy.rego", + "data.allow" + ], + "cwd": ".", + "env_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "limits": { + "timeout_ms": 30000 + } + }, + "compiled_obligation": { + "digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "media_type": "application/json", + "path": "compiled_obligation.json" + }, + "completed_at": "2026-07-24T16:01:01Z", + "exit": { + "code": 0, + "kind": "exited" + }, + "guarantee_class": "runtime_observed", + "input_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "integrity": { + "artifact_digest": "sha256:a8761bfcdda71d47ac64d26513d378d258404f9042d4ad1a8816dcfc935a6ac1", + "canonicalization_version": "v1" + }, + "invocation_id": "vi-ovk-pin-001", + "limits": [], + "normalized_result_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "normalizer_version": "ovk.normalize.v1", + "producer": "OVK", + "producer_version": "0.4.0", + "profile_ref": { + "artifact_type": "VerifierProfile.v1", + "profile_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "verifier_profile_id": "vp-opa-demo-001" + }, + "raw_backend_result_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "schema_version": "v1", + "source_commit": "e068794683959c52a19594a6d271dd5e69f3c999", + "source_repo": "https://github.com/SentinelOps-CI/pcs-core", + "started_at": "2026-07-24T16:01:00Z", + "stderr_digest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + "stdout_digest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" +} diff --git a/examples/verifier_assurance/VerifierMutationManifest.v1.valid.json b/examples/verifier_assurance/VerifierMutationManifest.v1.valid.json new file mode 100644 index 00000000..4b2ba5cd --- /dev/null +++ b/examples/verifier_assurance/VerifierMutationManifest.v1.valid.json @@ -0,0 +1,32 @@ +{ + "artifact_type": "VerifierMutationManifest.v1", + "base_profile_ref": { + "artifact_type": "VerifierProfile.v1", + "profile_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "verifier_profile_id": "vp-opa-demo-001" + }, + "canonicalization_version": "v1", + "created_at": "2026-07-24T16:03:00Z", + "expected_effect": "Changes execution timeout.", + "integrity": { + "artifact_digest": "sha256:35d8c217a200611e57fdc6039018429aa27e1169717678bed7c205d1dd95f025", + "canonicalization_version": "v1" + }, + "mutated_profile_ref": { + "artifact_type": "VerifierProfile.v1", + "profile_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "verifier_profile_id": "vp-opa-demo-001-mut-alter_timeout" + }, + "mutation_class": "alter_timeout", + "mutation_id": "mut-alter_timeout-001", + "parameters": { + "timeout_ms": 1000 + }, + "producer": "OVK", + "producer_version": "0.4.0", + "production_prohibition": true, + "schema_version": "v1", + "source_commit": "e068794683959c52a19594a6d271dd5e69f3c999", + "source_repo": "https://github.com/SentinelOps-CI/pcs-core", + "supported_by_adapter": true +} diff --git a/examples/verifier_assurance/VerifierProfile.v1.valid.json b/examples/verifier_assurance/VerifierProfile.v1.valid.json new file mode 100644 index 00000000..e80854ee --- /dev/null +++ b/examples/verifier_assurance/VerifierProfile.v1.valid.json @@ -0,0 +1,110 @@ +{ + "schema_version": "v1", + "artifact_type": "VerifierProfile.v1", + "verifier_profile_id": "vp-opa-demo-001", + "created_at": "2026-07-24T16:00:00Z", + "producer": "OVK", + "producer_version": "0.4.0", + "source_repo": "https://github.com/SentinelOps-CI/open-verification-kernel", + "source_commit": "e068794683959c52a19594a6d271dd5e69f3c999", + "implementation": { + "name": "open-verification-kernel", + "version": "0.4.0", + "language": "python", + "entry_point": "ovk.adapters.self_protection.opa_adapter", + "implementation_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "container_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "dependency_lock_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "normalization_implementation_version": "ovk.normalize.v1" + }, + "configuration": { + "config_digest": "sha256:b79cf2b918a1d94c5537f3adccd6671484e5e18d16c681c2234625ab59a26253", + "policy_digest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + "model_digest": null, + "prompt_digest": null, + "resource_limit_digest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + "rubric_digest": null, + "test_suite_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "threshold_digest": null, + "ensemble_digest": null + }, + "mechanism": { + "mechanism_class": "policy_engine", + "determinism": "deterministic", + "allows_abstention": true, + "description": "OPA eval against pinned policy digest.", + "requires_external_execution": true + }, + "claim_surface": { + "supported_claim_ids": [ + "policy.allow" + ], + "guarantee_class": "runtime_observed", + "out_of_scope_claim_ids": [ + "formal.safety" + ], + "decision_space": [ + "accept", + "reject", + "indeterminate_execution_error", + "indeterminate_out_of_scope" + ] + }, + "applicability": { + "status": "active", + "valid_from": "2026-07-24T00:00:00Z", + "environment_families": [ + "ci", + "local" + ] + }, + "assumptions": [ + "opa binary digest matches implementation.container_digest or dependency pin" + ], + "known_blind_spots": [ + "Does not prove policy correctness; observational allow/deny only" + ], + "configuration_schema_digest": "sha256:453c4bd8f59a21a4207c4ab537f2edc9c3b952a9357b7c6006812229c831848d", + "configuration_schema": { + "type": "object", + "required": [ + "backend", + "rego_package", + "query" + ] + }, + "canonical_configuration": { + "backend": "opa-native", + "rego_package": "data.ovk.policy", + "query": "data.ovk.policy.allow" + }, + "redacted_environment": { + "redaction_policy_id": "ovk-redact-v1", + "entries": { + "OPA_LOG_LEVEL": "error", + "CI": "true" + }, + "redacted_keys": [ + "OPA_TOKEN", + "GITHUB_TOKEN" + ], + "environment_digest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" + }, + "external_dependencies": [ + { + "dependency_id": "opa", + "kind": "binary", + "identity": "opa", + "version": "0.68.0", + "digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } + ], + "execution_controls": { + "timeout_ms": 30000 + }, + "limitations_notice": "Profile binds configuration and digests only; it does not claim policy correctness beyond recorded checker results.", + "integrity": { + "canonicalization_version": "v1", + "artifact_digest": "sha256:a4499695423937d2fedb86b5dfd21e65a053cc02c78c38f35e77ce8a98a42a44" + } +} diff --git a/examples/verifier_assurance/VerifierReplayReport.v1.valid.json b/examples/verifier_assurance/VerifierReplayReport.v1.valid.json new file mode 100644 index 00000000..5f04df1b --- /dev/null +++ b/examples/verifier_assurance/VerifierReplayReport.v1.valid.json @@ -0,0 +1,36 @@ +{ + "artifact_type": "VerifierReplayReport.v1", + "canonicalization_version": "v1", + "created_at": "2026-07-24T16:02:00Z", + "determinism": "deterministic", + "drift": { + "claim_match": true, + "normalized_digest_match": true, + "raw_digest_match": true + }, + "integrity": { + "artifact_digest": "sha256:f2af1afd1ee217db70273ebce5d695af47bfbb66e975b4411b5860b3b79f85af", + "canonicalization_version": "v1" + }, + "original_invocation_ref": { + "artifact_type": "VerifierInvocationRecord.v1", + "invocation_digest": "sha256:a8761bfcdda71d47ac64d26513d378d258404f9042d4ad1a8816dcfc935a6ac1", + "invocation_id": "vi-ovk-pin-001" + }, + "original_normalized_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "original_raw_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "producer": "OVK", + "producer_version": "0.4.0", + "profile_ref": { + "artifact_type": "VerifierProfile.v1", + "profile_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "verifier_profile_id": "vp-opa-demo-001" + }, + "replay_id": "rr-ovk-pin-001", + "replay_normalized_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "replay_raw_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "replay_status": "matched", + "schema_version": "v1", + "source_commit": "e068794683959c52a19594a6d271dd5e69f3c999", + "source_repo": "https://github.com/SentinelOps-CI/pcs-core" +} diff --git a/examples/verifier_assurance/invalid/accept_mandatory_failure/artifact.json b/examples/verifier_assurance/invalid/accept_mandatory_failure/artifact.json new file mode 100644 index 00000000..82cf441e --- /dev/null +++ b/examples/verifier_assurance/invalid/accept_mandatory_failure/artifact.json @@ -0,0 +1,46 @@ +{ + "artifact_type": "VerificationResult.v1", + "check_groups": [ + { + "checks": [ + { + "check_id": "process.complete", + "mandatory": true, + "status": "failed" + } + ], + "kind": "process" + } + ], + "claim_ids": [ + "claim.safety.no_egress" + ], + "created_at": "2026-07-24T12:05:00Z", + "decision": "accept", + "execution_status": "completed", + "initial_state_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "input_bundle_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "integrity": { + "artifact_digest": "sha256:5a5e5701bc48315dbfde02dda4b363ac6f7de108388f275d7ea6c7f646732190", + "canonicalization_version": "v1" + }, + "normalization_applied": true, + "normalized_result_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "producer": "OVK", + "producer_version": "0.1.0", + "raw_backend_output_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "resource_limits": { + "memory_bytes": 1048576, + "wall_time_ms": 1000 + }, + "schema_version": "v1", + "source_commit": "e068794683959c52a19594a6d271dd5e69f3c999", + "source_repo": "https://github.com/SentinelOps-CI/pcs-core", + "terminal_state_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "trajectory_digest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + "verification_result_id": "vr-001", + "verifier_profile": { + "profile_digest": "sha256:c515b34f1f61d9b8d139d2a14a92409b2d45ffee16d16bb4c89913db1ebd53b0", + "verifier_profile_id": "vp-ovk-static-001" + } +} diff --git a/examples/verifier_assurance/invalid/accept_mandatory_failure/manifest.json b/examples/verifier_assurance/invalid/accept_mandatory_failure/manifest.json new file mode 100644 index 00000000..e3d00f58 --- /dev/null +++ b/examples/verifier_assurance/invalid/accept_mandatory_failure/manifest.json @@ -0,0 +1,5 @@ +{ + "artifact_file": "artifact.json", + "artifact_type": "VerificationResult.v1", + "expected_error": "AcceptWithMandatoryFailure" +} diff --git a/examples/verifier_assurance/invalid/active_reward_unresolved/artifact.json b/examples/verifier_assurance/invalid/active_reward_unresolved/artifact.json new file mode 100644 index 00000000..3aebf277 --- /dev/null +++ b/examples/verifier_assurance/invalid/active_reward_unresolved/artifact.json @@ -0,0 +1,64 @@ +{ + "artifact_type": "RewardEvidenceEnvelope.v1", + "authority": { + "authority_class": "producer", + "authority_id": "labtrust-reward" + }, + "claims_issued": [ + "claim.safety.no_egress" + ], + "claims_rejected": [], + "claims_unresolved": [], + "components": [ + { + "claim_id": "claim.safety.no_egress", + "component_id": "safety", + "guarantee_class": "certificate_checked", + "value": "1.0" + }, + { + "component_id": "quality", + "guarantee_class": "empirically_measured", + "value": "0.5" + } + ], + "composition_function": "sum", + "created_at": "2026-07-24T12:10:00Z", + "env_profile_id": "env-lab", + "env_profile_version": "1.0.0", + "episode_id": "ep-1", + "integrity": { + "artifact_digest": "sha256:5631bd636274efb492a4da598a581b0155aa4774846c481ca27b945855a498ba", + "canonicalization_version": "v1" + }, + "lifecycle": { + "status": "active" + }, + "mandatory_unresolved_claim_ids": [ + "claim.safety.no_egress" + ], + "producer": "LabTrust-Gym", + "producer_version": "0.1.0", + "profile_refs": [ + { + "applicability_status": "active", + "profile_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "verifier_profile_id": "vp-ovk-static-001" + } + ], + "reward_envelope_id": "rew-001", + "scalar_total": "1.5", + "schema_version": "v1", + "source_commit": "e068794683959c52a19594a6d271dd5e69f3c999", + "source_repo": "https://github.com/SentinelOps-CI/pcs-core", + "state_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "step_index": 0, + "trajectory_digest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + "verifier_result_refs": [ + { + "artifact_digest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + "artifact_id": "vr-001", + "artifact_type": "VerificationResult.v1" + } + ] +} diff --git a/examples/verifier_assurance/invalid/active_reward_unresolved/manifest.json b/examples/verifier_assurance/invalid/active_reward_unresolved/manifest.json new file mode 100644 index 00000000..be29a044 --- /dev/null +++ b/examples/verifier_assurance/invalid/active_reward_unresolved/manifest.json @@ -0,0 +1,5 @@ +{ + "artifact_file": "artifact.json", + "artifact_type": "RewardEvidenceEnvelope.v1", + "expected_error": "ActiveRewardUnresolvedClaims" +} diff --git a/examples/verifier_assurance/invalid/cohort_count_mismatch/artifact.json b/examples/verifier_assurance/invalid/cohort_count_mismatch/artifact.json new file mode 100644 index 00000000..74979325 --- /dev/null +++ b/examples/verifier_assurance/invalid/cohort_count_mismatch/artifact.json @@ -0,0 +1,192 @@ +{ + "applicability_limits": [ + "synthetic fixture only" + ], + "artifact_type": "VerifierAssuranceReport.v1", + "campaign_ref": { + "artifact_digest": "sha256:fd790a71e8eeefcd40d6b2b3b1642b3f93b93669d518564a62fc4f9f2d93ff8f", + "artifact_id": "camp-001", + "artifact_type": "OptimizationCampaignManifest.v1" + }, + "cohorts": [ + { + "accept_count": 99, + "access_class": "black_box", + "cohort_id": "cohort-ordinary", + "cohort_kind": "ordinary", + "compute_exposure": { + "accounting_method": "wall_clock", + "query_count": 50, + "wall_time_seconds": 600 + }, + "included_result_count": 1, + "indeterminate_count": 0, + "reject_count": 0 + }, + { + "accept_count": 0, + "access_class": "adaptive", + "cohort_id": "cohort-optimized", + "cohort_kind": "optimized", + "compute_exposure": { + "accounting_method": "wall_clock", + "query_count": 950, + "wall_time_seconds": 3000 + }, + "included_result_count": 1, + "indeterminate_count": 0, + "reject_count": 1 + } + ], + "created_at": "2026-07-24T15:00:00Z", + "excluded_items": [ + { + "item_id": "ex-1", + "reason_code": "out_of_scope" + } + ], + "independent_adjudication": true, + "integrity": { + "artifact_digest": "sha256:f40022df20f8b795149fa42e91d11ef35abdaac5e7ae40b945d617c5a053fe2e", + "canonicalization_version": "v1" + }, + "metrics": { + "abstention_rate": { + "confidence_interval": { + "lower": "0.000000", + "method": "wilson", + "parameters": { + "alpha": "0.05", + "declared": true + }, + "sample_size": 2, + "upper": "0.000000" + }, + "denominator": 2, + "numerator": 0, + "rate": "0.000000" + }, + "adjudication_coverage": { + "confidence_interval": { + "lower": "1.000000", + "method": "wilson", + "parameters": { + "alpha": "0.05", + "declared": true + }, + "sample_size": 2, + "upper": "1.000000" + }, + "denominator": 2, + "numerator": 2, + "rate": "1.000000" + }, + "cost_decimal": "0", + "excluded_count": 1, + "exploit_family_counts": {}, + "false_accept_rate": { + "confidence_interval": { + "lower": "0.000000", + "method": "wilson", + "parameters": { + "alpha": "0.05", + "declared": true + }, + "sample_size": 2, + "upper": "0.000000" + }, + "denominator": 2, + "numerator": 0, + "rate": "0.000000" + }, + "false_reject_rate": { + "confidence_interval": { + "lower": "0.000000", + "method": "wilson", + "parameters": { + "alpha": "0.05", + "declared": true + }, + "sample_size": 2, + "upper": "0.000000" + }, + "denominator": 2, + "numerator": 0, + "rate": "0.000000" + }, + "inter_verifier_disagreement_rate": { + "confidence_interval": { + "lower": "0.000000", + "method": "wilson", + "parameters": { + "alpha": "0.05", + "declared": true + }, + "sample_size": 2, + "upper": "0.000000" + }, + "denominator": 2, + "numerator": 0, + "rate": "0.000000" + }, + "latency_ms_p50": 0, + "optimization_gap": "-1.000000", + "optimized_accept_rate": { + "confidence_interval": { + "lower": "0.000000", + "method": "wilson", + "parameters": { + "alpha": "0.05", + "declared": true + }, + "sample_size": 1, + "upper": "0.000000" + }, + "denominator": 1, + "numerator": 0, + "rate": "0.000000" + }, + "ordinary_accept_rate": { + "confidence_interval": { + "lower": "1.000000", + "method": "wilson", + "parameters": { + "alpha": "0.05", + "declared": true + }, + "sample_size": 1, + "upper": "1.000000" + }, + "denominator": 1, + "numerator": 1, + "rate": "1.000000" + }, + "queries_to_first_exploit": null, + "reward_invalidation_rate": { + "confidence_interval": { + "lower": "0.000000", + "method": "wilson", + "parameters": { + "alpha": "0.05", + "declared": true + }, + "sample_size": 2, + "upper": "0.000000" + }, + "denominator": 2, + "numerator": 0, + "rate": "0.000000" + }, + "sample_size": 2, + "time_to_first_exploit_seconds": null, + "unadjudicated_count": 0 + }, + "producer": "pcs-core", + "producer_version": "0.1.0", + "release_grade": true, + "report_id": "rep-001", + "schema_version": "v1", + "source_commit": "e068794683959c52a19594a6d271dd5e69f3c999", + "source_repo": "https://github.com/SentinelOps-CI/pcs-core", + "unadjudicated_items": [] +} diff --git a/examples/verifier_assurance/invalid/cohort_count_mismatch/manifest.json b/examples/verifier_assurance/invalid/cohort_count_mismatch/manifest.json new file mode 100644 index 00000000..665d2d94 --- /dev/null +++ b/examples/verifier_assurance/invalid/cohort_count_mismatch/manifest.json @@ -0,0 +1,5 @@ +{ + "artifact_file": "artifact.json", + "artifact_type": "VerifierAssuranceReport.v1", + "expected_error": "CohortCountMismatch" +} diff --git a/examples/verifier_assurance/invalid/cohort_missing_access/artifact.json b/examples/verifier_assurance/invalid/cohort_missing_access/artifact.json new file mode 100644 index 00000000..db3b5f6c --- /dev/null +++ b/examples/verifier_assurance/invalid/cohort_missing_access/artifact.json @@ -0,0 +1,68 @@ +{ + "access_class": "black_box", + "algorithm": "random-search", + "artifact_type": "OptimizationCampaignManifest.v1", + "campaign_id": "camp-001", + "checkpoint_ids": [ + "ckpt-0" + ], + "cohorts": [ + { + "cohort_id": "cohort-ordinary", + "cohort_kind": "ordinary", + "compute_exposure": { + "accounting_method": "wall_clock", + "query_count": 50, + "wall_time_seconds": 600 + } + } + ], + "compute_budget": { + "accounting_method": "wall_clock", + "gpu_hours": "0", + "query_budget": 1000, + "wall_time_seconds": 3600 + }, + "containment_policy": "offline-only", + "created_at": "2026-07-24T11:00:00Z", + "disclosure_policy": "public-summary", + "env_profile_id": "env-lab", + "env_profile_version": "1.0.0", + "episode_count": 10, + "harness_id": "harness-a", + "integrity": { + "artifact_digest": "sha256:b4e187d91adca8deb1a7f53441430a1fc0682015b6d8f43ee756eadfdab8d4a8", + "canonicalization_version": "v1" + }, + "model_id": "model-a", + "optimizer_settings": { + "batch_size": 8, + "learning_rate": "0.01" + }, + "policy_id": "policy-a", + "producer": "pcs-core", + "producer_version": "0.1.0", + "query_count": 100, + "schema_version": "v1", + "seeds": [ + 1, + 2, + 3 + ], + "source_commit": "e068794683959c52a19594a6d271dd5e69f3c999", + "source_repo": "https://github.com/SentinelOps-CI/pcs-core", + "splits": { + "holdout": "hold-v1", + "train": "train-v1", + "validation": "val-v1" + }, + "target_verifier_profile": { + "profile_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "verifier_profile_id": "vp-ovk-static-001" + }, + "trajectory_count": 10, + "visibility": { + "full_traces_retained": false, + "public_summary": true + } +} diff --git a/examples/verifier_assurance/invalid/cohort_missing_access/manifest.json b/examples/verifier_assurance/invalid/cohort_missing_access/manifest.json new file mode 100644 index 00000000..1cd3186e --- /dev/null +++ b/examples/verifier_assurance/invalid/cohort_missing_access/manifest.json @@ -0,0 +1,5 @@ +{ + "artifact_file": "artifact.json", + "artifact_type": "OptimizationCampaignManifest.v1", + "expected_error": "CohortAccessClass" +} diff --git a/examples/verifier_assurance/invalid/cross_artifact_version_mismatch/artifact.json b/examples/verifier_assurance/invalid/cross_artifact_version_mismatch/artifact.json new file mode 100644 index 00000000..ee1ed186 --- /dev/null +++ b/examples/verifier_assurance/invalid/cross_artifact_version_mismatch/artifact.json @@ -0,0 +1,62 @@ +{ + "artifact_type": "RewardEvidenceEnvelope.v1", + "authority": { + "authority_class": "producer", + "authority_id": "labtrust-reward" + }, + "claims_issued": [ + "claim.safety.no_egress" + ], + "claims_rejected": [], + "claims_unresolved": [], + "components": [ + { + "claim_id": "claim.safety.no_egress", + "component_id": "safety", + "guarantee_class": "certificate_checked", + "value": "1.0" + }, + { + "component_id": "quality", + "guarantee_class": "empirically_measured", + "value": "0.5" + } + ], + "composition_function": "sum", + "created_at": "2026-07-24T12:10:00Z", + "env_profile_id": "env-other", + "env_profile_version": "9.9.9", + "episode_id": "ep-1", + "integrity": { + "artifact_digest": "sha256:2a65b811af09c4bd341fa3b29ee649cd228fa8a67b48d5e347f02e6354aafbd4", + "canonicalization_version": "v1" + }, + "lifecycle": { + "status": "active" + }, + "mandatory_unresolved_claim_ids": [], + "producer": "LabTrust-Gym", + "producer_version": "0.1.0", + "profile_refs": [ + { + "applicability_status": "active", + "profile_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "verifier_profile_id": "vp-ovk-static-001" + } + ], + "reward_envelope_id": "rew-001", + "scalar_total": "1.5", + "schema_version": "v1", + "source_commit": "e068794683959c52a19594a6d271dd5e69f3c999", + "source_repo": "https://github.com/SentinelOps-CI/pcs-core", + "state_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "step_index": 0, + "trajectory_digest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + "verifier_result_refs": [ + { + "artifact_digest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + "artifact_id": "vr-001", + "artifact_type": "VerificationResult.v1" + } + ] +} diff --git a/examples/verifier_assurance/invalid/cross_artifact_version_mismatch/campaign.json b/examples/verifier_assurance/invalid/cross_artifact_version_mismatch/campaign.json new file mode 100644 index 00000000..d96f6a8b --- /dev/null +++ b/examples/verifier_assurance/invalid/cross_artifact_version_mismatch/campaign.json @@ -0,0 +1,79 @@ +{ + "access_class": "black_box", + "algorithm": "random-search", + "artifact_type": "OptimizationCampaignManifest.v1", + "campaign_id": "camp-001", + "checkpoint_ids": [ + "ckpt-0" + ], + "cohorts": [ + { + "access_class": "black_box", + "cohort_id": "cohort-ordinary", + "cohort_kind": "ordinary", + "compute_exposure": { + "accounting_method": "wall_clock", + "query_count": 50, + "wall_time_seconds": 600 + } + }, + { + "access_class": "adaptive", + "cohort_id": "cohort-optimized", + "cohort_kind": "optimized", + "compute_exposure": { + "accounting_method": "wall_clock", + "query_count": 950, + "wall_time_seconds": 3000 + } + } + ], + "compute_budget": { + "accounting_method": "wall_clock", + "gpu_hours": "0", + "query_budget": 1000, + "wall_time_seconds": 3600 + }, + "containment_policy": "offline-only", + "created_at": "2026-07-24T11:00:00Z", + "disclosure_policy": "public-summary", + "env_profile_id": "env-lab", + "env_profile_version": "1.0.0", + "episode_count": 10, + "harness_id": "harness-a", + "integrity": { + "artifact_digest": "sha256:1adebaaa9854973c44977fb1bcf22c4792f827623804be8ac67279189811d4b8", + "canonicalization_version": "v1" + }, + "model_id": "model-a", + "optimizer_settings": { + "batch_size": 8, + "learning_rate": "0.01" + }, + "policy_id": "policy-a", + "producer": "pcs-core", + "producer_version": "0.1.0", + "query_count": 100, + "schema_version": "v1", + "seeds": [ + 1, + 2, + 3 + ], + "source_commit": "e068794683959c52a19594a6d271dd5e69f3c999", + "source_repo": "https://github.com/SentinelOps-CI/pcs-core", + "splits": { + "holdout": "hold-v1", + "train": "train-v1", + "validation": "val-v1" + }, + "target_verifier_profile": { + "profile_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "verifier_profile_id": "vp-ovk-static-001" + }, + "trajectory_count": 10, + "visibility": { + "full_traces_retained": false, + "public_summary": true + } +} diff --git a/examples/verifier_assurance/invalid/cross_artifact_version_mismatch/manifest.json b/examples/verifier_assurance/invalid/cross_artifact_version_mismatch/manifest.json new file mode 100644 index 00000000..a1cc662f --- /dev/null +++ b/examples/verifier_assurance/invalid/cross_artifact_version_mismatch/manifest.json @@ -0,0 +1,5 @@ +{ + "artifact_file": "artifact.json", + "artifact_type": "RewardEvidenceEnvelope.v1", + "expected_error": "CrossArtifactVersionMismatch" +} diff --git a/examples/verifier_assurance/invalid/cross_artifact_version_mismatch/result.json b/examples/verifier_assurance/invalid/cross_artifact_version_mismatch/result.json new file mode 100644 index 00000000..b4e5675e --- /dev/null +++ b/examples/verifier_assurance/invalid/cross_artifact_version_mismatch/result.json @@ -0,0 +1,56 @@ +{ + "artifact_type": "VerificationResult.v1", + "check_groups": [ + { + "checks": [ + { + "check_id": "process.complete", + "mandatory": true, + "status": "passed" + } + ], + "kind": "process" + }, + { + "checks": [ + { + "check_id": "evidence.present", + "mandatory": true, + "status": "passed" + } + ], + "kind": "evidence" + } + ], + "claim_ids": [ + "claim.safety.no_egress" + ], + "created_at": "2026-07-24T12:05:00Z", + "decision": "accept", + "execution_status": "completed", + "initial_state_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "input_bundle_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "integrity": { + "artifact_digest": "sha256:7fa5afbf3d931d448c724fb0542101f7c8bc5f14cd19456c3344d3d8d23a0ad2", + "canonicalization_version": "v1" + }, + "normalization_applied": true, + "normalized_result_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "producer": "OVK", + "producer_version": "0.1.0", + "raw_backend_output_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "resource_limits": { + "memory_bytes": 1048576, + "wall_time_ms": 1000 + }, + "schema_version": "v1", + "source_commit": "e068794683959c52a19594a6d271dd5e69f3c999", + "source_repo": "https://github.com/SentinelOps-CI/pcs-core", + "terminal_state_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "trajectory_digest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + "verification_result_id": "vr-001", + "verifier_profile": { + "profile_digest": "sha256:c515b34f1f61d9b8d139d2a14a92409b2d45ffee16d16bb4c89913db1ebd53b0", + "verifier_profile_id": "vp-ovk-static-001" + } +} diff --git a/examples/verifier_assurance/invalid/excluded_items_invisible/artifact.json b/examples/verifier_assurance/invalid/excluded_items_invisible/artifact.json new file mode 100644 index 00000000..76ba6eb9 --- /dev/null +++ b/examples/verifier_assurance/invalid/excluded_items_invisible/artifact.json @@ -0,0 +1,187 @@ +{ + "applicability_limits": [ + "synthetic fixture only" + ], + "artifact_type": "VerifierAssuranceReport.v1", + "campaign_ref": { + "artifact_digest": "sha256:fd790a71e8eeefcd40d6b2b3b1642b3f93b93669d518564a62fc4f9f2d93ff8f", + "artifact_id": "camp-001", + "artifact_type": "OptimizationCampaignManifest.v1" + }, + "cohorts": [ + { + "accept_count": 1, + "access_class": "black_box", + "cohort_id": "cohort-ordinary", + "cohort_kind": "ordinary", + "compute_exposure": { + "accounting_method": "wall_clock", + "query_count": 50, + "wall_time_seconds": 600 + }, + "included_result_count": 1, + "indeterminate_count": 0, + "reject_count": 0 + }, + { + "accept_count": 0, + "access_class": "adaptive", + "cohort_id": "cohort-optimized", + "cohort_kind": "optimized", + "compute_exposure": { + "accounting_method": "wall_clock", + "query_count": 950, + "wall_time_seconds": 3000 + }, + "included_result_count": 1, + "indeterminate_count": 0, + "reject_count": 1 + } + ], + "created_at": "2026-07-24T15:00:00Z", + "excluded_items": [], + "independent_adjudication": true, + "integrity": { + "artifact_digest": "sha256:9999f1ed6d18a900cc2cce39d8bb97a22b05693065c99dc8c5059bf5f9e098af", + "canonicalization_version": "v1" + }, + "metrics": { + "abstention_rate": { + "confidence_interval": { + "lower": "0.000000", + "method": "wilson", + "parameters": { + "alpha": "0.05", + "declared": true + }, + "sample_size": 2, + "upper": "0.000000" + }, + "denominator": 2, + "numerator": 0, + "rate": "0.000000" + }, + "adjudication_coverage": { + "confidence_interval": { + "lower": "1.000000", + "method": "wilson", + "parameters": { + "alpha": "0.05", + "declared": true + }, + "sample_size": 2, + "upper": "1.000000" + }, + "denominator": 2, + "numerator": 2, + "rate": "1.000000" + }, + "cost_decimal": "0", + "excluded_count": 3, + "exploit_family_counts": {}, + "false_accept_rate": { + "confidence_interval": { + "lower": "0.000000", + "method": "wilson", + "parameters": { + "alpha": "0.05", + "declared": true + }, + "sample_size": 2, + "upper": "0.000000" + }, + "denominator": 2, + "numerator": 0, + "rate": "0.000000" + }, + "false_reject_rate": { + "confidence_interval": { + "lower": "0.000000", + "method": "wilson", + "parameters": { + "alpha": "0.05", + "declared": true + }, + "sample_size": 2, + "upper": "0.000000" + }, + "denominator": 2, + "numerator": 0, + "rate": "0.000000" + }, + "inter_verifier_disagreement_rate": { + "confidence_interval": { + "lower": "0.000000", + "method": "wilson", + "parameters": { + "alpha": "0.05", + "declared": true + }, + "sample_size": 2, + "upper": "0.000000" + }, + "denominator": 2, + "numerator": 0, + "rate": "0.000000" + }, + "latency_ms_p50": 0, + "optimization_gap": "-1.000000", + "optimized_accept_rate": { + "confidence_interval": { + "lower": "0.000000", + "method": "wilson", + "parameters": { + "alpha": "0.05", + "declared": true + }, + "sample_size": 1, + "upper": "0.000000" + }, + "denominator": 1, + "numerator": 0, + "rate": "0.000000" + }, + "ordinary_accept_rate": { + "confidence_interval": { + "lower": "1.000000", + "method": "wilson", + "parameters": { + "alpha": "0.05", + "declared": true + }, + "sample_size": 1, + "upper": "1.000000" + }, + "denominator": 1, + "numerator": 1, + "rate": "1.000000" + }, + "queries_to_first_exploit": null, + "reward_invalidation_rate": { + "confidence_interval": { + "lower": "0.000000", + "method": "wilson", + "parameters": { + "alpha": "0.05", + "declared": true + }, + "sample_size": 2, + "upper": "0.000000" + }, + "denominator": 2, + "numerator": 0, + "rate": "0.000000" + }, + "sample_size": 2, + "time_to_first_exploit_seconds": null, + "unadjudicated_count": 0 + }, + "producer": "pcs-core", + "producer_version": "0.1.0", + "release_grade": true, + "report_id": "rep-001", + "schema_version": "v1", + "source_commit": "e068794683959c52a19594a6d271dd5e69f3c999", + "source_repo": "https://github.com/SentinelOps-CI/pcs-core", + "unadjudicated_items": [] +} diff --git a/examples/verifier_assurance/invalid/excluded_items_invisible/manifest.json b/examples/verifier_assurance/invalid/excluded_items_invisible/manifest.json new file mode 100644 index 00000000..24725b50 --- /dev/null +++ b/examples/verifier_assurance/invalid/excluded_items_invisible/manifest.json @@ -0,0 +1,5 @@ +{ + "artifact_file": "artifact.json", + "artifact_type": "VerifierAssuranceReport.v1", + "expected_error": "ExcludedItemsVisible" +} diff --git a/examples/verifier_assurance/invalid/float_reward/artifact.json b/examples/verifier_assurance/invalid/float_reward/artifact.json new file mode 100644 index 00000000..a90ad97f --- /dev/null +++ b/examples/verifier_assurance/invalid/float_reward/artifact.json @@ -0,0 +1,58 @@ +{ + "artifact_type": "RewardEvidenceEnvelope.v1", + "authority": { + "authority_class": "producer", + "authority_id": "labtrust-reward" + }, + "claims_issued": [ + "claim.safety.no_egress" + ], + "claims_rejected": [], + "claims_unresolved": [], + "components": [ + { + "claim_id": "claim.safety.no_egress", + "component_id": "safety", + "guarantee_class": "certificate_checked", + "value": "1.0" + }, + { + "component_id": "quality", + "guarantee_class": "empirically_measured", + "value": "0.5" + } + ], + "composition_function": "sum", + "created_at": "2026-07-24T12:10:00Z", + "env_profile_id": "env-lab", + "env_profile_version": "1.0.0", + "episode_id": "ep-1", + "lifecycle": { + "status": "active" + }, + "mandatory_unresolved_claim_ids": [], + "producer": "LabTrust-Gym", + "producer_version": "0.1.0", + "profile_refs": [ + { + "applicability_status": "active", + "profile_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "verifier_profile_id": "vp-ovk-static-001" + } + ], + "reward_envelope_id": "rew-001", + "scalar_total": 1.5, + "schema_version": "v1", + "source_commit": "e068794683959c52a19594a6d271dd5e69f3c999", + "source_repo": "https://github.com/SentinelOps-CI/pcs-core", + "state_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "step_index": 0, + "trajectory_digest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + "verifier_result_refs": [ + { + "artifact_digest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + "artifact_id": "vr-001", + "artifact_type": "VerificationResult.v1" + } + ] +} diff --git a/examples/verifier_assurance/invalid/float_reward/manifest.json b/examples/verifier_assurance/invalid/float_reward/manifest.json new file mode 100644 index 00000000..a00b43b9 --- /dev/null +++ b/examples/verifier_assurance/invalid/float_reward/manifest.json @@ -0,0 +1,5 @@ +{ + "artifact_file": "artifact.json", + "artifact_type": "RewardEvidenceEnvelope.v1", + "expected_error": "is not of type 'string'" +} diff --git a/examples/verifier_assurance/invalid/identical_normalization_digests/artifact.json b/examples/verifier_assurance/invalid/identical_normalization_digests/artifact.json new file mode 100644 index 00000000..3bcad410 --- /dev/null +++ b/examples/verifier_assurance/invalid/identical_normalization_digests/artifact.json @@ -0,0 +1,56 @@ +{ + "artifact_type": "VerificationResult.v1", + "check_groups": [ + { + "checks": [ + { + "check_id": "process.complete", + "mandatory": true, + "status": "passed" + } + ], + "kind": "process" + }, + { + "checks": [ + { + "check_id": "evidence.present", + "mandatory": true, + "status": "passed" + } + ], + "kind": "evidence" + } + ], + "claim_ids": [ + "claim.safety.no_egress" + ], + "created_at": "2026-07-24T12:05:00Z", + "decision": "accept", + "execution_status": "completed", + "initial_state_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "input_bundle_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "integrity": { + "artifact_digest": "sha256:22f58cbeb33fc9d2c93bdce81bc5a475834bb777099e880bada261d0251f6a22", + "canonicalization_version": "v1" + }, + "normalization_applied": true, + "normalized_result_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "producer": "OVK", + "producer_version": "0.1.0", + "raw_backend_output_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "resource_limits": { + "memory_bytes": 1048576, + "wall_time_ms": 1000 + }, + "schema_version": "v1", + "source_commit": "e068794683959c52a19594a6d271dd5e69f3c999", + "source_repo": "https://github.com/SentinelOps-CI/pcs-core", + "terminal_state_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "trajectory_digest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + "verification_result_id": "vr-001", + "verifier_profile": { + "profile_digest": "sha256:c515b34f1f61d9b8d139d2a14a92409b2d45ffee16d16bb4c89913db1ebd53b0", + "verifier_profile_id": "vp-ovk-static-001" + } +} diff --git a/examples/verifier_assurance/invalid/identical_normalization_digests/manifest.json b/examples/verifier_assurance/invalid/identical_normalization_digests/manifest.json new file mode 100644 index 00000000..5344e6c5 --- /dev/null +++ b/examples/verifier_assurance/invalid/identical_normalization_digests/manifest.json @@ -0,0 +1,5 @@ +{ + "artifact_file": "artifact.json", + "artifact_type": "VerificationResult.v1", + "expected_error": "IdenticalNormalizationDigests" +} diff --git a/examples/verifier_assurance/invalid/indeterminate_misclassification/artifact.json b/examples/verifier_assurance/invalid/indeterminate_misclassification/artifact.json new file mode 100644 index 00000000..8164b6c1 --- /dev/null +++ b/examples/verifier_assurance/invalid/indeterminate_misclassification/artifact.json @@ -0,0 +1,192 @@ +{ + "applicability_limits": [ + "synthetic fixture only" + ], + "artifact_type": "VerifierAssuranceReport.v1", + "campaign_ref": { + "artifact_digest": "sha256:fd790a71e8eeefcd40d6b2b3b1642b3f93b93669d518564a62fc4f9f2d93ff8f", + "artifact_id": "camp-001", + "artifact_type": "OptimizationCampaignManifest.v1" + }, + "cohorts": [ + { + "accept_count": 2, + "access_class": "black_box", + "cohort_id": "cohort-ordinary", + "cohort_kind": "ordinary", + "compute_exposure": { + "accounting_method": "wall_clock", + "query_count": 50, + "wall_time_seconds": 600 + }, + "included_result_count": 1, + "indeterminate_count": -1, + "reject_count": 0 + }, + { + "accept_count": 0, + "access_class": "adaptive", + "cohort_id": "cohort-optimized", + "cohort_kind": "optimized", + "compute_exposure": { + "accounting_method": "wall_clock", + "query_count": 950, + "wall_time_seconds": 3000 + }, + "included_result_count": 1, + "indeterminate_count": 0, + "reject_count": 1 + } + ], + "created_at": "2026-07-24T15:00:00Z", + "excluded_items": [ + { + "item_id": "ex-1", + "reason_code": "out_of_scope" + } + ], + "independent_adjudication": true, + "integrity": { + "artifact_digest": "sha256:d1fc2d194c513250a381e77d5c7089ca458f791a5318811d7fe6fd36aa915656", + "canonicalization_version": "v1" + }, + "metrics": { + "abstention_rate": { + "confidence_interval": { + "lower": "0.000000", + "method": "wilson", + "parameters": { + "alpha": "0.05", + "declared": true + }, + "sample_size": 2, + "upper": "0.000000" + }, + "denominator": 2, + "numerator": 0, + "rate": "0.000000" + }, + "adjudication_coverage": { + "confidence_interval": { + "lower": "1.000000", + "method": "wilson", + "parameters": { + "alpha": "0.05", + "declared": true + }, + "sample_size": 2, + "upper": "1.000000" + }, + "denominator": 2, + "numerator": 2, + "rate": "1.000000" + }, + "cost_decimal": "0", + "excluded_count": 1, + "exploit_family_counts": {}, + "false_accept_rate": { + "confidence_interval": { + "lower": "0.000000", + "method": "wilson", + "parameters": { + "alpha": "0.05", + "declared": true + }, + "sample_size": 2, + "upper": "0.000000" + }, + "denominator": 2, + "numerator": 0, + "rate": "0.000000" + }, + "false_reject_rate": { + "confidence_interval": { + "lower": "0.000000", + "method": "wilson", + "parameters": { + "alpha": "0.05", + "declared": true + }, + "sample_size": 2, + "upper": "0.000000" + }, + "denominator": 2, + "numerator": 0, + "rate": "0.000000" + }, + "inter_verifier_disagreement_rate": { + "confidence_interval": { + "lower": "0.000000", + "method": "wilson", + "parameters": { + "alpha": "0.05", + "declared": true + }, + "sample_size": 2, + "upper": "0.000000" + }, + "denominator": 2, + "numerator": 0, + "rate": "0.000000" + }, + "latency_ms_p50": 0, + "optimization_gap": "-1.000000", + "optimized_accept_rate": { + "confidence_interval": { + "lower": "0.000000", + "method": "wilson", + "parameters": { + "alpha": "0.05", + "declared": true + }, + "sample_size": 1, + "upper": "0.000000" + }, + "denominator": 1, + "numerator": 0, + "rate": "0.000000" + }, + "ordinary_accept_rate": { + "confidence_interval": { + "lower": "1.000000", + "method": "wilson", + "parameters": { + "alpha": "0.05", + "declared": true + }, + "sample_size": 1, + "upper": "1.000000" + }, + "denominator": 1, + "numerator": 1, + "rate": "1.000000" + }, + "queries_to_first_exploit": null, + "reward_invalidation_rate": { + "confidence_interval": { + "lower": "0.000000", + "method": "wilson", + "parameters": { + "alpha": "0.05", + "declared": true + }, + "sample_size": 2, + "upper": "0.000000" + }, + "denominator": 2, + "numerator": 0, + "rate": "0.000000" + }, + "sample_size": 2, + "time_to_first_exploit_seconds": null, + "unadjudicated_count": 0 + }, + "producer": "pcs-core", + "producer_version": "0.1.0", + "release_grade": true, + "report_id": "rep-001", + "schema_version": "v1", + "source_commit": "e068794683959c52a19594a6d271dd5e69f3c999", + "source_repo": "https://github.com/SentinelOps-CI/pcs-core", + "unadjudicated_items": [] +} diff --git a/examples/verifier_assurance/invalid/indeterminate_misclassification/manifest.json b/examples/verifier_assurance/invalid/indeterminate_misclassification/manifest.json new file mode 100644 index 00000000..e8f2fbc1 --- /dev/null +++ b/examples/verifier_assurance/invalid/indeterminate_misclassification/manifest.json @@ -0,0 +1,5 @@ +{ + "artifact_file": "artifact.json", + "artifact_type": "VerifierAssuranceReport.v1", + "expected_error": "IndeterminateMisclassification" +} diff --git a/examples/verifier_assurance/invalid/missing_ci_method/artifact.json b/examples/verifier_assurance/invalid/missing_ci_method/artifact.json new file mode 100644 index 00000000..8c6a51ed --- /dev/null +++ b/examples/verifier_assurance/invalid/missing_ci_method/artifact.json @@ -0,0 +1,190 @@ +{ + "applicability_limits": [ + "synthetic fixture only" + ], + "artifact_type": "VerifierAssuranceReport.v1", + "campaign_ref": { + "artifact_digest": "sha256:fd790a71e8eeefcd40d6b2b3b1642b3f93b93669d518564a62fc4f9f2d93ff8f", + "artifact_id": "camp-001", + "artifact_type": "OptimizationCampaignManifest.v1" + }, + "cohorts": [ + { + "accept_count": 1, + "access_class": "black_box", + "cohort_id": "cohort-ordinary", + "cohort_kind": "ordinary", + "compute_exposure": { + "accounting_method": "wall_clock", + "query_count": 50, + "wall_time_seconds": 600 + }, + "included_result_count": 1, + "indeterminate_count": 0, + "reject_count": 0 + }, + { + "accept_count": 0, + "access_class": "adaptive", + "cohort_id": "cohort-optimized", + "cohort_kind": "optimized", + "compute_exposure": { + "accounting_method": "wall_clock", + "query_count": 950, + "wall_time_seconds": 3000 + }, + "included_result_count": 1, + "indeterminate_count": 0, + "reject_count": 1 + } + ], + "created_at": "2026-07-24T15:00:00Z", + "excluded_items": [ + { + "item_id": "ex-1", + "reason_code": "out_of_scope" + } + ], + "independent_adjudication": true, + "integrity": { + "artifact_digest": "sha256:e43da9e8d7807bc3be18339447a6914cd3cd200d878f4b9ae4ca1fb130fb146b", + "canonicalization_version": "v1" + }, + "metrics": { + "abstention_rate": { + "confidence_interval": { + "lower": "0.000000", + "method": "wilson", + "parameters": { + "alpha": "0.05", + "declared": true + }, + "sample_size": 2, + "upper": "0.000000" + }, + "denominator": 2, + "numerator": 0, + "rate": "0.000000" + }, + "adjudication_coverage": { + "confidence_interval": { + "lower": "1.000000", + "method": "wilson", + "parameters": { + "alpha": "0.05", + "declared": true + }, + "sample_size": 2, + "upper": "1.000000" + }, + "denominator": 2, + "numerator": 2, + "rate": "1.000000" + }, + "cost_decimal": "0", + "excluded_count": 1, + "exploit_family_counts": {}, + "false_accept_rate": { + "confidence_interval": { + "lower": "0", + "parameters": { + "alpha": "0.05" + }, + "sample_size": 2, + "upper": "0" + }, + "denominator": 2, + "numerator": 0, + "rate": "0.000000" + }, + "false_reject_rate": { + "confidence_interval": { + "lower": "0.000000", + "method": "wilson", + "parameters": { + "alpha": "0.05", + "declared": true + }, + "sample_size": 2, + "upper": "0.000000" + }, + "denominator": 2, + "numerator": 0, + "rate": "0.000000" + }, + "inter_verifier_disagreement_rate": { + "confidence_interval": { + "lower": "0.000000", + "method": "wilson", + "parameters": { + "alpha": "0.05", + "declared": true + }, + "sample_size": 2, + "upper": "0.000000" + }, + "denominator": 2, + "numerator": 0, + "rate": "0.000000" + }, + "latency_ms_p50": 0, + "optimization_gap": "-1.000000", + "optimized_accept_rate": { + "confidence_interval": { + "lower": "0.000000", + "method": "wilson", + "parameters": { + "alpha": "0.05", + "declared": true + }, + "sample_size": 1, + "upper": "0.000000" + }, + "denominator": 1, + "numerator": 0, + "rate": "0.000000" + }, + "ordinary_accept_rate": { + "confidence_interval": { + "lower": "1.000000", + "method": "wilson", + "parameters": { + "alpha": "0.05", + "declared": true + }, + "sample_size": 1, + "upper": "1.000000" + }, + "denominator": 1, + "numerator": 1, + "rate": "1.000000" + }, + "queries_to_first_exploit": null, + "reward_invalidation_rate": { + "confidence_interval": { + "lower": "0.000000", + "method": "wilson", + "parameters": { + "alpha": "0.05", + "declared": true + }, + "sample_size": 2, + "upper": "0.000000" + }, + "denominator": 2, + "numerator": 0, + "rate": "0.000000" + }, + "sample_size": 2, + "time_to_first_exploit_seconds": null, + "unadjudicated_count": 0 + }, + "producer": "pcs-core", + "producer_version": "0.1.0", + "release_grade": true, + "report_id": "rep-001", + "schema_version": "v1", + "source_commit": "e068794683959c52a19594a6d271dd5e69f3c999", + "source_repo": "https://github.com/SentinelOps-CI/pcs-core", + "unadjudicated_items": [] +} diff --git a/examples/verifier_assurance/invalid/missing_ci_method/manifest.json b/examples/verifier_assurance/invalid/missing_ci_method/manifest.json new file mode 100644 index 00000000..f4525aee --- /dev/null +++ b/examples/verifier_assurance/invalid/missing_ci_method/manifest.json @@ -0,0 +1,5 @@ +{ + "artifact_file": "artifact.json", + "artifact_type": "VerifierAssuranceReport.v1", + "expected_error": "CIMethodsDeclared" +} diff --git a/examples/verifier_assurance/invalid/missing_rationale_commitment/artifact.json b/examples/verifier_assurance/invalid/missing_rationale_commitment/artifact.json new file mode 100644 index 00000000..99e57ef5 --- /dev/null +++ b/examples/verifier_assurance/invalid/missing_rationale_commitment/artifact.json @@ -0,0 +1,39 @@ +{ + "adjudication_id": "adj-001", + "artifact_type": "AdjudicationRecord.v1", + "blinded": true, + "conflict_of_interest_declared": false, + "created_at": "2026-07-24T13:00:00Z", + "disagreement": false, + "escalation": false, + "evidence_refs": [], + "independence_declared": true, + "integrity": { + "artifact_digest": "sha256:a77af889efdf22215e539ce0b30ad1403a9f8e46e975db57d45aa142934e28d6", + "canonicalization_version": "v1" + }, + "label": "valid", + "label_released_at": "2026-07-24T14:00:00Z", + "producer": "pcs-core", + "producer_version": "0.1.0", + "protected_rationale": { + "location_class": "partner_vault" + }, + "protocol_id": "adj-protocol", + "protocol_version": "1.0.0", + "schema_version": "v1", + "source_commit": "e068794683959c52a19594a6d271dd5e69f3c999", + "source_repo": "https://github.com/SentinelOps-CI/pcs-core", + "subject": { + "artifact_digest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + "artifact_id": "vr-001", + "artifact_type": "VerificationResult.v1" + }, + "votes": [ + { + "role": "primary", + "vote": "valid", + "voter_pseudonym_id": "voter-a" + } + ] +} diff --git a/examples/verifier_assurance/invalid/missing_rationale_commitment/manifest.json b/examples/verifier_assurance/invalid/missing_rationale_commitment/manifest.json new file mode 100644 index 00000000..45546f68 --- /dev/null +++ b/examples/verifier_assurance/invalid/missing_rationale_commitment/manifest.json @@ -0,0 +1,5 @@ +{ + "artifact_file": "artifact.json", + "artifact_type": "AdjudicationRecord.v1", + "expected_error": "RationaleCommitment" +} diff --git a/examples/verifier_assurance/invalid/mutation_production_overwrite/artifact.json b/examples/verifier_assurance/invalid/mutation_production_overwrite/artifact.json new file mode 100644 index 00000000..1b052d46 --- /dev/null +++ b/examples/verifier_assurance/invalid/mutation_production_overwrite/artifact.json @@ -0,0 +1,32 @@ +{ + "artifact_type": "VerifierMutationManifest.v1", + "base_profile_ref": { + "artifact_type": "VerifierProfile.v1", + "profile_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "verifier_profile_id": "vp-opa-demo-001" + }, + "canonicalization_version": "v1", + "created_at": "2026-07-24T16:03:00Z", + "expected_effect": "Changes execution timeout.", + "integrity": { + "artifact_digest": "sha256:365ce59b91b13df1770c6a6ce251e21e725959b9aee2dc1a980ad97997a2be1b", + "canonicalization_version": "v1" + }, + "mutated_profile_ref": { + "artifact_type": "VerifierProfile.v1", + "profile_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "verifier_profile_id": "vp-opa-demo-001-mut-alter_timeout" + }, + "mutation_class": "alter_timeout", + "mutation_id": "mut-alter_timeout-001", + "parameters": { + "timeout_ms": 1000 + }, + "producer": "OVK", + "producer_version": "0.4.0", + "production_prohibition": false, + "schema_version": "v1", + "source_commit": "e068794683959c52a19594a6d271dd5e69f3c999", + "source_repo": "https://github.com/SentinelOps-CI/pcs-core", + "supported_by_adapter": true +} diff --git a/examples/verifier_assurance/invalid/mutation_production_overwrite/manifest.json b/examples/verifier_assurance/invalid/mutation_production_overwrite/manifest.json new file mode 100644 index 00000000..1ed0f095 --- /dev/null +++ b/examples/verifier_assurance/invalid/mutation_production_overwrite/manifest.json @@ -0,0 +1,5 @@ +{ + "artifact_file": "artifact.json", + "artifact_type": "VerifierMutationManifest.v1", + "expected_error": "production_prohibition" +} diff --git a/examples/verifier_assurance/invalid/optimization_gap_missing_cohort/artifact.json b/examples/verifier_assurance/invalid/optimization_gap_missing_cohort/artifact.json new file mode 100644 index 00000000..2add7550 --- /dev/null +++ b/examples/verifier_assurance/invalid/optimization_gap_missing_cohort/artifact.json @@ -0,0 +1,178 @@ +{ + "applicability_limits": [ + "synthetic fixture only" + ], + "artifact_type": "VerifierAssuranceReport.v1", + "campaign_ref": { + "artifact_digest": "sha256:fd790a71e8eeefcd40d6b2b3b1642b3f93b93669d518564a62fc4f9f2d93ff8f", + "artifact_id": "camp-001", + "artifact_type": "OptimizationCampaignManifest.v1" + }, + "cohorts": [ + { + "accept_count": 1, + "access_class": "black_box", + "cohort_id": "cohort-ordinary", + "cohort_kind": "ordinary", + "compute_exposure": { + "accounting_method": "wall_clock", + "query_count": 50, + "wall_time_seconds": 600 + }, + "included_result_count": 1, + "indeterminate_count": 0, + "reject_count": 0 + } + ], + "created_at": "2026-07-24T15:00:00Z", + "excluded_items": [ + { + "item_id": "ex-1", + "reason_code": "out_of_scope" + } + ], + "independent_adjudication": true, + "integrity": { + "artifact_digest": "sha256:35623bba542885d5dbfb5500f61951ccaf2787fd18d3bdc87623e0f6181a9f70", + "canonicalization_version": "v1" + }, + "metrics": { + "abstention_rate": { + "confidence_interval": { + "lower": "0.000000", + "method": "wilson", + "parameters": { + "alpha": "0.05", + "declared": true + }, + "sample_size": 2, + "upper": "0.000000" + }, + "denominator": 2, + "numerator": 0, + "rate": "0.000000" + }, + "adjudication_coverage": { + "confidence_interval": { + "lower": "1.000000", + "method": "wilson", + "parameters": { + "alpha": "0.05", + "declared": true + }, + "sample_size": 2, + "upper": "1.000000" + }, + "denominator": 2, + "numerator": 2, + "rate": "1.000000" + }, + "cost_decimal": "0", + "excluded_count": 1, + "exploit_family_counts": {}, + "false_accept_rate": { + "confidence_interval": { + "lower": "0.000000", + "method": "wilson", + "parameters": { + "alpha": "0.05", + "declared": true + }, + "sample_size": 2, + "upper": "0.000000" + }, + "denominator": 2, + "numerator": 0, + "rate": "0.000000" + }, + "false_reject_rate": { + "confidence_interval": { + "lower": "0.000000", + "method": "wilson", + "parameters": { + "alpha": "0.05", + "declared": true + }, + "sample_size": 2, + "upper": "0.000000" + }, + "denominator": 2, + "numerator": 0, + "rate": "0.000000" + }, + "inter_verifier_disagreement_rate": { + "confidence_interval": { + "lower": "0.000000", + "method": "wilson", + "parameters": { + "alpha": "0.05", + "declared": true + }, + "sample_size": 2, + "upper": "0.000000" + }, + "denominator": 2, + "numerator": 0, + "rate": "0.000000" + }, + "latency_ms_p50": 0, + "optimization_gap": "0.500000", + "optimized_accept_rate": { + "confidence_interval": { + "lower": "0.000000", + "method": "wilson", + "parameters": { + "alpha": "0.05", + "declared": true + }, + "sample_size": 1, + "upper": "0.000000" + }, + "denominator": 1, + "numerator": 0, + "rate": "0.000000" + }, + "ordinary_accept_rate": { + "confidence_interval": { + "lower": "1.000000", + "method": "wilson", + "parameters": { + "alpha": "0.05", + "declared": true + }, + "sample_size": 1, + "upper": "1.000000" + }, + "denominator": 1, + "numerator": 1, + "rate": "1.000000" + }, + "queries_to_first_exploit": null, + "reward_invalidation_rate": { + "confidence_interval": { + "lower": "0.000000", + "method": "wilson", + "parameters": { + "alpha": "0.05", + "declared": true + }, + "sample_size": 2, + "upper": "0.000000" + }, + "denominator": 2, + "numerator": 0, + "rate": "0.000000" + }, + "sample_size": 2, + "time_to_first_exploit_seconds": null, + "unadjudicated_count": 0 + }, + "producer": "pcs-core", + "producer_version": "0.1.0", + "release_grade": true, + "report_id": "rep-001", + "schema_version": "v1", + "source_commit": "e068794683959c52a19594a6d271dd5e69f3c999", + "source_repo": "https://github.com/SentinelOps-CI/pcs-core", + "unadjudicated_items": [] +} diff --git a/examples/verifier_assurance/invalid/optimization_gap_missing_cohort/manifest.json b/examples/verifier_assurance/invalid/optimization_gap_missing_cohort/manifest.json new file mode 100644 index 00000000..1e26b250 --- /dev/null +++ b/examples/verifier_assurance/invalid/optimization_gap_missing_cohort/manifest.json @@ -0,0 +1,5 @@ +{ + "artifact_file": "artifact.json", + "artifact_type": "VerifierAssuranceReport.v1", + "expected_error": "OptimizationGapCohorts" +} diff --git a/examples/verifier_assurance/invalid/profile_digest_mismatch/artifact.json b/examples/verifier_assurance/invalid/profile_digest_mismatch/artifact.json new file mode 100644 index 00000000..788b8289 --- /dev/null +++ b/examples/verifier_assurance/invalid/profile_digest_mismatch/artifact.json @@ -0,0 +1,56 @@ +{ + "artifact_type": "VerificationResult.v1", + "check_groups": [ + { + "checks": [ + { + "check_id": "process.complete", + "mandatory": true, + "status": "passed" + } + ], + "kind": "process" + }, + { + "checks": [ + { + "check_id": "evidence.present", + "mandatory": true, + "status": "passed" + } + ], + "kind": "evidence" + } + ], + "claim_ids": [ + "claim.safety.no_egress" + ], + "created_at": "2026-07-24T12:05:00Z", + "decision": "accept", + "execution_status": "completed", + "initial_state_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "input_bundle_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "integrity": { + "artifact_digest": "sha256:9a9a217add1e2f6857cc4d1cb6b81e68dcb6b86dc24f96c92e6878dcf36d540a", + "canonicalization_version": "v1" + }, + "normalization_applied": true, + "normalized_result_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "producer": "OVK", + "producer_version": "0.1.0", + "raw_backend_output_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "resource_limits": { + "memory_bytes": 1048576, + "wall_time_ms": 1000 + }, + "schema_version": "v1", + "source_commit": "e068794683959c52a19594a6d271dd5e69f3c999", + "source_repo": "https://github.com/SentinelOps-CI/pcs-core", + "terminal_state_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "trajectory_digest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + "verification_result_id": "vr-001", + "verifier_profile": { + "profile_digest": "sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "verifier_profile_id": "vp-ovk-static-001" + } +} diff --git a/examples/verifier_assurance/invalid/profile_digest_mismatch/manifest.json b/examples/verifier_assurance/invalid/profile_digest_mismatch/manifest.json new file mode 100644 index 00000000..e670ab08 --- /dev/null +++ b/examples/verifier_assurance/invalid/profile_digest_mismatch/manifest.json @@ -0,0 +1,5 @@ +{ + "artifact_file": "artifact.json", + "artifact_type": "VerificationResult.v1", + "expected_error": "ProfileDigestMismatch" +} diff --git a/examples/verifier_assurance/invalid/profile_digest_mismatch/profile.json b/examples/verifier_assurance/invalid/profile_digest_mismatch/profile.json new file mode 100644 index 00000000..81e4d685 --- /dev/null +++ b/examples/verifier_assurance/invalid/profile_digest_mismatch/profile.json @@ -0,0 +1,52 @@ +{ + "applicability": { + "status": "active", + "valid_from": "2026-07-01T00:00:00Z" + }, + "artifact_type": "VerifierProfile.v1", + "assumptions": [ + "host OS is Linux" + ], + "claim_surface": { + "guarantee_class": "certificate_checked", + "out_of_scope_claim_ids": [ + "claim.formal.full_correctness" + ], + "supported_claim_ids": [ + "claim.safety.no_egress" + ] + }, + "configuration": { + "config_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "model_digest": null, + "policy_digest": null, + "prompt_digest": null, + "resource_limit_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + }, + "created_at": "2026-07-24T12:00:00Z", + "implementation": { + "implementation_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "language": "rust", + "name": "ovk-static-checker", + "version": "1.0.0" + }, + "integrity": { + "artifact_digest": "sha256:3af8ff33a41e4a1af25cd184b4f610f7d80da942100fb01667419195b3c5ff91", + "canonicalization_version": "v1" + }, + "known_blind_spots": [ + "does not observe runtime network" + ], + "mechanism": { + "allows_abstention": true, + "description": "Static claim checker", + "determinism": "deterministic", + "mechanism_class": "static_analysis" + }, + "producer": "OVK", + "producer_version": "0.1.0", + "schema_version": "v1", + "source_commit": "e068794683959c52a19594a6d271dd5e69f3c999", + "source_repo": "https://github.com/SentinelOps-CI/pcs-core", + "verifier_profile_id": "vp-ovk-static-001" +} diff --git a/examples/verifier_assurance/invalid/release_grade_no_adjudication/artifact.json b/examples/verifier_assurance/invalid/release_grade_no_adjudication/artifact.json new file mode 100644 index 00000000..fa931ff4 --- /dev/null +++ b/examples/verifier_assurance/invalid/release_grade_no_adjudication/artifact.json @@ -0,0 +1,192 @@ +{ + "applicability_limits": [ + "synthetic fixture only" + ], + "artifact_type": "VerifierAssuranceReport.v1", + "campaign_ref": { + "artifact_digest": "sha256:fd790a71e8eeefcd40d6b2b3b1642b3f93b93669d518564a62fc4f9f2d93ff8f", + "artifact_id": "camp-001", + "artifact_type": "OptimizationCampaignManifest.v1" + }, + "cohorts": [ + { + "accept_count": 1, + "access_class": "black_box", + "cohort_id": "cohort-ordinary", + "cohort_kind": "ordinary", + "compute_exposure": { + "accounting_method": "wall_clock", + "query_count": 50, + "wall_time_seconds": 600 + }, + "included_result_count": 1, + "indeterminate_count": 0, + "reject_count": 0 + }, + { + "accept_count": 0, + "access_class": "adaptive", + "cohort_id": "cohort-optimized", + "cohort_kind": "optimized", + "compute_exposure": { + "accounting_method": "wall_clock", + "query_count": 950, + "wall_time_seconds": 3000 + }, + "included_result_count": 1, + "indeterminate_count": 0, + "reject_count": 1 + } + ], + "created_at": "2026-07-24T15:00:00Z", + "excluded_items": [ + { + "item_id": "ex-1", + "reason_code": "out_of_scope" + } + ], + "independent_adjudication": false, + "integrity": { + "artifact_digest": "sha256:f19dec11c2a6931dd74f567b62327aa8f6c1d47db4e3f6718df0e1c1f78a8af4", + "canonicalization_version": "v1" + }, + "metrics": { + "abstention_rate": { + "confidence_interval": { + "lower": "0.000000", + "method": "wilson", + "parameters": { + "alpha": "0.05", + "declared": true + }, + "sample_size": 2, + "upper": "0.000000" + }, + "denominator": 2, + "numerator": 0, + "rate": "0.000000" + }, + "adjudication_coverage": { + "confidence_interval": { + "lower": "1.000000", + "method": "wilson", + "parameters": { + "alpha": "0.05", + "declared": true + }, + "sample_size": 2, + "upper": "1.000000" + }, + "denominator": 2, + "numerator": 2, + "rate": "1.000000" + }, + "cost_decimal": "0", + "excluded_count": 1, + "exploit_family_counts": {}, + "false_accept_rate": { + "confidence_interval": { + "lower": "0.000000", + "method": "wilson", + "parameters": { + "alpha": "0.05", + "declared": true + }, + "sample_size": 2, + "upper": "0.000000" + }, + "denominator": 2, + "numerator": 0, + "rate": "0.000000" + }, + "false_reject_rate": { + "confidence_interval": { + "lower": "0.000000", + "method": "wilson", + "parameters": { + "alpha": "0.05", + "declared": true + }, + "sample_size": 2, + "upper": "0.000000" + }, + "denominator": 2, + "numerator": 0, + "rate": "0.000000" + }, + "inter_verifier_disagreement_rate": { + "confidence_interval": { + "lower": "0.000000", + "method": "wilson", + "parameters": { + "alpha": "0.05", + "declared": true + }, + "sample_size": 2, + "upper": "0.000000" + }, + "denominator": 2, + "numerator": 0, + "rate": "0.000000" + }, + "latency_ms_p50": 0, + "optimization_gap": "-1.000000", + "optimized_accept_rate": { + "confidence_interval": { + "lower": "0.000000", + "method": "wilson", + "parameters": { + "alpha": "0.05", + "declared": true + }, + "sample_size": 1, + "upper": "0.000000" + }, + "denominator": 1, + "numerator": 0, + "rate": "0.000000" + }, + "ordinary_accept_rate": { + "confidence_interval": { + "lower": "1.000000", + "method": "wilson", + "parameters": { + "alpha": "0.05", + "declared": true + }, + "sample_size": 1, + "upper": "1.000000" + }, + "denominator": 1, + "numerator": 1, + "rate": "1.000000" + }, + "queries_to_first_exploit": null, + "reward_invalidation_rate": { + "confidence_interval": { + "lower": "0.000000", + "method": "wilson", + "parameters": { + "alpha": "0.05", + "declared": true + }, + "sample_size": 2, + "upper": "0.000000" + }, + "denominator": 2, + "numerator": 0, + "rate": "0.000000" + }, + "sample_size": 2, + "time_to_first_exploit_seconds": null, + "unadjudicated_count": 0 + }, + "producer": "pcs-core", + "producer_version": "0.1.0", + "release_grade": true, + "report_id": "rep-001", + "schema_version": "v1", + "source_commit": "e068794683959c52a19594a6d271dd5e69f3c999", + "source_repo": "https://github.com/SentinelOps-CI/pcs-core", + "unadjudicated_items": [] +} diff --git a/examples/verifier_assurance/invalid/release_grade_no_adjudication/manifest.json b/examples/verifier_assurance/invalid/release_grade_no_adjudication/manifest.json new file mode 100644 index 00000000..c124f89d --- /dev/null +++ b/examples/verifier_assurance/invalid/release_grade_no_adjudication/manifest.json @@ -0,0 +1,5 @@ +{ + "artifact_file": "artifact.json", + "artifact_type": "VerifierAssuranceReport.v1", + "expected_error": "ReleaseGradeAdjudication" +} diff --git a/examples/verifier_assurance/invalid/replay_matched_with_drift/artifact.json b/examples/verifier_assurance/invalid/replay_matched_with_drift/artifact.json new file mode 100644 index 00000000..e802bd75 --- /dev/null +++ b/examples/verifier_assurance/invalid/replay_matched_with_drift/artifact.json @@ -0,0 +1,36 @@ +{ + "artifact_type": "VerifierReplayReport.v1", + "canonicalization_version": "v1", + "created_at": "2026-07-24T16:02:00Z", + "determinism": "deterministic", + "drift": { + "claim_match": false, + "normalized_digest_match": false, + "raw_digest_match": false + }, + "integrity": { + "artifact_digest": "sha256:3f52c7f923c65c32e04905432b7c41ddb7c98dddc3569d8d22e33aa151eeb51b", + "canonicalization_version": "v1" + }, + "original_invocation_ref": { + "artifact_type": "VerifierInvocationRecord.v1", + "invocation_digest": "sha256:a8761bfcdda71d47ac64d26513d378d258404f9042d4ad1a8816dcfc935a6ac1", + "invocation_id": "vi-ovk-pin-001" + }, + "original_normalized_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "original_raw_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "producer": "OVK", + "producer_version": "0.4.0", + "profile_ref": { + "artifact_type": "VerifierProfile.v1", + "profile_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "verifier_profile_id": "vp-opa-demo-001" + }, + "replay_id": "rr-ovk-pin-001", + "replay_normalized_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "replay_raw_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "replay_status": "matched", + "schema_version": "v1", + "source_commit": "e068794683959c52a19594a6d271dd5e69f3c999", + "source_repo": "https://github.com/SentinelOps-CI/pcs-core" +} diff --git a/examples/verifier_assurance/invalid/replay_matched_with_drift/manifest.json b/examples/verifier_assurance/invalid/replay_matched_with_drift/manifest.json new file mode 100644 index 00000000..ffbbfd8d --- /dev/null +++ b/examples/verifier_assurance/invalid/replay_matched_with_drift/manifest.json @@ -0,0 +1,5 @@ +{ + "artifact_file": "artifact.json", + "artifact_type": "VerifierReplayReport.v1", + "expected_error": "ReplayMatchedWithDrift" +} diff --git a/examples/verifier_assurance/invalid/revoked_profile_active_reward/artifact.json b/examples/verifier_assurance/invalid/revoked_profile_active_reward/artifact.json new file mode 100644 index 00000000..e740018d --- /dev/null +++ b/examples/verifier_assurance/invalid/revoked_profile_active_reward/artifact.json @@ -0,0 +1,62 @@ +{ + "artifact_type": "RewardEvidenceEnvelope.v1", + "authority": { + "authority_class": "producer", + "authority_id": "labtrust-reward" + }, + "claims_issued": [ + "claim.safety.no_egress" + ], + "claims_rejected": [], + "claims_unresolved": [], + "components": [ + { + "claim_id": "claim.safety.no_egress", + "component_id": "safety", + "guarantee_class": "certificate_checked", + "value": "1.0" + }, + { + "component_id": "quality", + "guarantee_class": "empirically_measured", + "value": "0.5" + } + ], + "composition_function": "sum", + "created_at": "2026-07-24T12:10:00Z", + "env_profile_id": "env-lab", + "env_profile_version": "1.0.0", + "episode_id": "ep-1", + "integrity": { + "artifact_digest": "sha256:192be3bb4348a1d1c8e0d0358fad022bf54a0409a41f3e369f91deec9ea68479", + "canonicalization_version": "v1" + }, + "lifecycle": { + "status": "active" + }, + "mandatory_unresolved_claim_ids": [], + "producer": "LabTrust-Gym", + "producer_version": "0.1.0", + "profile_refs": [ + { + "applicability_status": "revoked", + "profile_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "verifier_profile_id": "vp-ovk-static-001" + } + ], + "reward_envelope_id": "rew-001", + "scalar_total": "1.5", + "schema_version": "v1", + "source_commit": "e068794683959c52a19594a6d271dd5e69f3c999", + "source_repo": "https://github.com/SentinelOps-CI/pcs-core", + "state_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "step_index": 0, + "trajectory_digest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + "verifier_result_refs": [ + { + "artifact_digest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + "artifact_id": "vr-001", + "artifact_type": "VerificationResult.v1" + } + ] +} diff --git a/examples/verifier_assurance/invalid/revoked_profile_active_reward/manifest.json b/examples/verifier_assurance/invalid/revoked_profile_active_reward/manifest.json new file mode 100644 index 00000000..540a2238 --- /dev/null +++ b/examples/verifier_assurance/invalid/revoked_profile_active_reward/manifest.json @@ -0,0 +1,5 @@ +{ + "artifact_file": "artifact.json", + "artifact_type": "RewardEvidenceEnvelope.v1", + "expected_error": "RevokedProfileGate" +} diff --git a/examples/verifier_assurance/invalid/reward_total_mismatch/artifact.json b/examples/verifier_assurance/invalid/reward_total_mismatch/artifact.json new file mode 100644 index 00000000..912693e9 --- /dev/null +++ b/examples/verifier_assurance/invalid/reward_total_mismatch/artifact.json @@ -0,0 +1,62 @@ +{ + "artifact_type": "RewardEvidenceEnvelope.v1", + "authority": { + "authority_class": "producer", + "authority_id": "labtrust-reward" + }, + "claims_issued": [ + "claim.safety.no_egress" + ], + "claims_rejected": [], + "claims_unresolved": [], + "components": [ + { + "claim_id": "claim.safety.no_egress", + "component_id": "safety", + "guarantee_class": "certificate_checked", + "value": "1.0" + }, + { + "component_id": "quality", + "guarantee_class": "empirically_measured", + "value": "0.5" + } + ], + "composition_function": "sum", + "created_at": "2026-07-24T12:10:00Z", + "env_profile_id": "env-lab", + "env_profile_version": "1.0.0", + "episode_id": "ep-1", + "integrity": { + "artifact_digest": "sha256:73c730808c16ef0c62c920a61d0a1e03744a41c5e3cf5d1a9fea494296890ae8", + "canonicalization_version": "v1" + }, + "lifecycle": { + "status": "active" + }, + "mandatory_unresolved_claim_ids": [], + "producer": "LabTrust-Gym", + "producer_version": "0.1.0", + "profile_refs": [ + { + "applicability_status": "active", + "profile_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "verifier_profile_id": "vp-ovk-static-001" + } + ], + "reward_envelope_id": "rew-001", + "scalar_total": "9.9", + "schema_version": "v1", + "source_commit": "e068794683959c52a19594a6d271dd5e69f3c999", + "source_repo": "https://github.com/SentinelOps-CI/pcs-core", + "state_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "step_index": 0, + "trajectory_digest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + "verifier_result_refs": [ + { + "artifact_digest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + "artifact_id": "vr-001", + "artifact_type": "VerificationResult.v1" + } + ] +} diff --git a/examples/verifier_assurance/invalid/reward_total_mismatch/manifest.json b/examples/verifier_assurance/invalid/reward_total_mismatch/manifest.json new file mode 100644 index 00000000..308a5a11 --- /dev/null +++ b/examples/verifier_assurance/invalid/reward_total_mismatch/manifest.json @@ -0,0 +1,5 @@ +{ + "artifact_file": "artifact.json", + "artifact_type": "RewardEvidenceEnvelope.v1", + "expected_error": "RewardCompositionMismatch" +} diff --git a/examples/verifier_assurance/invalid/reward_trajectory_mismatch/artifact.json b/examples/verifier_assurance/invalid/reward_trajectory_mismatch/artifact.json new file mode 100644 index 00000000..53fd64bd --- /dev/null +++ b/examples/verifier_assurance/invalid/reward_trajectory_mismatch/artifact.json @@ -0,0 +1,62 @@ +{ + "artifact_type": "RewardEvidenceEnvelope.v1", + "authority": { + "authority_class": "producer", + "authority_id": "labtrust-reward" + }, + "claims_issued": [ + "claim.safety.no_egress" + ], + "claims_rejected": [], + "claims_unresolved": [], + "components": [ + { + "claim_id": "claim.safety.no_egress", + "component_id": "safety", + "guarantee_class": "certificate_checked", + "value": "1.0" + }, + { + "component_id": "quality", + "guarantee_class": "empirically_measured", + "value": "0.5" + } + ], + "composition_function": "sum", + "created_at": "2026-07-24T12:10:00Z", + "env_profile_id": "env-lab", + "env_profile_version": "1.0.0", + "episode_id": "ep-1", + "integrity": { + "artifact_digest": "sha256:273e8032bb1587723199b242d58539d2d2e4784b3d582dc486e3305003301f6c", + "canonicalization_version": "v1" + }, + "lifecycle": { + "status": "active" + }, + "mandatory_unresolved_claim_ids": [], + "producer": "LabTrust-Gym", + "producer_version": "0.1.0", + "profile_refs": [ + { + "applicability_status": "active", + "profile_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "verifier_profile_id": "vp-ovk-static-001" + } + ], + "reward_envelope_id": "rew-001", + "scalar_total": "1.5", + "schema_version": "v1", + "source_commit": "e068794683959c52a19594a6d271dd5e69f3c999", + "source_repo": "https://github.com/SentinelOps-CI/pcs-core", + "state_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "step_index": 0, + "trajectory_digest": "sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "verifier_result_refs": [ + { + "artifact_digest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + "artifact_id": "vr-001", + "artifact_type": "VerificationResult.v1" + } + ] +} diff --git a/examples/verifier_assurance/invalid/reward_trajectory_mismatch/manifest.json b/examples/verifier_assurance/invalid/reward_trajectory_mismatch/manifest.json new file mode 100644 index 00000000..b51b36d9 --- /dev/null +++ b/examples/verifier_assurance/invalid/reward_trajectory_mismatch/manifest.json @@ -0,0 +1,5 @@ +{ + "artifact_file": "artifact.json", + "artifact_type": "RewardEvidenceEnvelope.v1", + "expected_error": "RewardTrajectoryMismatch" +} diff --git a/examples/verifier_assurance/invalid/reward_trajectory_mismatch/result.json b/examples/verifier_assurance/invalid/reward_trajectory_mismatch/result.json new file mode 100644 index 00000000..b4e5675e --- /dev/null +++ b/examples/verifier_assurance/invalid/reward_trajectory_mismatch/result.json @@ -0,0 +1,56 @@ +{ + "artifact_type": "VerificationResult.v1", + "check_groups": [ + { + "checks": [ + { + "check_id": "process.complete", + "mandatory": true, + "status": "passed" + } + ], + "kind": "process" + }, + { + "checks": [ + { + "check_id": "evidence.present", + "mandatory": true, + "status": "passed" + } + ], + "kind": "evidence" + } + ], + "claim_ids": [ + "claim.safety.no_egress" + ], + "created_at": "2026-07-24T12:05:00Z", + "decision": "accept", + "execution_status": "completed", + "initial_state_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "input_bundle_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "integrity": { + "artifact_digest": "sha256:7fa5afbf3d931d448c724fb0542101f7c8bc5f14cd19456c3344d3d8d23a0ad2", + "canonicalization_version": "v1" + }, + "normalization_applied": true, + "normalized_result_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "producer": "OVK", + "producer_version": "0.1.0", + "raw_backend_output_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "resource_limits": { + "memory_bytes": 1048576, + "wall_time_ms": 1000 + }, + "schema_version": "v1", + "source_commit": "e068794683959c52a19594a6d271dd5e69f3c999", + "source_repo": "https://github.com/SentinelOps-CI/pcs-core", + "terminal_state_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "trajectory_digest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + "verification_result_id": "vr-001", + "verifier_profile": { + "profile_digest": "sha256:c515b34f1f61d9b8d139d2a14a92409b2d45ffee16d16bb4c89913db1ebd53b0", + "verifier_profile_id": "vp-ovk-static-001" + } +} diff --git a/examples/verifier_assurance/invalid/short_source_commit/artifact.json b/examples/verifier_assurance/invalid/short_source_commit/artifact.json new file mode 100644 index 00000000..ec780719 --- /dev/null +++ b/examples/verifier_assurance/invalid/short_source_commit/artifact.json @@ -0,0 +1,52 @@ +{ + "applicability": { + "status": "active", + "valid_from": "2026-07-01T00:00:00Z" + }, + "artifact_type": "VerifierProfile.v1", + "assumptions": [ + "host OS is Linux" + ], + "claim_surface": { + "guarantee_class": "certificate_checked", + "out_of_scope_claim_ids": [ + "claim.formal.full_correctness" + ], + "supported_claim_ids": [ + "claim.safety.no_egress" + ] + }, + "configuration": { + "config_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "model_digest": null, + "policy_digest": null, + "prompt_digest": null, + "resource_limit_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + }, + "created_at": "2026-07-24T12:00:00Z", + "implementation": { + "implementation_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "language": "rust", + "name": "ovk-static-checker", + "version": "1.0.0" + }, + "integrity": { + "artifact_digest": "sha256:405cbdcce40f391dc52a0b3608890ad95c75f911e2c08e72d307fae403aeb171", + "canonicalization_version": "v1" + }, + "known_blind_spots": [ + "does not observe runtime network" + ], + "mechanism": { + "allows_abstention": true, + "description": "Static claim checker", + "determinism": "deterministic", + "mechanism_class": "static_analysis" + }, + "producer": "OVK", + "producer_version": "0.1.0", + "schema_version": "v1", + "source_commit": "abc1234", + "source_repo": "https://github.com/SentinelOps-CI/pcs-core", + "verifier_profile_id": "vp-ovk-static-001" +} diff --git a/examples/verifier_assurance/invalid/short_source_commit/manifest.json b/examples/verifier_assurance/invalid/short_source_commit/manifest.json new file mode 100644 index 00000000..f82deed5 --- /dev/null +++ b/examples/verifier_assurance/invalid/short_source_commit/manifest.json @@ -0,0 +1,5 @@ +{ + "artifact_file": "artifact.json", + "artifact_type": "VerifierProfile.v1", + "expected_error": "InvalidSourceCommit" +} diff --git a/examples/verifier_assurance/invalid/timeout_accept/artifact.json b/examples/verifier_assurance/invalid/timeout_accept/artifact.json new file mode 100644 index 00000000..6848d319 --- /dev/null +++ b/examples/verifier_assurance/invalid/timeout_accept/artifact.json @@ -0,0 +1,56 @@ +{ + "artifact_type": "VerificationResult.v1", + "check_groups": [ + { + "checks": [ + { + "check_id": "process.complete", + "mandatory": true, + "status": "passed" + } + ], + "kind": "process" + }, + { + "checks": [ + { + "check_id": "evidence.present", + "mandatory": true, + "status": "passed" + } + ], + "kind": "evidence" + } + ], + "claim_ids": [ + "claim.safety.no_egress" + ], + "created_at": "2026-07-24T12:05:00Z", + "decision": "accept", + "execution_status": "timeout", + "initial_state_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "input_bundle_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "integrity": { + "artifact_digest": "sha256:07a522dcc2e210ac21a00de975f2d64ae7cf11a38c06ccb74ae52074a7b5c042", + "canonicalization_version": "v1" + }, + "normalization_applied": true, + "normalized_result_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "producer": "OVK", + "producer_version": "0.1.0", + "raw_backend_output_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "resource_limits": { + "memory_bytes": 1048576, + "wall_time_ms": 1000 + }, + "schema_version": "v1", + "source_commit": "e068794683959c52a19594a6d271dd5e69f3c999", + "source_repo": "https://github.com/SentinelOps-CI/pcs-core", + "terminal_state_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "trajectory_digest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + "verification_result_id": "vr-001", + "verifier_profile": { + "profile_digest": "sha256:c515b34f1f61d9b8d139d2a14a92409b2d45ffee16d16bb4c89913db1ebd53b0", + "verifier_profile_id": "vp-ovk-static-001" + } +} diff --git a/examples/verifier_assurance/invalid/timeout_accept/manifest.json b/examples/verifier_assurance/invalid/timeout_accept/manifest.json new file mode 100644 index 00000000..2232967e --- /dev/null +++ b/examples/verifier_assurance/invalid/timeout_accept/manifest.json @@ -0,0 +1,5 @@ +{ + "artifact_file": "artifact.json", + "artifact_type": "VerificationResult.v1", + "expected_error": "FailClosedDecision" +} diff --git a/examples/verifier_assurance/invalid/unknown_field/artifact.json b/examples/verifier_assurance/invalid/unknown_field/artifact.json new file mode 100644 index 00000000..4bd4f145 --- /dev/null +++ b/examples/verifier_assurance/invalid/unknown_field/artifact.json @@ -0,0 +1,53 @@ +{ + "applicability": { + "status": "active", + "valid_from": "2026-07-01T00:00:00Z" + }, + "artifact_type": "VerifierProfile.v1", + "assumptions": [ + "host OS is Linux" + ], + "claim_surface": { + "guarantee_class": "certificate_checked", + "out_of_scope_claim_ids": [ + "claim.formal.full_correctness" + ], + "supported_claim_ids": [ + "claim.safety.no_egress" + ] + }, + "configuration": { + "config_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "model_digest": null, + "policy_digest": null, + "prompt_digest": null, + "resource_limit_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + }, + "created_at": "2026-07-24T12:00:00Z", + "extra_field": "nope", + "implementation": { + "implementation_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "language": "rust", + "name": "ovk-static-checker", + "version": "1.0.0" + }, + "integrity": { + "artifact_digest": "sha256:150ceda191b6d8160a0c7e621ed1475942f73aa08e809c944b6a14df1e76b90e", + "canonicalization_version": "v1" + }, + "known_blind_spots": [ + "does not observe runtime network" + ], + "mechanism": { + "allows_abstention": true, + "description": "Static claim checker", + "determinism": "deterministic", + "mechanism_class": "static_analysis" + }, + "producer": "OVK", + "producer_version": "0.1.0", + "schema_version": "v1", + "source_commit": "e068794683959c52a19594a6d271dd5e69f3c999", + "source_repo": "https://github.com/SentinelOps-CI/pcs-core", + "verifier_profile_id": "vp-ovk-static-001" +} diff --git a/examples/verifier_assurance/invalid/unknown_field/manifest.json b/examples/verifier_assurance/invalid/unknown_field/manifest.json new file mode 100644 index 00000000..18d08803 --- /dev/null +++ b/examples/verifier_assurance/invalid/unknown_field/manifest.json @@ -0,0 +1,5 @@ +{ + "artifact_file": "artifact.json", + "artifact_type": "VerifierProfile.v1", + "expected_error": "Additional properties are not allowed ('extra_field' was unexpected)" +} diff --git a/examples/verifier_assurance/valid/adjudication_basic/adjudication.json b/examples/verifier_assurance/valid/adjudication_basic/adjudication.json new file mode 100644 index 00000000..bd02f9d1 --- /dev/null +++ b/examples/verifier_assurance/valid/adjudication_basic/adjudication.json @@ -0,0 +1,41 @@ +{ + "adjudication_id": "adj-001", + "artifact_type": "AdjudicationRecord.v1", + "blinded": true, + "conflict_of_interest_declared": false, + "created_at": "2026-07-24T13:00:00Z", + "disagreement": false, + "escalation": false, + "evidence_refs": [], + "independence_declared": true, + "integrity": { + "artifact_digest": "sha256:86f586ae5b592104734272f11cdfa141a104fd2c11f4485b6c2f0e7ab35a5dc4", + "canonicalization_version": "v1" + }, + "label": "valid", + "label_released_at": "2026-07-24T14:00:00Z", + "producer": "pcs-core", + "producer_version": "0.1.0", + "protected_rationale": { + "commitment_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "content_omitted": true, + "location_class": "access_controlled_store" + }, + "protocol_id": "adj-protocol", + "protocol_version": "1.0.0", + "schema_version": "v1", + "source_commit": "e068794683959c52a19594a6d271dd5e69f3c999", + "source_repo": "https://github.com/SentinelOps-CI/pcs-core", + "subject": { + "artifact_digest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + "artifact_id": "vr-001", + "artifact_type": "VerificationResult.v1" + }, + "votes": [ + { + "role": "primary", + "vote": "valid", + "voter_pseudonym_id": "voter-a" + } + ] +} diff --git a/examples/verifier_assurance/valid/campaign_basic/campaign.json b/examples/verifier_assurance/valid/campaign_basic/campaign.json new file mode 100644 index 00000000..d96f6a8b --- /dev/null +++ b/examples/verifier_assurance/valid/campaign_basic/campaign.json @@ -0,0 +1,79 @@ +{ + "access_class": "black_box", + "algorithm": "random-search", + "artifact_type": "OptimizationCampaignManifest.v1", + "campaign_id": "camp-001", + "checkpoint_ids": [ + "ckpt-0" + ], + "cohorts": [ + { + "access_class": "black_box", + "cohort_id": "cohort-ordinary", + "cohort_kind": "ordinary", + "compute_exposure": { + "accounting_method": "wall_clock", + "query_count": 50, + "wall_time_seconds": 600 + } + }, + { + "access_class": "adaptive", + "cohort_id": "cohort-optimized", + "cohort_kind": "optimized", + "compute_exposure": { + "accounting_method": "wall_clock", + "query_count": 950, + "wall_time_seconds": 3000 + } + } + ], + "compute_budget": { + "accounting_method": "wall_clock", + "gpu_hours": "0", + "query_budget": 1000, + "wall_time_seconds": 3600 + }, + "containment_policy": "offline-only", + "created_at": "2026-07-24T11:00:00Z", + "disclosure_policy": "public-summary", + "env_profile_id": "env-lab", + "env_profile_version": "1.0.0", + "episode_count": 10, + "harness_id": "harness-a", + "integrity": { + "artifact_digest": "sha256:1adebaaa9854973c44977fb1bcf22c4792f827623804be8ac67279189811d4b8", + "canonicalization_version": "v1" + }, + "model_id": "model-a", + "optimizer_settings": { + "batch_size": 8, + "learning_rate": "0.01" + }, + "policy_id": "policy-a", + "producer": "pcs-core", + "producer_version": "0.1.0", + "query_count": 100, + "schema_version": "v1", + "seeds": [ + 1, + 2, + 3 + ], + "source_commit": "e068794683959c52a19594a6d271dd5e69f3c999", + "source_repo": "https://github.com/SentinelOps-CI/pcs-core", + "splits": { + "holdout": "hold-v1", + "train": "train-v1", + "validation": "val-v1" + }, + "target_verifier_profile": { + "profile_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "verifier_profile_id": "vp-ovk-static-001" + }, + "trajectory_count": 10, + "visibility": { + "full_traces_retained": false, + "public_summary": true + } +} diff --git a/examples/verifier_assurance/valid/invocation_basic/invocation.json b/examples/verifier_assurance/valid/invocation_basic/invocation.json new file mode 100644 index 00000000..c7eb618b --- /dev/null +++ b/examples/verifier_assurance/valid/invocation_basic/invocation.json @@ -0,0 +1,53 @@ +{ + "artifact_type": "VerifierInvocationRecord.v1", + "assumptions": [], + "canonicalization_version": "v1", + "command": { + "argv": [ + "opa", + "eval", + "-d", + "policy.rego", + "data.allow" + ], + "cwd": ".", + "env_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "limits": { + "timeout_ms": 30000 + } + }, + "compiled_obligation": { + "digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "media_type": "application/json", + "path": "compiled_obligation.json" + }, + "completed_at": "2026-07-24T16:01:01Z", + "exit": { + "code": 0, + "kind": "exited" + }, + "guarantee_class": "runtime_observed", + "input_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "integrity": { + "artifact_digest": "sha256:a8761bfcdda71d47ac64d26513d378d258404f9042d4ad1a8816dcfc935a6ac1", + "canonicalization_version": "v1" + }, + "invocation_id": "vi-ovk-pin-001", + "limits": [], + "normalized_result_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "normalizer_version": "ovk.normalize.v1", + "producer": "OVK", + "producer_version": "0.4.0", + "profile_ref": { + "artifact_type": "VerifierProfile.v1", + "profile_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "verifier_profile_id": "vp-opa-demo-001" + }, + "raw_backend_result_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "schema_version": "v1", + "source_commit": "e068794683959c52a19594a6d271dd5e69f3c999", + "source_repo": "https://github.com/SentinelOps-CI/pcs-core", + "started_at": "2026-07-24T16:01:00Z", + "stderr_digest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + "stdout_digest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" +} diff --git a/examples/verifier_assurance/valid/mutation_timeout/mutation.json b/examples/verifier_assurance/valid/mutation_timeout/mutation.json new file mode 100644 index 00000000..4b2ba5cd --- /dev/null +++ b/examples/verifier_assurance/valid/mutation_timeout/mutation.json @@ -0,0 +1,32 @@ +{ + "artifact_type": "VerifierMutationManifest.v1", + "base_profile_ref": { + "artifact_type": "VerifierProfile.v1", + "profile_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "verifier_profile_id": "vp-opa-demo-001" + }, + "canonicalization_version": "v1", + "created_at": "2026-07-24T16:03:00Z", + "expected_effect": "Changes execution timeout.", + "integrity": { + "artifact_digest": "sha256:35d8c217a200611e57fdc6039018429aa27e1169717678bed7c205d1dd95f025", + "canonicalization_version": "v1" + }, + "mutated_profile_ref": { + "artifact_type": "VerifierProfile.v1", + "profile_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "verifier_profile_id": "vp-opa-demo-001-mut-alter_timeout" + }, + "mutation_class": "alter_timeout", + "mutation_id": "mut-alter_timeout-001", + "parameters": { + "timeout_ms": 1000 + }, + "producer": "OVK", + "producer_version": "0.4.0", + "production_prohibition": true, + "schema_version": "v1", + "source_commit": "e068794683959c52a19594a6d271dd5e69f3c999", + "source_repo": "https://github.com/SentinelOps-CI/pcs-core", + "supported_by_adapter": true +} diff --git a/examples/verifier_assurance/valid/profile_basic/profile.json b/examples/verifier_assurance/valid/profile_basic/profile.json new file mode 100644 index 00000000..81e4d685 --- /dev/null +++ b/examples/verifier_assurance/valid/profile_basic/profile.json @@ -0,0 +1,52 @@ +{ + "applicability": { + "status": "active", + "valid_from": "2026-07-01T00:00:00Z" + }, + "artifact_type": "VerifierProfile.v1", + "assumptions": [ + "host OS is Linux" + ], + "claim_surface": { + "guarantee_class": "certificate_checked", + "out_of_scope_claim_ids": [ + "claim.formal.full_correctness" + ], + "supported_claim_ids": [ + "claim.safety.no_egress" + ] + }, + "configuration": { + "config_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "model_digest": null, + "policy_digest": null, + "prompt_digest": null, + "resource_limit_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + }, + "created_at": "2026-07-24T12:00:00Z", + "implementation": { + "implementation_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "language": "rust", + "name": "ovk-static-checker", + "version": "1.0.0" + }, + "integrity": { + "artifact_digest": "sha256:3af8ff33a41e4a1af25cd184b4f610f7d80da942100fb01667419195b3c5ff91", + "canonicalization_version": "v1" + }, + "known_blind_spots": [ + "does not observe runtime network" + ], + "mechanism": { + "allows_abstention": true, + "description": "Static claim checker", + "determinism": "deterministic", + "mechanism_class": "static_analysis" + }, + "producer": "OVK", + "producer_version": "0.1.0", + "schema_version": "v1", + "source_commit": "e068794683959c52a19594a6d271dd5e69f3c999", + "source_repo": "https://github.com/SentinelOps-CI/pcs-core", + "verifier_profile_id": "vp-ovk-static-001" +} diff --git a/examples/verifier_assurance/valid/replay_matched/replay.json b/examples/verifier_assurance/valid/replay_matched/replay.json new file mode 100644 index 00000000..5f04df1b --- /dev/null +++ b/examples/verifier_assurance/valid/replay_matched/replay.json @@ -0,0 +1,36 @@ +{ + "artifact_type": "VerifierReplayReport.v1", + "canonicalization_version": "v1", + "created_at": "2026-07-24T16:02:00Z", + "determinism": "deterministic", + "drift": { + "claim_match": true, + "normalized_digest_match": true, + "raw_digest_match": true + }, + "integrity": { + "artifact_digest": "sha256:f2af1afd1ee217db70273ebce5d695af47bfbb66e975b4411b5860b3b79f85af", + "canonicalization_version": "v1" + }, + "original_invocation_ref": { + "artifact_type": "VerifierInvocationRecord.v1", + "invocation_digest": "sha256:a8761bfcdda71d47ac64d26513d378d258404f9042d4ad1a8816dcfc935a6ac1", + "invocation_id": "vi-ovk-pin-001" + }, + "original_normalized_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "original_raw_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "producer": "OVK", + "producer_version": "0.4.0", + "profile_ref": { + "artifact_type": "VerifierProfile.v1", + "profile_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "verifier_profile_id": "vp-opa-demo-001" + }, + "replay_id": "rr-ovk-pin-001", + "replay_normalized_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "replay_raw_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "replay_status": "matched", + "schema_version": "v1", + "source_commit": "e068794683959c52a19594a6d271dd5e69f3c999", + "source_repo": "https://github.com/SentinelOps-CI/pcs-core" +} diff --git a/examples/verifier_assurance/valid/report_rebuild/adjudications/adj-001.json b/examples/verifier_assurance/valid/report_rebuild/adjudications/adj-001.json new file mode 100644 index 00000000..bd02f9d1 --- /dev/null +++ b/examples/verifier_assurance/valid/report_rebuild/adjudications/adj-001.json @@ -0,0 +1,41 @@ +{ + "adjudication_id": "adj-001", + "artifact_type": "AdjudicationRecord.v1", + "blinded": true, + "conflict_of_interest_declared": false, + "created_at": "2026-07-24T13:00:00Z", + "disagreement": false, + "escalation": false, + "evidence_refs": [], + "independence_declared": true, + "integrity": { + "artifact_digest": "sha256:86f586ae5b592104734272f11cdfa141a104fd2c11f4485b6c2f0e7ab35a5dc4", + "canonicalization_version": "v1" + }, + "label": "valid", + "label_released_at": "2026-07-24T14:00:00Z", + "producer": "pcs-core", + "producer_version": "0.1.0", + "protected_rationale": { + "commitment_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "content_omitted": true, + "location_class": "access_controlled_store" + }, + "protocol_id": "adj-protocol", + "protocol_version": "1.0.0", + "schema_version": "v1", + "source_commit": "e068794683959c52a19594a6d271dd5e69f3c999", + "source_repo": "https://github.com/SentinelOps-CI/pcs-core", + "subject": { + "artifact_digest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + "artifact_id": "vr-001", + "artifact_type": "VerificationResult.v1" + }, + "votes": [ + { + "role": "primary", + "vote": "valid", + "voter_pseudonym_id": "voter-a" + } + ] +} diff --git a/examples/verifier_assurance/valid/report_rebuild/adjudications/adj-002.json b/examples/verifier_assurance/valid/report_rebuild/adjudications/adj-002.json new file mode 100644 index 00000000..1ed6098e --- /dev/null +++ b/examples/verifier_assurance/valid/report_rebuild/adjudications/adj-002.json @@ -0,0 +1,41 @@ +{ + "adjudication_id": "adj-002", + "artifact_type": "AdjudicationRecord.v1", + "blinded": true, + "conflict_of_interest_declared": false, + "created_at": "2026-07-24T13:00:00Z", + "disagreement": false, + "escalation": false, + "evidence_refs": [], + "independence_declared": true, + "integrity": { + "artifact_digest": "sha256:8bb310927530fc020d1f2df378aeb90bf46c224fe2f5f1e3b5f5e777e79705f3", + "canonicalization_version": "v1" + }, + "label": "invalid", + "label_released_at": "2026-07-24T14:00:00Z", + "producer": "pcs-core", + "producer_version": "0.1.0", + "protected_rationale": { + "commitment_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "content_omitted": true, + "location_class": "access_controlled_store" + }, + "protocol_id": "adj-protocol", + "protocol_version": "1.0.0", + "schema_version": "v1", + "source_commit": "e068794683959c52a19594a6d271dd5e69f3c999", + "source_repo": "https://github.com/SentinelOps-CI/pcs-core", + "subject": { + "artifact_digest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + "artifact_id": "vr-002", + "artifact_type": "VerificationResult.v1" + }, + "votes": [ + { + "role": "primary", + "vote": "invalid", + "voter_pseudonym_id": "voter-a" + } + ] +} diff --git a/examples/verifier_assurance/valid/report_rebuild/campaign.json b/examples/verifier_assurance/valid/report_rebuild/campaign.json new file mode 100644 index 00000000..d96f6a8b --- /dev/null +++ b/examples/verifier_assurance/valid/report_rebuild/campaign.json @@ -0,0 +1,79 @@ +{ + "access_class": "black_box", + "algorithm": "random-search", + "artifact_type": "OptimizationCampaignManifest.v1", + "campaign_id": "camp-001", + "checkpoint_ids": [ + "ckpt-0" + ], + "cohorts": [ + { + "access_class": "black_box", + "cohort_id": "cohort-ordinary", + "cohort_kind": "ordinary", + "compute_exposure": { + "accounting_method": "wall_clock", + "query_count": 50, + "wall_time_seconds": 600 + } + }, + { + "access_class": "adaptive", + "cohort_id": "cohort-optimized", + "cohort_kind": "optimized", + "compute_exposure": { + "accounting_method": "wall_clock", + "query_count": 950, + "wall_time_seconds": 3000 + } + } + ], + "compute_budget": { + "accounting_method": "wall_clock", + "gpu_hours": "0", + "query_budget": 1000, + "wall_time_seconds": 3600 + }, + "containment_policy": "offline-only", + "created_at": "2026-07-24T11:00:00Z", + "disclosure_policy": "public-summary", + "env_profile_id": "env-lab", + "env_profile_version": "1.0.0", + "episode_count": 10, + "harness_id": "harness-a", + "integrity": { + "artifact_digest": "sha256:1adebaaa9854973c44977fb1bcf22c4792f827623804be8ac67279189811d4b8", + "canonicalization_version": "v1" + }, + "model_id": "model-a", + "optimizer_settings": { + "batch_size": 8, + "learning_rate": "0.01" + }, + "policy_id": "policy-a", + "producer": "pcs-core", + "producer_version": "0.1.0", + "query_count": 100, + "schema_version": "v1", + "seeds": [ + 1, + 2, + 3 + ], + "source_commit": "e068794683959c52a19594a6d271dd5e69f3c999", + "source_repo": "https://github.com/SentinelOps-CI/pcs-core", + "splits": { + "holdout": "hold-v1", + "train": "train-v1", + "validation": "val-v1" + }, + "target_verifier_profile": { + "profile_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "verifier_profile_id": "vp-ovk-static-001" + }, + "trajectory_count": 10, + "visibility": { + "full_traces_retained": false, + "public_summary": true + } +} diff --git a/examples/verifier_assurance/valid/report_rebuild/report.json b/examples/verifier_assurance/valid/report_rebuild/report.json new file mode 100644 index 00000000..f958d44b --- /dev/null +++ b/examples/verifier_assurance/valid/report_rebuild/report.json @@ -0,0 +1,192 @@ +{ + "applicability_limits": [ + "synthetic fixture only" + ], + "artifact_type": "VerifierAssuranceReport.v1", + "campaign_ref": { + "artifact_digest": "sha256:fd790a71e8eeefcd40d6b2b3b1642b3f93b93669d518564a62fc4f9f2d93ff8f", + "artifact_id": "camp-001", + "artifact_type": "OptimizationCampaignManifest.v1" + }, + "cohorts": [ + { + "accept_count": 1, + "access_class": "black_box", + "cohort_id": "cohort-ordinary", + "cohort_kind": "ordinary", + "compute_exposure": { + "accounting_method": "wall_clock", + "query_count": 50, + "wall_time_seconds": 600 + }, + "included_result_count": 1, + "indeterminate_count": 0, + "reject_count": 0 + }, + { + "accept_count": 0, + "access_class": "adaptive", + "cohort_id": "cohort-optimized", + "cohort_kind": "optimized", + "compute_exposure": { + "accounting_method": "wall_clock", + "query_count": 950, + "wall_time_seconds": 3000 + }, + "included_result_count": 1, + "indeterminate_count": 0, + "reject_count": 1 + } + ], + "created_at": "2026-07-24T15:00:00Z", + "excluded_items": [ + { + "item_id": "ex-1", + "reason_code": "out_of_scope" + } + ], + "independent_adjudication": true, + "integrity": { + "artifact_digest": "sha256:b7396e3817f27ab95f8b28bf89faea4d7c670727233ac4eb6c54a2778d23ac84", + "canonicalization_version": "v1" + }, + "metrics": { + "abstention_rate": { + "confidence_interval": { + "lower": "0.000000", + "method": "wilson", + "parameters": { + "alpha": "0.05", + "declared": true + }, + "sample_size": 2, + "upper": "0.000000" + }, + "denominator": 2, + "numerator": 0, + "rate": "0.000000" + }, + "adjudication_coverage": { + "confidence_interval": { + "lower": "1.000000", + "method": "wilson", + "parameters": { + "alpha": "0.05", + "declared": true + }, + "sample_size": 2, + "upper": "1.000000" + }, + "denominator": 2, + "numerator": 2, + "rate": "1.000000" + }, + "cost_decimal": "0", + "excluded_count": 1, + "exploit_family_counts": {}, + "false_accept_rate": { + "confidence_interval": { + "lower": "0.000000", + "method": "wilson", + "parameters": { + "alpha": "0.05", + "declared": true + }, + "sample_size": 2, + "upper": "0.000000" + }, + "denominator": 2, + "numerator": 0, + "rate": "0.000000" + }, + "false_reject_rate": { + "confidence_interval": { + "lower": "0.000000", + "method": "wilson", + "parameters": { + "alpha": "0.05", + "declared": true + }, + "sample_size": 2, + "upper": "0.000000" + }, + "denominator": 2, + "numerator": 0, + "rate": "0.000000" + }, + "inter_verifier_disagreement_rate": { + "confidence_interval": { + "lower": "0.000000", + "method": "wilson", + "parameters": { + "alpha": "0.05", + "declared": true + }, + "sample_size": 2, + "upper": "0.000000" + }, + "denominator": 2, + "numerator": 0, + "rate": "0.000000" + }, + "latency_ms_p50": 0, + "optimization_gap": "-1.000000", + "optimized_accept_rate": { + "confidence_interval": { + "lower": "0.000000", + "method": "wilson", + "parameters": { + "alpha": "0.05", + "declared": true + }, + "sample_size": 1, + "upper": "0.000000" + }, + "denominator": 1, + "numerator": 0, + "rate": "0.000000" + }, + "ordinary_accept_rate": { + "confidence_interval": { + "lower": "1.000000", + "method": "wilson", + "parameters": { + "alpha": "0.05", + "declared": true + }, + "sample_size": 1, + "upper": "1.000000" + }, + "denominator": 1, + "numerator": 1, + "rate": "1.000000" + }, + "queries_to_first_exploit": null, + "reward_invalidation_rate": { + "confidence_interval": { + "lower": "0.000000", + "method": "wilson", + "parameters": { + "alpha": "0.05", + "declared": true + }, + "sample_size": 2, + "upper": "0.000000" + }, + "denominator": 2, + "numerator": 0, + "rate": "0.000000" + }, + "sample_size": 2, + "time_to_first_exploit_seconds": null, + "unadjudicated_count": 0 + }, + "producer": "pcs-core", + "producer_version": "0.1.0", + "release_grade": true, + "report_id": "rep-001", + "schema_version": "v1", + "source_commit": "e068794683959c52a19594a6d271dd5e69f3c999", + "source_repo": "https://github.com/SentinelOps-CI/pcs-core", + "unadjudicated_items": [] +} diff --git a/examples/verifier_assurance/valid/report_rebuild/results/vr-001.json b/examples/verifier_assurance/valid/report_rebuild/results/vr-001.json new file mode 100644 index 00000000..7f070e75 --- /dev/null +++ b/examples/verifier_assurance/valid/report_rebuild/results/vr-001.json @@ -0,0 +1,57 @@ +{ + "artifact_type": "VerificationResult.v1", + "check_groups": [ + { + "checks": [ + { + "check_id": "process.complete", + "mandatory": true, + "status": "passed" + } + ], + "kind": "process" + }, + { + "checks": [ + { + "check_id": "evidence.present", + "mandatory": true, + "status": "passed" + } + ], + "kind": "evidence" + } + ], + "claim_ids": [ + "claim.safety.no_egress" + ], + "cohort_id": "cohort-ordinary", + "created_at": "2026-07-24T12:05:00Z", + "decision": "accept", + "execution_status": "completed", + "initial_state_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "input_bundle_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "integrity": { + "artifact_digest": "sha256:06babb0442e5dc8a31b9376a0be19266de7ca021c5dafe2562ba4fe58d096412", + "canonicalization_version": "v1" + }, + "normalization_applied": true, + "normalized_result_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "producer": "OVK", + "producer_version": "0.1.0", + "raw_backend_output_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "resource_limits": { + "memory_bytes": 1048576, + "wall_time_ms": 1000 + }, + "schema_version": "v1", + "source_commit": "e068794683959c52a19594a6d271dd5e69f3c999", + "source_repo": "https://github.com/SentinelOps-CI/pcs-core", + "terminal_state_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "trajectory_digest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + "verification_result_id": "vr-001", + "verifier_profile": { + "profile_digest": "sha256:c515b34f1f61d9b8d139d2a14a92409b2d45ffee16d16bb4c89913db1ebd53b0", + "verifier_profile_id": "vp-ovk-static-001" + } +} diff --git a/examples/verifier_assurance/valid/report_rebuild/results/vr-002.json b/examples/verifier_assurance/valid/report_rebuild/results/vr-002.json new file mode 100644 index 00000000..4bf951a3 --- /dev/null +++ b/examples/verifier_assurance/valid/report_rebuild/results/vr-002.json @@ -0,0 +1,47 @@ +{ + "artifact_type": "VerificationResult.v1", + "check_groups": [ + { + "checks": [ + { + "check_id": "process.complete", + "mandatory": true, + "status": "failed" + } + ], + "kind": "process" + } + ], + "claim_ids": [ + "claim.safety.no_egress" + ], + "cohort_id": "cohort-optimized", + "created_at": "2026-07-24T12:05:00Z", + "decision": "reject", + "execution_status": "completed", + "initial_state_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "input_bundle_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "integrity": { + "artifact_digest": "sha256:86ae4815c7268f5502bfbc3531b73112133f2b4a2be2efe6c378ebfd799160ca", + "canonicalization_version": "v1" + }, + "normalization_applied": true, + "normalized_result_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "producer": "OVK", + "producer_version": "0.1.0", + "raw_backend_output_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "resource_limits": { + "memory_bytes": 1048576, + "wall_time_ms": 1000 + }, + "schema_version": "v1", + "source_commit": "e068794683959c52a19594a6d271dd5e69f3c999", + "source_repo": "https://github.com/SentinelOps-CI/pcs-core", + "terminal_state_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "trajectory_digest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + "verification_result_id": "vr-002", + "verifier_profile": { + "profile_digest": "sha256:c515b34f1f61d9b8d139d2a14a92409b2d45ffee16d16bb4c89913db1ebd53b0", + "verifier_profile_id": "vp-ovk-static-001" + } +} diff --git a/examples/verifier_assurance/valid/result_accept/result.json b/examples/verifier_assurance/valid/result_accept/result.json new file mode 100644 index 00000000..b4e5675e --- /dev/null +++ b/examples/verifier_assurance/valid/result_accept/result.json @@ -0,0 +1,56 @@ +{ + "artifact_type": "VerificationResult.v1", + "check_groups": [ + { + "checks": [ + { + "check_id": "process.complete", + "mandatory": true, + "status": "passed" + } + ], + "kind": "process" + }, + { + "checks": [ + { + "check_id": "evidence.present", + "mandatory": true, + "status": "passed" + } + ], + "kind": "evidence" + } + ], + "claim_ids": [ + "claim.safety.no_egress" + ], + "created_at": "2026-07-24T12:05:00Z", + "decision": "accept", + "execution_status": "completed", + "initial_state_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "input_bundle_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "integrity": { + "artifact_digest": "sha256:7fa5afbf3d931d448c724fb0542101f7c8bc5f14cd19456c3344d3d8d23a0ad2", + "canonicalization_version": "v1" + }, + "normalization_applied": true, + "normalized_result_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "producer": "OVK", + "producer_version": "0.1.0", + "raw_backend_output_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "resource_limits": { + "memory_bytes": 1048576, + "wall_time_ms": 1000 + }, + "schema_version": "v1", + "source_commit": "e068794683959c52a19594a6d271dd5e69f3c999", + "source_repo": "https://github.com/SentinelOps-CI/pcs-core", + "terminal_state_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "trajectory_digest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + "verification_result_id": "vr-001", + "verifier_profile": { + "profile_digest": "sha256:c515b34f1f61d9b8d139d2a14a92409b2d45ffee16d16bb4c89913db1ebd53b0", + "verifier_profile_id": "vp-ovk-static-001" + } +} diff --git a/examples/verifier_assurance/valid/reward_scalar/reward.json b/examples/verifier_assurance/valid/reward_scalar/reward.json new file mode 100644 index 00000000..65b8a5c2 --- /dev/null +++ b/examples/verifier_assurance/valid/reward_scalar/reward.json @@ -0,0 +1,62 @@ +{ + "artifact_type": "RewardEvidenceEnvelope.v1", + "authority": { + "authority_class": "producer", + "authority_id": "labtrust-reward" + }, + "claims_issued": [ + "claim.safety.no_egress" + ], + "claims_rejected": [], + "claims_unresolved": [], + "components": [ + { + "claim_id": "claim.safety.no_egress", + "component_id": "safety", + "guarantee_class": "certificate_checked", + "value": "1.0" + }, + { + "component_id": "quality", + "guarantee_class": "empirically_measured", + "value": "0.5" + } + ], + "composition_function": "sum", + "created_at": "2026-07-24T12:10:00Z", + "env_profile_id": "env-lab", + "env_profile_version": "1.0.0", + "episode_id": "ep-1", + "integrity": { + "artifact_digest": "sha256:5acb6ac017e8ff3e05eeb7a7453fec985b0424a05137191a4b505cccef2b7889", + "canonicalization_version": "v1" + }, + "lifecycle": { + "status": "active" + }, + "mandatory_unresolved_claim_ids": [], + "producer": "LabTrust-Gym", + "producer_version": "0.1.0", + "profile_refs": [ + { + "applicability_status": "active", + "profile_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "verifier_profile_id": "vp-ovk-static-001" + } + ], + "reward_envelope_id": "rew-001", + "scalar_total": "1.5", + "schema_version": "v1", + "source_commit": "e068794683959c52a19594a6d271dd5e69f3c999", + "source_repo": "https://github.com/SentinelOps-CI/pcs-core", + "state_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "step_index": 0, + "trajectory_digest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + "verifier_result_refs": [ + { + "artifact_digest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + "artifact_id": "vr-001", + "artifact_type": "VerificationResult.v1" + } + ] +} diff --git a/python/pcs_core/cli.py b/python/pcs_core/cli.py index 4831e84d..c5e26b68 100644 --- a/python/pcs_core/cli.py +++ b/python/pcs_core/cli.py @@ -66,6 +66,200 @@ def cmd_validate(path: Path) -> int: return 1 +def _cmd_va_validate(path: Path, *, expected_type: str, as_json: bool = False) -> int: + try: + data = _load_json(path) + from pcs_core.verifier_assurance_validate import ( + SemanticIssue, + validate_va_semantics, + ) + + validate_artifact(data, expected_type, release_grade=True) + issues = validate_va_semantics(data, expected_type, as_issues=True) + assert isinstance(issues, list) + if issues: + structured = [] + for item in issues: + if isinstance(item, SemanticIssue): + structured.append( + {"code": item.code, "path": item.path, "message": item.message} + ) + else: + structured.append( + {"code": "ValidationError", "path": "$", "message": str(item)} + ) + if as_json: + print( + json.dumps( + { + "ok": False, + "artifact_type": expected_type, + "path": str(path), + "errors": structured, + } + ) + ) + else: + print(f"FAIL {path}: semantic validation failed for {expected_type}", file=sys.stderr) + for item in structured: + print( + f" - {item['code']} at {item['path']}: {item['message']}", + file=sys.stderr, + ) + return 1 + if as_json: + print(json.dumps({"ok": True, "artifact_type": expected_type, "path": str(path)})) + else: + print(f"OK {expected_type} {path}") + return 0 + except ValidationError as exc: + errors: list[dict[str, str]] = [] + for err in exc.errors: + if " at " in err and ": " in err: + code, rest = err.split(" at ", 1) + path_part, message = rest.split(": ", 1) + errors.append({"code": code, "path": path_part, "message": message}) + else: + errors.append({"code": "ValidationError", "path": "$", "message": err}) + if not errors: + errors = [{"code": "ValidationError", "path": "$", "message": str(exc)}] + payload = { + "ok": False, + "artifact_type": expected_type, + "path": str(path), + "errors": errors, + } + if as_json: + print(json.dumps(payload)) + else: + print(f"FAIL {path}: {exc}", file=sys.stderr) + for err in exc.errors: + print(f" - {err}", file=sys.stderr) + return 1 + + +def cmd_assurance_build_report( + *, + campaign: Path, + results: Path, + adjudications: Path, + out: Path, + report_id: str, + created_at: str, + source_commit: str, + release_grade: bool, + as_json: bool, +) -> int: + from pcs_core.verifier_assurance_report import ( + ReportBuildError, + build_assurance_report_from_paths, + ) + from pcs_core.verifier_assurance_validate import SemanticIssue + + try: + report = build_assurance_report_from_paths( + campaign_path=campaign, + results_dir=results, + adjudications_dir=adjudications, + report_id=report_id, + created_at=created_at, + source_commit=source_commit, + release_grade=release_grade, + out_path=out, + ) + if as_json: + print(json.dumps({"ok": True, "out": str(out), "report_id": report["report_id"]})) + else: + print(f"OK wrote {out}") + return 0 + except ReportBuildError as exc: + structured = [] + for item in exc.issues: + if isinstance(item, SemanticIssue): + structured.append( + {"code": item.code, "path": item.path, "message": item.message} + ) + else: + structured.append( + {"code": "BuildFailure", "path": "$", "message": str(item)} + ) + if as_json: + print(json.dumps({"ok": False, "errors": structured})) + else: + print(f"FAIL build-report: {exc}", file=sys.stderr) + for item in structured: + print( + f" - {item['code']} at {item['path']}: {item['message']}", + file=sys.stderr, + ) + return 1 + except (ValidationError, OSError, ValueError) as exc: + if as_json: + print( + json.dumps( + { + "ok": False, + "errors": [ + {"code": "BuildFailure", "path": "$", "message": str(exc)} + ], + } + ) + ) + else: + print(f"FAIL build-report: {exc}", file=sys.stderr) + return 1 + + +def cmd_assurance_verify_report(path: Path, *, as_json: bool = False) -> int: + try: + from pcs_core.verifier_assurance_report import verify_assurance_report + from pcs_core.verifier_assurance_validate import SemanticIssue + + data = _load_json(path) + validate_artifact(data, "VerifierAssuranceReport.v1", release_grade=True) + issues = verify_assurance_report(data) + if issues: + structured = [] + for item in issues: + if isinstance(item, SemanticIssue): + structured.append( + {"code": item.code, "path": item.path, "message": item.message} + ) + else: + structured.append( + {"code": "ValidationError", "path": "$", "message": str(item)} + ) + if as_json: + print(json.dumps({"ok": False, "path": str(path), "errors": structured})) + else: + print( + f"FAIL {path}: assurance report verification failed", + file=sys.stderr, + ) + for item in structured: + print( + f" - {item['code']} at {item['path']}: {item['message']}", + file=sys.stderr, + ) + return 1 + if as_json: + print(json.dumps({"ok": True, "path": str(path)})) + else: + print(f"OK VerifierAssuranceReport.v1 {path}") + return 0 + except ValidationError as exc: + errors = [{"code": "ValidationError", "path": "$", "message": str(exc)}] + for err in exc.errors: + errors.append({"code": "ValidationError", "path": "$", "message": err}) + if as_json: + print(json.dumps({"ok": False, "path": str(path), "errors": errors})) + else: + print(f"FAIL {path}: {exc}", file=sys.stderr) + for err in exc.errors: + print(f" - {err}", file=sys.stderr) + return 1 + + def cmd_hash(path: Path) -> int: data = _load_json(path) print(canonical_hash(data)) @@ -1254,6 +1448,62 @@ def main(argv: list[str] | None = None) -> int: help="Emit ReleaseGateCheckReport.v0 JSON", ) + verifier_parser = sub.add_parser("verifier", help="Verifier Assurance profile/result commands") + verifier_sub = verifier_parser.add_subparsers(dest="verifier_cmd", required=True) + verifier_profile = verifier_sub.add_parser("profile", help="VerifierProfile.v1 commands") + verifier_profile_sub = verifier_profile.add_subparsers(dest="verifier_profile_cmd", required=True) + p_vp_validate = verifier_profile_sub.add_parser("validate", help="Validate VerifierProfile.v1") + p_vp_validate.add_argument("path", type=Path) + p_vp_validate.add_argument("--json", action="store_true") + verifier_result = verifier_sub.add_parser("result", help="VerificationResult.v1 commands") + verifier_result_sub = verifier_result.add_subparsers(dest="verifier_result_cmd", required=True) + p_vr_validate = verifier_result_sub.add_parser("validate", help="Validate VerificationResult.v1") + p_vr_validate.add_argument("path", type=Path) + p_vr_validate.add_argument("--json", action="store_true") + + reward_parser = sub.add_parser("reward", help="RewardEvidenceEnvelope.v1 commands") + reward_sub = reward_parser.add_subparsers(dest="reward_cmd", required=True) + p_reward_validate = reward_sub.add_parser("validate", help="Validate RewardEvidenceEnvelope.v1") + p_reward_validate.add_argument("path", type=Path) + p_reward_validate.add_argument("--json", action="store_true") + + campaign_parser = sub.add_parser("campaign", help="OptimizationCampaignManifest.v1 commands") + campaign_sub = campaign_parser.add_subparsers(dest="campaign_cmd", required=True) + p_campaign_validate = campaign_sub.add_parser( + "validate", help="Validate OptimizationCampaignManifest.v1" + ) + p_campaign_validate.add_argument("path", type=Path) + p_campaign_validate.add_argument("--json", action="store_true") + + adjudication_parser = sub.add_parser("adjudication", help="AdjudicationRecord.v1 commands") + adjudication_sub = adjudication_parser.add_subparsers(dest="adjudication_cmd", required=True) + p_adj_validate = adjudication_sub.add_parser("validate", help="Validate AdjudicationRecord.v1") + p_adj_validate.add_argument("path", type=Path) + p_adj_validate.add_argument("--json", action="store_true") + + assurance_parser = sub.add_parser("assurance", help="VerifierAssuranceReport.v1 commands") + assurance_sub = assurance_parser.add_subparsers(dest="assurance_cmd", required=True) + p_assurance_build = assurance_sub.add_parser( + "build-report", help="Build VerifierAssuranceReport.v1 offline" + ) + p_assurance_build.add_argument("--campaign", type=Path, required=True) + p_assurance_build.add_argument("--results", type=Path, required=True) + p_assurance_build.add_argument("--adjudications", type=Path, required=True) + p_assurance_build.add_argument("--out", type=Path, required=True) + p_assurance_build.add_argument("--report-id", default="report-local") + p_assurance_build.add_argument("--created-at", default="2026-07-24T15:00:00Z") + p_assurance_build.add_argument( + "--source-commit", + default="e068794683959c52a19594a6d271dd5e69f3c999", + ) + p_assurance_build.add_argument("--release-grade", action="store_true") + p_assurance_build.add_argument("--json", action="store_true") + p_assurance_verify = assurance_sub.add_parser( + "verify-report", help="Verify VerifierAssuranceReport.v1" + ) + p_assurance_verify.add_argument("path", type=Path) + p_assurance_verify.add_argument("--json", action="store_true") + args = parser.parse_args(argv) if args.command == "capabilities": @@ -1443,6 +1693,40 @@ def main(argv: list[str] | None = None) -> int: require_oci_publish=args.require_oci_publish, as_json=args.json, ) + if args.command == "verifier" and args.verifier_cmd == "profile": + return _cmd_va_validate( + args.path, expected_type="VerifierProfile.v1", as_json=args.json + ) + if args.command == "verifier" and args.verifier_cmd == "result": + return _cmd_va_validate( + args.path, expected_type="VerificationResult.v1", as_json=args.json + ) + if args.command == "reward" and args.reward_cmd == "validate": + return _cmd_va_validate( + args.path, expected_type="RewardEvidenceEnvelope.v1", as_json=args.json + ) + if args.command == "campaign" and args.campaign_cmd == "validate": + return _cmd_va_validate( + args.path, expected_type="OptimizationCampaignManifest.v1", as_json=args.json + ) + if args.command == "adjudication" and args.adjudication_cmd == "validate": + return _cmd_va_validate( + args.path, expected_type="AdjudicationRecord.v1", as_json=args.json + ) + if args.command == "assurance" and args.assurance_cmd == "build-report": + return cmd_assurance_build_report( + campaign=args.campaign, + results=args.results, + adjudications=args.adjudications, + out=args.out, + report_id=args.report_id, + created_at=args.created_at, + source_commit=args.source_commit, + release_grade=args.release_grade, + as_json=args.json, + ) + if args.command == "assurance" and args.assurance_cmd == "verify-report": + return cmd_assurance_verify_report(args.path, as_json=args.json) parser.print_help() return 2 diff --git a/python/pcs_core/conformance.py b/python/pcs_core/conformance.py index 65a77b4b..361cfdde 100644 --- a/python/pcs_core/conformance.py +++ b/python/pcs_core/conformance.py @@ -910,6 +910,143 @@ def _check_pf_core_generated_lean_proof(errors: list[str], checks: int) -> int: return checks +@_record("verifier-assurance") +def _suite_verifier_assurance() -> tuple[list[str], list[str], int]: + from pcs_core.hash import canonical_hash + from pcs_core.verifier_assurance_report import ( + build_assurance_report, + report_body_without_integrity, + verify_assurance_report, + ) + from pcs_core.verifier_assurance_validate import ( + SemanticIssue, + check_verifier_assurance_invalid_fixtures, + check_verifier_assurance_valid_fixtures, + iter_va_example_dirs, + load_va_context_from_dir, + validate_va_semantics, + ) + + errors = check_verifier_assurance_valid_fixtures() + errors.extend(check_verifier_assurance_invalid_fixtures()) + checks = len(iter_va_example_dirs("valid")) + len(iter_va_example_dirs("invalid")) + + # Report rebuild determinism (same inputs → identical body excluding integrity). + rebuild = repo_root() / "examples" / "verifier_assurance" / "valid" / "report_rebuild" + if rebuild.is_dir(): + checks += 1 + campaign = json.loads((rebuild / "campaign.json").read_text(encoding="utf-8")) + results = [ + json.loads(p.read_text(encoding="utf-8")) + for p in sorted((rebuild / "results").glob("*.json")) + ] + adjudications = [ + json.loads(p.read_text(encoding="utf-8")) + for p in sorted((rebuild / "adjudications").glob("*.json")) + ] + kwargs = { + "report_id": "rep-det", + "created_at": "2026-07-24T15:00:00Z", + "source_commit": "e068794683959c52a19594a6d271dd5e69f3c999", + "release_grade": True, + "excluded_items": [{"item_id": "ex-1", "reason_code": "out_of_scope"}], + "unadjudicated_items": [], + "applicability_limits": ["synthetic fixture only"], + } + a = build_assurance_report( + campaign=campaign, results=results, adjudications=adjudications, **kwargs + ) + b = build_assurance_report( + campaign=campaign, results=results, adjudications=adjudications, **kwargs + ) + if report_body_without_integrity(a) != report_body_without_integrity(b): + errors.append("report rebuild body not deterministic") + if a["integrity"]["artifact_digest"] != b["integrity"]["artifact_digest"]: + errors.append("report rebuild digest not deterministic") + golden = json.loads((rebuild / "report.json").read_text(encoding="utf-8")) + if verify_assurance_report(golden): + errors.append("golden report_rebuild/report.json failed verify") + tampered = dict(golden) + tampered["report_id"] = "tampered" + tamper_codes = { + i.code if isinstance(i, SemanticIssue) else str(i) + for i in verify_assurance_report(tampered) + } + if "ReportDigestMismatch" not in tamper_codes: + errors.append("tampered golden report did not yield ReportDigestMismatch") + checks += 2 + + # Producer dialect gates under benchmarks/verifier_assurance_conformance. + bench = repo_root() / "benchmarks" / "verifier_assurance_conformance" + if bench.is_dir(): + from pcs_core.validate_detect import validate_schema + + for case_dir in sorted((bench / "invalid").iterdir() if (bench / "invalid").is_dir() else []): + if not case_dir.is_dir() or not (case_dir / "manifest.json").is_file(): + continue + checks += 1 + manifest = json.loads((case_dir / "manifest.json").read_text(encoding="utf-8")) + artifact = json.loads( + (case_dir / str(manifest.get("artifact_file") or "artifact.json")).read_text( + encoding="utf-8" + ) + ) + artifact_type = str(manifest["artifact_type"]) + expected = str(manifest["expected_error"]) + schema_errors = validate_schema(artifact, artifact_type) + context = load_va_context_from_dir(case_dir) + semantic = validate_va_semantics( + artifact, artifact_type, as_issues=True, context=context + ) + joined = " ".join(schema_errors) + " " + " ".join(str(i) for i in semantic) + codes = { + i.code if isinstance(i, SemanticIssue) else str(i) for i in semantic + } + if expected not in codes and expected not in joined: + errors.append( + f"benchmark {case_dir.name}: expected {expected!r}, got {sorted(codes)}" + ) + + for path in sorted((bench / "valid").rglob("*.json")) if (bench / "valid").is_dir() else []: + if path.name == "manifest.json": + continue + checks += 1 + data = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(data, dict) or "artifact_type" not in data: + continue + artifact_type = str(data["artifact_type"]) + schema_errors = validate_schema(data, artifact_type) + semantic = validate_va_semantics(data, artifact_type, as_issues=True) + if schema_errors or semantic: + errors.append(f"benchmark valid {path}: {schema_errors or semantic}") + + # Shared hash vectors for the six core types. + from pcs_core.shared_hash_vectors import VECTOR_SPECS, load_vector + + for artifact_type in ( + "VerifierProfile.v1", + "VerificationResult.v1", + "VerifierInvocationRecord.v1", + "VerifierReplayReport.v1", + "VerifierMutationManifest.v1", + "RewardEvidenceEnvelope.v1", + "OptimizationCampaignManifest.v1", + "AdjudicationRecord.v1", + "VerifierAssuranceReport.v1", + ): + if artifact_type not in VECTOR_SPECS: + errors.append(f"missing shared hash vector for {artifact_type}") + continue + checks += 1 + vector = load_vector(artifact_type) + input_rel = vector.get("input") or vector.get("input_file") + data = json.loads((repo_root() / str(input_rel)).read_text(encoding="utf-8")) + if canonical_hash(data) != vector["expected_digest"]: + errors.append(f"hash vector mismatch for {artifact_type}") + + return errors, [], max(checks, 1) + + @_record("pf-core-cross-language") def _suite_pf_core_cross_language() -> tuple[list[str], list[str], int]: import subprocess diff --git a/python/pcs_core/registry_data.py b/python/pcs_core/registry_data.py index ea82b376..055d1c7b 100644 --- a/python/pcs_core/registry_data.py +++ b/python/pcs_core/registry_data.py @@ -1396,6 +1396,205 @@ def _pf_core_release_entry( consumer_repos=[PCS_CORE], release_mode_required=False, ), + "VerifierProfile.v1": _entry( + artifact_type="VerifierProfile.v1", + schema="schemas/VerifierProfile.v1.schema.json", + schema_owner=PCS_CORE, + runtime_producer="OVK", + allowed_runtime_producers=["OVK", PCS_CORE], + allowed_statuses=["Draft", "Validated", "Deprecated"], + required_release_fields=[ + "schema_version", + "artifact_type", + "verifier_profile_id", + "source_commit", + "configuration", + "integrity", + ], + semantic_checks=[ + _sc("explicit_artifact_type", "release_blocking", PCS_CORE), + _sc("full_source_commit", "release_blocking", PCS_CORE), + _sc("null_config_digest_slots", "release_blocking", PCS_CORE), + _sc("no_signature_or_digest", "release_blocking", PCS_CORE), + ], + consumer_repos=[PCS_CORE, "OVK", LABTRUST], + ), + "VerificationResult.v1": _entry( + artifact_type="VerificationResult.v1", + schema="schemas/VerificationResult.v1.schema.json", + schema_owner=PCS_CORE, + runtime_producer="OVK", + allowed_runtime_producers=["OVK", PCS_CORE], + allowed_statuses=["Draft", "Validated", "Deprecated"], + required_release_fields=[ + "schema_version", + "artifact_type", + "verification_result_id", + "decision", + "execution_status", + "integrity", + ], + semantic_checks=[ + _sc("fail_closed_decision", "release_blocking", PCS_CORE), + _sc("typed_indeterminate_or_failure", "release_blocking", PCS_CORE), + _sc("raw_normalized_digests", "release_blocking", PCS_CORE), + _sc("no_guarantee_upgrade", "release_blocking", PCS_CORE), + _sc("no_signature_or_digest", "release_blocking", PCS_CORE), + ], + consumer_repos=[PCS_CORE, "OVK", LABTRUST], + ), + + "VerifierInvocationRecord.v1": _entry( + artifact_type="VerifierInvocationRecord.v1", + schema="schemas/VerifierInvocationRecord.v1.schema.json", + schema_owner=PCS_CORE, + runtime_producer="OVK", + allowed_runtime_producers=["OVK", PCS_CORE], + allowed_statuses=["Draft", "Validated", "Deprecated"], + required_release_fields=[ + "schema_version", + "artifact_type", + "invocation_id", + "profile_ref", + "integrity", + ], + semantic_checks=[ + _sc("explicit_artifact_type", "release_blocking", PCS_CORE), + _sc("raw_normalized_digests", "release_blocking", PCS_CORE), + _sc("no_signature_or_digest", "release_blocking", PCS_CORE), + ], + consumer_repos=[PCS_CORE, "OVK"], + ), + "VerifierReplayReport.v1": _entry( + artifact_type="VerifierReplayReport.v1", + schema="schemas/VerifierReplayReport.v1.schema.json", + schema_owner=PCS_CORE, + runtime_producer="OVK", + allowed_runtime_producers=["OVK", PCS_CORE], + allowed_statuses=["Draft", "Validated", "Deprecated"], + required_release_fields=[ + "schema_version", + "artifact_type", + "replay_id", + "replay_status", + "integrity", + ], + semantic_checks=[ + _sc("explicit_artifact_type", "release_blocking", PCS_CORE), + _sc("replay_matched_consistency", "release_blocking", PCS_CORE), + _sc("no_signature_or_digest", "release_blocking", PCS_CORE), + ], + consumer_repos=[PCS_CORE, "OVK"], + ), + "VerifierMutationManifest.v1": _entry( + artifact_type="VerifierMutationManifest.v1", + schema="schemas/VerifierMutationManifest.v1.schema.json", + schema_owner=PCS_CORE, + runtime_producer="OVK", + allowed_runtime_producers=["OVK", PCS_CORE], + allowed_statuses=["Draft", "Validated", "Deprecated"], + required_release_fields=[ + "schema_version", + "artifact_type", + "mutation_id", + "mutation_class", + "production_prohibition", + "integrity", + ], + semantic_checks=[ + _sc("explicit_artifact_type", "release_blocking", PCS_CORE), + _sc("production_prohibition_true", "release_blocking", PCS_CORE), + _sc("no_signature_or_digest", "release_blocking", PCS_CORE), + ], + consumer_repos=[PCS_CORE, "OVK"], + ), + "RewardEvidenceEnvelope.v1": _entry( + artifact_type="RewardEvidenceEnvelope.v1", + schema="schemas/RewardEvidenceEnvelope.v1.schema.json", + schema_owner=PCS_CORE, + runtime_producer=LABTRUST, + allowed_runtime_producers=[LABTRUST, PCS_CORE], + allowed_statuses=["Draft", "Validated", "Deprecated"], + required_release_fields=[ + "schema_version", + "artifact_type", + "reward_envelope_id", + "scalar_total", + "integrity", + ], + semantic_checks=[ + _sc("reward_composition", "release_blocking", PCS_CORE), + _sc("claims_need_verifier_refs", "release_blocking", PCS_CORE), + _sc("revoked_profile_gate", "release_blocking", PCS_CORE), + _sc("no_signature_or_digest", "release_blocking", PCS_CORE), + ], + consumer_repos=[PCS_CORE, LABTRUST], + ), + "OptimizationCampaignManifest.v1": _entry( + artifact_type="OptimizationCampaignManifest.v1", + schema="schemas/OptimizationCampaignManifest.v1.schema.json", + schema_owner=PCS_CORE, + runtime_producer=PCS_CORE, + allowed_runtime_producers=[PCS_CORE, "OVK"], + allowed_statuses=["Draft", "Validated", "Deprecated"], + required_release_fields=[ + "schema_version", + "artifact_type", + "campaign_id", + "access_class", + "integrity", + ], + semantic_checks=[ + _sc("access_class_required", "release_blocking", PCS_CORE), + _sc("cohort_compute_exposure", "release_blocking", PCS_CORE), + _sc("no_signature_or_digest", "release_blocking", PCS_CORE), + ], + consumer_repos=[PCS_CORE, "OVK"], + ), + "AdjudicationRecord.v1": _entry( + artifact_type="AdjudicationRecord.v1", + schema="schemas/AdjudicationRecord.v1.schema.json", + schema_owner=PCS_CORE, + runtime_producer=PCS_CORE, + allowed_runtime_producers=[PCS_CORE], + allowed_statuses=["Draft", "Validated", "Deprecated"], + required_release_fields=[ + "schema_version", + "artifact_type", + "adjudication_id", + "label", + "protected_rationale", + "integrity", + ], + semantic_checks=[ + _sc("rationale_commitment", "release_blocking", PCS_CORE), + _sc("independence_for_release_grade", "release_blocking", PCS_CORE), + _sc("no_signature_or_digest", "release_blocking", PCS_CORE), + ], + consumer_repos=[PCS_CORE], + ), + "VerifierAssuranceReport.v1": _entry( + artifact_type="VerifierAssuranceReport.v1", + schema="schemas/VerifierAssuranceReport.v1.schema.json", + schema_owner=PCS_CORE, + runtime_producer=PCS_CORE, + allowed_runtime_producers=[PCS_CORE], + allowed_statuses=["Draft", "Validated", "Deprecated"], + required_release_fields=[ + "schema_version", + "artifact_type", + "report_id", + "metrics", + "integrity", + ], + semantic_checks=[ + _sc("aggregate_count_reconcile", "release_blocking", PCS_CORE), + _sc("ci_methods_declared", "release_blocking", PCS_CORE), + _sc("release_grade_adjudication", "release_blocking", PCS_CORE), + _sc("no_signature_or_digest", "release_blocking", PCS_CORE), + ], + consumer_repos=[PCS_CORE, "OVK", LABTRUST], + ), } diff --git a/python/pcs_core/shared_hash_vectors.py b/python/pcs_core/shared_hash_vectors.py index daf8f458..dd34e555 100644 --- a/python/pcs_core/shared_hash_vectors.py +++ b/python/pcs_core/shared_hash_vectors.py @@ -28,6 +28,15 @@ "ComputationRunReceipt.v0": "examples/computation_run_receipt.valid.json", "ResultArtifact.v0": "examples/result_artifact.valid.json", "ComputationWitness.v0": "examples/computation_witness.valid.json", + "VerifierProfile.v1": "examples/verifier_assurance/valid/profile_basic/profile.json", + "VerificationResult.v1": "examples/verifier_assurance/valid/result_accept/result.json", + "VerifierInvocationRecord.v1": "examples/verifier_assurance/valid/invocation_basic/invocation.json", + "VerifierReplayReport.v1": "examples/verifier_assurance/valid/replay_matched/replay.json", + "VerifierMutationManifest.v1": "examples/verifier_assurance/valid/mutation_timeout/mutation.json", + "RewardEvidenceEnvelope.v1": "examples/verifier_assurance/valid/reward_scalar/reward.json", + "OptimizationCampaignManifest.v1": "examples/verifier_assurance/valid/campaign_basic/campaign.json", + "AdjudicationRecord.v1": "examples/verifier_assurance/valid/adjudication_basic/adjudication.json", + "VerifierAssuranceReport.v1": "examples/verifier_assurance/valid/report_rebuild/report.json", } VECTOR_FILENAMES: dict[str, str] = { @@ -49,6 +58,15 @@ "ComputationRunReceipt.v0": "computation_run_receipt.vector.json", "ResultArtifact.v0": "result_artifact.vector.json", "ComputationWitness.v0": "computation_witness.vector.json", + "VerifierProfile.v1": "verifierprofile_v1.vector.json", + "VerificationResult.v1": "verificationresult_v1.vector.json", + "VerifierInvocationRecord.v1": "verifierinvocationrecord_v1.vector.json", + "VerifierReplayReport.v1": "verifierreplayreport_v1.vector.json", + "VerifierMutationManifest.v1": "verifiermutationmanifest_v1.vector.json", + "RewardEvidenceEnvelope.v1": "rewardevidenceenvelope_v1.vector.json", + "OptimizationCampaignManifest.v1": "optimizationcampaignmanifest_v1.vector.json", + "AdjudicationRecord.v1": "adjudicationrecord_v1.vector.json", + "VerifierAssuranceReport.v1": "verifierassurancereport_v1.vector.json", } @@ -88,7 +106,7 @@ def write_shared_vectors(*, force: bool = False) -> None: data = json.loads(example.read_text(encoding="utf-8")) payload = { "artifact_type": artifact_type, - "input_file": relative, + "input_file": relative.replace("\\", "/"), "expected_digest": canonical_hash(data), "canonical_json": canonical_json_bytes(data).decode("utf-8"), } @@ -99,7 +117,9 @@ def verify_shared_vectors() -> list[str]: errors: list[str] = [] for artifact_type, relative in VECTOR_SPECS.items(): vector = load_vector(artifact_type) - example_name = str(vector.get("input", vector.get("input_file", relative))) + example_name = str(vector.get("input", vector.get("input_file", relative))).replace( + "\\", "/" + ) if not example_name.startswith("examples/"): example_name = relative data = json.loads(_example_path(example_name).read_text(encoding="utf-8")) diff --git a/python/pcs_core/validate_detect.py b/python/pcs_core/validate_detect.py index 794053b1..93e5d470 100644 --- a/python/pcs_core/validate_detect.py +++ b/python/pcs_core/validate_detect.py @@ -129,6 +129,15 @@ class DetectionMode(str, Enum): "FormatAssertionProbe.v0": "FormatAssertionProbe.v0.schema.json", "ExternalAttestation.v0": "ExternalAttestation.v0.schema.json", "ReleaseProvenanceBinding.v0": "ReleaseProvenanceBinding.v0.schema.json", + "VerifierProfile.v1": "VerifierProfile.v1.schema.json", + "VerificationResult.v1": "VerificationResult.v1.schema.json", + "VerifierInvocationRecord.v1": "VerifierInvocationRecord.v1.schema.json", + "VerifierReplayReport.v1": "VerifierReplayReport.v1.schema.json", + "VerifierMutationManifest.v1": "VerifierMutationManifest.v1.schema.json", + "RewardEvidenceEnvelope.v1": "RewardEvidenceEnvelope.v1.schema.json", + "OptimizationCampaignManifest.v1": "OptimizationCampaignManifest.v1.schema.json", + "AdjudicationRecord.v1": "AdjudicationRecord.v1.schema.json", + "VerifierAssuranceReport.v1": "VerifierAssuranceReport.v1.schema.json", } @@ -208,7 +217,7 @@ def _detect_artifact_type_heuristic(data: dict[str, Any]) -> str | None: return "SignedScienceClaimBundle.v0" if "bundle_id" in data and "claim_artifact" in data: return "ScienceClaimBundle.v0" - if "verification_id" in data: + if "verification_id" in data and data.get("schema_version") != "v1": return "VerificationResult.v0" if "receipt_id" in data: return "RuntimeReceipt.v0" @@ -465,7 +474,7 @@ def _detect_artifact_type_heuristic(data: dict[str, Any]) -> str | None: return "SignedScienceClaimBundle.v0" if "bundle_id" in data and "claim_artifact" in data: return "ScienceClaimBundle.v0" - if "verification_id" in data: + if "verification_id" in data and data.get("schema_version") != "v1": return "VerificationResult.v0" if "receipt_id" in data: return "RuntimeReceipt.v0" @@ -520,6 +529,18 @@ def _detect_artifact_type_heuristic(data: dict[str, Any]) -> str | None: and "signature" in data ): return "ArtifactIntegrity.v1" + if data.get("schema_version") == "v1" and data.get("artifact_type") in { + "VerifierProfile.v1", + "VerificationResult.v1", + "VerifierInvocationRecord.v1", + "VerifierReplayReport.v1", + "VerifierMutationManifest.v1", + "RewardEvidenceEnvelope.v1", + "OptimizationCampaignManifest.v1", + "AdjudicationRecord.v1", + "VerifierAssuranceReport.v1", + }: + return str(data["artifact_type"]) if ( data.get("schema_version") == "v0" and data.get("artifact_type") == "TrustedKeyRegistry.v0" diff --git a/python/pcs_core/validate_semantics.py b/python/pcs_core/validate_semantics.py index 5599abfe..4edb6769 100644 --- a/python/pcs_core/validate_semantics.py +++ b/python/pcs_core/validate_semantics.py @@ -74,6 +74,19 @@ def validate_semantics(data: dict[str, Any], artifact_type: str) -> list[str]: errors: list[str] = [] + if artifact_type in { + "VerifierProfile.v1", + "VerificationResult.v1", + "RewardEvidenceEnvelope.v1", + "OptimizationCampaignManifest.v1", + "AdjudicationRecord.v1", + "VerifierAssuranceReport.v1", + }: + from pcs_core.verifier_assurance_validate import validate_va_semantics + + errors.extend(validate_va_semantics(data, artifact_type)) + return errors + if artifact_type == "ArtifactRegistry.v0": errors.extend(validate_artifact_registry_semantics(data)) return errors @@ -457,6 +470,15 @@ def check_valid_examples(examples_dir: Path | None = None) -> None: for path in sorted(ingest_examples.glob("*.pcs_bench_ingest.valid.json")): validate_file(path) + from pcs_core.verifier_assurance_validate import check_verifier_assurance_valid_fixtures + + va_errors = check_verifier_assurance_valid_fixtures() + if va_errors: + raise ValidationError( + "Verifier Assurance valid fixtures failed", + errors=va_errors, + ) + check_pf_core_valid_fixtures() @@ -484,3 +506,11 @@ def check_invalid_examples(examples_dir: Path | None = None) -> None: if not schema_errors and not semantic_errors: raise ValidationError(f"Expected {filename} to fail validation") check_pf_core_invalid_fixtures() + from pcs_core.verifier_assurance_validate import check_verifier_assurance_invalid_fixtures + + va_invalid_errors = check_verifier_assurance_invalid_fixtures() + if va_invalid_errors: + raise ValidationError( + "Verifier Assurance invalid fixtures failed", + errors=va_invalid_errors, + ) \ No newline at end of file diff --git a/python/pcs_core/verifier_assurance_report.py b/python/pcs_core/verifier_assurance_report.py new file mode 100644 index 00000000..20d8eca1 --- /dev/null +++ b/python/pcs_core/verifier_assurance_report.py @@ -0,0 +1,430 @@ +"""Deterministic VerifierAssuranceReport.v1 builder (offline, fail-closed).""" + +from __future__ import annotations + +import json +from decimal import Decimal, ROUND_HALF_EVEN +from pathlib import Path +from typing import Any + +from pcs_core.hash import CANONICALIZATION_VERSION, canonical_hash +from pcs_core.verifier_assurance_validate import ( + SemanticIssue, + _INDETERMINATE, + validate_adjudication_record_semantics, + validate_assurance_report_semantics, + validate_campaign_manifest_semantics, + validate_verification_result_semantics, +) + + +def attach_nested_integrity(data: dict[str, Any]) -> dict[str, Any]: + """Attach nested ArtifactIntegrity.v1-pattern integrity envelope.""" + body = {k: v for k, v in data.items() if k != "integrity"} + digest = canonical_hash(body) + out = dict(body) + out["integrity"] = { + "canonicalization_version": CANONICALIZATION_VERSION, + "artifact_digest": digest, + } + return out + + +class ReportBuildError(ValueError): + """Raised when report construction fails closed.""" + + def __init__(self, issues: list[SemanticIssue] | list[str]) -> None: + self.issues = issues + message = "; ".join(str(i) for i in issues) + super().__init__(message) + + +def _rate(numerator: int, denominator: int, *, method: str = "wilson") -> dict[str, Any]: + if denominator < 0 or numerator < 0: + raise ReportBuildError( + [ + SemanticIssue( + "InvalidRateCounts", + "metrics", + "numerator/denominator must be non-negative", + ) + ] + ) + if denominator == 0: + rate = "0" + else: + rate = str( + (Decimal(numerator) / Decimal(denominator)).quantize( + Decimal("0.000001"), rounding=ROUND_HALF_EVEN + ) + ) + # Placeholder CI bounds equal rate when sample is empty; parameters declared explicitly. + return { + "rate": rate if denominator else "0", + "numerator": numerator, + "denominator": denominator, + "confidence_interval": { + "method": method, + "parameters": {"alpha": "0.05", "declared": True}, + "lower": rate if denominator else "0", + "upper": rate if denominator else "0", + "sample_size": denominator, + }, + } + + +def _load_json_dir(path: Path) -> list[dict[str, Any]]: + if not path.is_dir(): + raise ReportBuildError( + [SemanticIssue("MissingDirectory", str(path), "required input directory missing")] + ) + items: list[dict[str, Any]] = [] + for file_path in sorted(path.glob("*.json")): + data = json.loads(file_path.read_text(encoding="utf-8")) + if isinstance(data, dict): + items.append(data) + return items + + +def _decision_bucket(status_or_decision: str) -> str: + if status_or_decision in {"accept", "accepted"}: + return "accept" + if status_or_decision in {"reject", "rejected"}: + return "reject" + if ( + status_or_decision in _INDETERMINATE + or status_or_decision == "indeterminate" + or str(status_or_decision).startswith("indeterminate_") + or status_or_decision == "failed" + ): + return "indeterminate" + return "indeterminate" + + +def build_assurance_report( + *, + campaign: dict[str, Any], + results: list[dict[str, Any]], + adjudications: list[dict[str, Any]], + report_id: str, + created_at: str, + producer: str = "pcs-core", + producer_version: str = "0.1.0", + source_repo: str = "https://github.com/SentinelOps-CI/pcs-core", + source_commit: str, + release_grade: bool = False, + excluded_items: list[dict[str, str]] | None = None, + unadjudicated_items: list[dict[str, str]] | None = None, + preregistration_ref: str | None = None, + applicability_limits: list[str] | None = None, +) -> dict[str, Any]: + """Build a VerifierAssuranceReport.v1 body (integrity attached at end). + + Fail closed: never infer denominators from missing cohort/adjudication data. + Indeterminate decisions are never counted as accept/reject. + """ + issues: list[SemanticIssue] = [] + campaign_issues = validate_campaign_manifest_semantics(campaign, as_issues=True) + assert isinstance(campaign_issues, list) + issues.extend(campaign_issues) # type: ignore[arg-type] + + if not results: + issues.append( + SemanticIssue( + "MissingResults", + "results", + "results cohort is required; denominators are never inferred from absence", + ) + ) + if release_grade and not adjudications: + issues.append( + SemanticIssue( + "MissingAdjudications", + "adjudications", + "release-grade reports require adjudication records", + ) + ) + + for index, result in enumerate(results): + result_issues = validate_verification_result_semantics(result, as_issues=True) + assert isinstance(result_issues, list) + for issue in result_issues: + assert isinstance(issue, SemanticIssue) + issues.append( + SemanticIssue(issue.code, f"results[{index}].{issue.path}", issue.message) + ) + + independent = False + for index, adj in enumerate(adjudications): + adj_issues = validate_adjudication_record_semantics( + adj, release_grade=release_grade, as_issues=True + ) + assert isinstance(adj_issues, list) + for issue in adj_issues: + assert isinstance(issue, SemanticIssue) + issues.append( + SemanticIssue(issue.code, f"adjudications[{index}].{issue.path}", issue.message) + ) + if adj.get("independence_declared") is True: + independent = True + + if release_grade and not independent: + issues.append( + SemanticIssue( + "ReleaseGradeNeedsIndependentAdjudication", + "independent_adjudication", + "release-grade reports require at least one independent adjudication", + ) + ) + + if issues: + raise ReportBuildError(issues) + + excluded_items = list(excluded_items or []) + unadjudicated_items = list(unadjudicated_items or []) + + # Map adjudications by subject id for FAR/FRR style counts. + adj_by_subject: dict[str, str] = {} + for adj in adjudications: + subject = adj.get("subject") + if isinstance(subject, dict) and isinstance(subject.get("artifact_id"), str): + adj_by_subject[subject["artifact_id"]] = str(adj.get("label")) + + false_accept = 0 + false_reject = 0 + abstain = 0 + adjudicated = 0 + accept_n = 0 + reject_n = 0 + indeterminate_n = 0 + + for result in results: + decision = str(result.get("status") or result.get("decision") or "") + bucket = _decision_bucket(decision) + if bucket == "accept": + accept_n += 1 + elif bucket == "reject": + reject_n += 1 + else: + indeterminate_n += 1 + abstain += 1 + result_id = str(result.get("result_id") or result.get("verification_result_id") or "") + label = adj_by_subject.get(result_id) + if label is None: + continue + adjudicated += 1 + if bucket == "accept" and label == "invalid": + false_accept += 1 + if bucket == "reject" and label == "valid": + false_reject += 1 + + sample_size = len(results) + # Never invent adjudication denominators: coverage uses available adjudications only. + far = _rate(false_accept, adjudicated if adjudicated else 0) + frr = _rate(false_reject, adjudicated if adjudicated else 0) + abstention = _rate(abstain, sample_size) + coverage = _rate(adjudicated, sample_size) + + campaign_cohorts = campaign.get("cohorts") or [] + if not isinstance(campaign_cohorts, list) or not campaign_cohorts: + raise ReportBuildError( + [ + SemanticIssue( + "MissingCohorts", + "campaign.cohorts", + "campaign must declare cohorts; counts are never inferred from missing records", + ) + ] + ) + + report_cohorts: list[dict[str, Any]] = [] + ordinary_accept = 0 + ordinary_n = 0 + optimized_accept = 0 + optimized_n = 0 + + # Assign all results to the first matching cohort kind by declaration order. + # Fail closed if aggregate counts cannot reconcile with included records. + remaining = list(results) + for cohort in campaign_cohorts: + if not isinstance(cohort, dict): + continue + kind = str(cohort.get("cohort_kind") or "other") + # Without per-result cohort tags, bind all results into declared cohorts only when + # a single cohort exists; multiple cohorts require explicit result.cohort_id. + assigned: list[dict[str, Any]] = [] + still: list[dict[str, Any]] = [] + for result in remaining: + result_cohort = result.get("cohort_id") + if result_cohort is None and len(campaign_cohorts) == 1: + assigned.append(result) + elif result_cohort == cohort.get("cohort_id"): + assigned.append(result) + else: + still.append(result) + remaining = still + + a = r = i = 0 + for result in assigned: + bucket = _decision_bucket(str(result.get("status") or result.get("decision") or "")) + if bucket == "accept": + a += 1 + elif bucket == "reject": + r += 1 + else: + i += 1 + included = len(assigned) + if a + r + i != included: + raise ReportBuildError( + [ + SemanticIssue( + "CohortCountMismatch", + f"cohorts[{cohort.get('cohort_id')}]", + "aggregate counts must reconcile exactly with included records", + ) + ] + ) + if kind == "ordinary": + ordinary_accept += a + ordinary_n += included + if kind == "optimized": + optimized_accept += a + optimized_n += included + report_cohorts.append( + { + "cohort_id": cohort["cohort_id"], + "cohort_kind": kind, + "access_class": cohort["access_class"], + "compute_exposure": cohort["compute_exposure"], + "included_result_count": included, + "accept_count": a, + "reject_count": r, + "indeterminate_count": i, + } + ) + + if remaining: + raise ReportBuildError( + [ + SemanticIssue( + "UnassignedResults", + "results", + "every result must map to a declared campaign cohort", + ) + ] + ) + + ordinary_rate = _rate(ordinary_accept, ordinary_n) + optimized_rate = _rate(optimized_accept, optimized_n) + if ordinary_n and optimized_n: + gap = str( + ( + Decimal(optimized_rate["rate"]) - Decimal(ordinary_rate["rate"]) + ).quantize(Decimal("0.000001"), rounding=ROUND_HALF_EVEN) + ) + else: + gap = "0" + + campaign_digest = canonical_hash(campaign) + report: dict[str, Any] = { + "schema_version": "v1", + "artifact_type": "VerifierAssuranceReport.v1", + "report_id": report_id, + "created_at": created_at, + "producer": producer, + "producer_version": producer_version, + "source_repo": source_repo, + "source_commit": source_commit, + "campaign_ref": { + "artifact_type": "OptimizationCampaignManifest.v1", + "artifact_id": campaign["campaign_id"], + "artifact_digest": campaign_digest, + }, + "release_grade": release_grade, + "independent_adjudication": independent, + "metrics": { + "false_accept_rate": far, + "false_reject_rate": frr, + "abstention_rate": abstention, + "adjudication_coverage": coverage, + "ordinary_accept_rate": ordinary_rate, + "optimized_accept_rate": optimized_rate, + "optimization_gap": gap, + "time_to_first_exploit_seconds": None, + "queries_to_first_exploit": None, + "exploit_family_counts": {}, + "inter_verifier_disagreement_rate": _rate(0, adjudicated if adjudicated else 0), + "reward_invalidation_rate": _rate(0, sample_size), + "latency_ms_p50": 0, + "cost_decimal": "0", + "sample_size": sample_size, + "excluded_count": len(excluded_items), + "unadjudicated_count": len(unadjudicated_items), + }, + "cohorts": report_cohorts, + "excluded_items": excluded_items, + "unadjudicated_items": unadjudicated_items, + "applicability_limits": list(applicability_limits or []), + } + if preregistration_ref: + report["preregistration_ref"] = preregistration_ref + + report = attach_nested_integrity(report) + semantic = validate_assurance_report_semantics(report, as_issues=True) + assert isinstance(semantic, list) + if semantic: + raise ReportBuildError(semantic) # type: ignore[arg-type] + + return report + + +def build_assurance_report_from_paths( + *, + campaign_path: Path, + results_dir: Path, + adjudications_dir: Path, + report_id: str, + created_at: str, + source_commit: str, + release_grade: bool = False, + out_path: Path | None = None, +) -> dict[str, Any]: + campaign = json.loads(campaign_path.read_text(encoding="utf-8")) + results = _load_json_dir(results_dir) + adjudications = _load_json_dir(adjudications_dir) + report = build_assurance_report( + campaign=campaign, + results=results, + adjudications=adjudications, + report_id=report_id, + created_at=created_at, + source_commit=source_commit, + release_grade=release_grade, + ) + if out_path is not None: + out_path.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return report + + +def verify_assurance_report(report: dict[str, Any]) -> list[SemanticIssue]: + issues = validate_assurance_report_semantics(report, as_issues=True) + assert isinstance(issues, list) + typed = [i for i in issues if isinstance(i, SemanticIssue)] + integrity = report.get("integrity") + if isinstance(integrity, dict): + body = {k: v for k, v in report.items() if k != "integrity"} + expected = canonical_hash(body) + got = integrity.get("artifact_digest") + if got != expected: + typed.append( + SemanticIssue( + "ReportDigestMismatch", + "integrity.artifact_digest", + "artifact_digest does not match report body", + ) + ) + return typed + + +def report_body_without_integrity(report: dict[str, Any]) -> dict[str, Any]: + return {k: v for k, v in report.items() if k != "integrity"} diff --git a/python/pcs_core/verifier_assurance_validate.py b/python/pcs_core/verifier_assurance_validate.py new file mode 100644 index 00000000..1ec5278f --- /dev/null +++ b/python/pcs_core/verifier_assurance_validate.py @@ -0,0 +1,1162 @@ +"""Semantic validation for Verifier Assurance (VA) *.v1 artifacts.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from decimal import Decimal, InvalidOperation +from pathlib import Path +from typing import Any + +from pcs_core.hash import SIGNATURE_FIELD, canonical_hash +from pcs_core.paths import examples_dir, repo_root +from pcs_core.validate_detect import ValidationError, validate_schema + +VA_ARTIFACT_TYPES = frozenset( + { + "VerifierProfile.v1", + "VerificationResult.v1", + "VerifierInvocationRecord.v1", + "VerifierReplayReport.v1", + "VerifierMutationManifest.v1", + "RewardEvidenceEnvelope.v1", + "OptimizationCampaignManifest.v1", + "AdjudicationRecord.v1", + "VerifierAssuranceReport.v1", + } +) + +# OVK pin surface: profile, result, invocation, replay, mutation. +OVK_VA_ARTIFACT_TYPES = frozenset( + { + "VerifierProfile.v1", + "VerificationResult.v1", + "VerifierInvocationRecord.v1", + "VerifierReplayReport.v1", + "VerifierMutationManifest.v1", + } +) + +_INDETERMINATE = frozenset( + { + "indeterminate", + "indeterminate_insufficient_evidence", + "indeterminate_execution_error", + "indeterminate_out_of_scope", + "indeterminate_configuration_drift", + # Legacy / producer dialects collapsed to indeterminate buckets + "indeterminate_missing_checker", + "indeterminate_timeout", + "indeterminate_parser_failure", + "indeterminate_unsupported_input", + "indeterminate_external_service_error", + "indeterminate_missing_evidence", + "indeterminate_domain_disagreement", + } +) + + +@dataclass(frozen=True) +class VaValidationContext: + """Optional multi-artifact context for cross-file semantic rules.""" + + profiles: tuple[dict[str, Any], ...] = () + results: tuple[dict[str, Any], ...] = () + campaign: dict[str, Any] | None = None + rewards: tuple[dict[str, Any], ...] = () + + +def _profile_body_digest(profile: dict[str, Any]) -> str: + body = {k: v for k, v in profile.items() if k != "integrity"} + return canonical_hash(body) + + +def _index_profiles( + profiles: tuple[dict[str, Any], ...] | list[dict[str, Any]], +) -> dict[str, dict[str, Any]]: + out: dict[str, dict[str, Any]] = {} + for profile in profiles: + pid = profile.get("verifier_profile_id") + if isinstance(pid, str): + out[pid] = profile + return out + + +def _index_results( + results: tuple[dict[str, Any], ...] | list[dict[str, Any]], +) -> dict[str, dict[str, Any]]: + out: dict[str, dict[str, Any]] = {} + for result in results: + rid = result.get("verification_result_id") or result.get("result_id") + if isinstance(rid, str): + out[rid] = result + return out + +_GUARANTEE_RANK: dict[str, int] = { + "unchecked_advisory": 0, + "observational": 1, + "runtime_observed": 1, + "empirically_measured": 2, + "human_reviewed": 3, + "certificate_checked": 4, + "formally_checked": 5, +} + +_SECRET_KEY_HINTS = ( + "password", + "passwd", + "secret", + "token", + "api_key", + "apikey", + "authorization", + "private_key", + "access_key", +) + + +@dataclass(frozen=True) +class SemanticIssue: + code: str + path: str + message: str + + def __str__(self) -> str: + return f"{self.code} at {self.path}: {self.message}" + + +def _as_strings(issues: list[SemanticIssue] | list[str], *, as_issues: bool) -> list[Any]: + if as_issues: + return issues + return [str(i) for i in issues] + + +def _forbid_legacy_signature_field( + data: dict[str, Any], artifact_type: str +) -> list[SemanticIssue]: + if SIGNATURE_FIELD in data: + return [ + SemanticIssue( + "LegacySignatureOrDigest", + SIGNATURE_FIELD, + f"{artifact_type}: {SIGNATURE_FIELD} is forbidden on VA *.v1 roots", + ) + ] + return [] + + +def _require_integrity(data: dict[str, Any], artifact_type: str) -> list[SemanticIssue]: + integrity = data.get("integrity") + if not isinstance(integrity, dict): + return [ + SemanticIssue( + "MissingIntegrity", + "integrity", + f"{artifact_type}: nested integrity envelope is required", + ) + ] + issues: list[SemanticIssue] = [] + if integrity.get("canonicalization_version") != "v1": + issues.append( + SemanticIssue( + "BadCanonicalizationVersion", + "integrity.canonicalization_version", + "must be v1", + ) + ) + digest = integrity.get("artifact_digest") + if not isinstance(digest, str) or not digest.startswith("sha256:"): + issues.append( + SemanticIssue( + "BadArtifactDigest", + "integrity.artifact_digest", + "must be sha256:<64 hex>", + ) + ) + return issues + + +def _check_zero_commit(commit: Any, path: str) -> list[SemanticIssue]: + if isinstance(commit, str) and commit == "0" * 40: + return [ + SemanticIssue("PlaceholderCommit", path, "placeholder zero git commit rejected") + ] + return [] + + +def _check_secret_leaks(env_entries: Any, path: str) -> list[SemanticIssue]: + issues: list[SemanticIssue] = [] + if not isinstance(env_entries, dict): + return issues + for key, value in env_entries.items(): + key_l = str(key).lower() + if any(hint in key_l for hint in _SECRET_KEY_HINTS): + issues.append( + SemanticIssue( + "SecretKeyName", + f"{path}.{key}", + f"redacted_environment key {key!r} looks like a secret name", + ) + ) + if isinstance(value, str): + value_l = value.lower() + if value.startswith("sk-") or "begin private key" in value_l: + issues.append( + SemanticIssue( + "SecretValue", + f"{path}.{key}", + "value appears to contain a secret", + ) + ) + return issues + + +def _parse_decimal(value: Any, path: str) -> tuple[Decimal | None, list[SemanticIssue]]: + if not isinstance(value, str): + return None, [SemanticIssue("DecimalType", path, "decimal values must be strings")] + try: + return Decimal(value), [] + except (InvalidOperation, ValueError): + return None, [SemanticIssue("DecimalParse", path, f"invalid decimal string {value!r}")] + + +def validate_verifier_profile_semantics( + data: dict[str, Any], *, as_issues: bool = False +) -> list[Any]: + issues = _forbid_legacy_signature_field(data, "VerifierProfile.v1") + issues.extend(_require_integrity(data, "VerifierProfile.v1")) + commit = data.get("source_commit") + if not isinstance(commit, str) or len(commit) != 40 or any( + c not in "0123456789abcdef" for c in commit + ): + issues.append( + SemanticIssue( + "InvalidSourceCommit", + "source_commit", + "source_commit must be a full 40-char lowercase hex SHA", + ) + ) + issues.extend(_check_zero_commit(commit, "source_commit")) + configuration = data.get("configuration") + if isinstance(configuration, dict): + for key in ( + "policy_digest", + "model_digest", + "prompt_digest", + "resource_limit_digest", + ): + if key not in configuration: + issues.append( + SemanticIssue( + "MissingNullDigestSlot", + f"configuration.{key}", + "inapplicable config digests must be present as explicit null", + ) + ) + impl = data.get("implementation") + if isinstance(impl, dict) and not impl.get("implementation_digest"): + issues.append( + SemanticIssue( + "MissingImplementationDigest", + "implementation.implementation_digest", + "implementation_digest is required", + ) + ) + applicability = data.get("applicability") + if isinstance(applicability, dict): + status = applicability.get("status") + if status == "revoked" and not applicability.get("revocation_reason"): + issues.append( + SemanticIssue( + "MissingRevocationReason", + "applicability.revocation_reason", + "revoked profiles require revocation_reason", + ) + ) + if status == "superseded" and not applicability.get("superseded_by_profile_id"): + issues.append( + SemanticIssue( + "MissingSupersededBy", + "applicability.superseded_by_profile_id", + "superseded profiles require superseded_by_profile_id", + ) + ) + redacted = data.get("redacted_environment") + if isinstance(redacted, dict): + issues.extend(_check_secret_leaks(redacted.get("entries"), "redacted_environment.entries")) + schema_doc = data.get("configuration_schema") + schema_digest = data.get("configuration_schema_digest") + if isinstance(schema_doc, dict) and isinstance(schema_digest, str): + try: + recomputed = canonical_hash(schema_doc) + except Exception as exc: # noqa: BLE001 + issues.append( + SemanticIssue("ConfigSchemaUnhashable", "configuration_schema", str(exc)) + ) + else: + if recomputed != schema_digest: + issues.append( + SemanticIssue( + "ConfigSchemaDigestMismatch", + "configuration_schema_digest", + f"recorded {schema_digest!r} != recomputed {recomputed!r}", + ) + ) + return _as_strings(issues, as_issues=as_issues) + + +def validate_verification_result_semantics( + data: dict[str, Any], + *, + as_issues: bool = False, + context: VaValidationContext | None = None, +) -> list[Any]: + issues = _forbid_legacy_signature_field(data, "VerificationResult.v1") + issues.extend(_require_integrity(data, "VerificationResult.v1")) + commit = data.get("source_commit") + if not isinstance(commit, str) or len(commit) != 40 or any( + c not in "0123456789abcdef" for c in commit + ): + issues.append( + SemanticIssue( + "InvalidSourceCommit", + "source_commit", + "source_commit must be a full 40-char lowercase hex SHA", + ) + ) + decision = data.get("decision") + execution_status = data.get("execution_status") + fail_closed = { + "timeout", + "unavailable", + "malformed_input", + "unsupported_scope", + "error", + "cancelled", + "resource_exhausted", + } + if execution_status in fail_closed and decision in {"accept", "reject"}: + issues.append( + SemanticIssue( + "FailClosedDecision", + "decision", + f"execution_status {execution_status!r} cannot yield accept/reject", + ) + ) + if data.get("normalization_applied") is True: + raw = data.get("raw_backend_output_digest") + normalized = data.get("normalized_result_digest") + if not isinstance(raw, str) or not isinstance(normalized, str): + issues.append( + SemanticIssue( + "MissingNormalizationDigests", + "normalized_result_digest", + "normalization_applied requires raw and normalized digests", + ) + ) + elif raw == normalized: + issues.append( + SemanticIssue( + "IdenticalNormalizationDigests", + "normalized_result_digest", + "raw and normalized digests must be distinct when normalization occurs", + ) + ) + mandatory_failed = False + for g_index, group in enumerate(data.get("check_groups") or []): + if not isinstance(group, dict): + continue + for c_index, check in enumerate(group.get("checks") or []): + if not isinstance(check, dict): + continue + if check.get("mandatory") is True and check.get("status") == "failed": + mandatory_failed = True + if check.get("status") == "skipped" and not ( + check.get("reason_code") or check.get("skip_reason_code") + ): + issues.append( + SemanticIssue( + "MissingSkipReason", + f"check_groups[{g_index}].checks[{c_index}].reason_code", + "skipped checks require reason_code", + ) + ) + if decision == "accept" and mandatory_failed: + issues.append( + SemanticIssue( + "AcceptWithMandatoryFailure", + "decision", + "accept cannot coexist with a mandatory failed check", + ) + ) + declared = data.get("declared_input_guarantee_class") + result_class = data.get("guarantee_class") + if isinstance(declared, str) and isinstance(result_class, str): + if declared in _GUARANTEE_RANK and result_class in _GUARANTEE_RANK: + if _GUARANTEE_RANK[result_class] > _GUARANTEE_RANK[declared]: + issues.append( + SemanticIssue( + "GuaranteeUpgrade", + "guarantee_class", + f"must not upgrade declared_input_guarantee_class ({declared!r} -> {result_class!r})", + ) + ) + # Rule 1: profile refs resolve; digests match when profile bodies supplied. + if context is not None and context.profiles: + pref = data.get("verifier_profile") + if isinstance(pref, dict): + profile_id = pref.get("verifier_profile_id") + recorded = pref.get("profile_digest") + by_id = _index_profiles(context.profiles) + if not isinstance(profile_id, str) or profile_id not in by_id: + issues.append( + SemanticIssue( + "ProfileRefUnresolved", + "verifier_profile.verifier_profile_id", + f"profile {profile_id!r} not found in supplied profiles", + ) + ) + else: + recomputed = _profile_body_digest(by_id[profile_id]) + if isinstance(recorded, str) and recorded != recomputed: + issues.append( + SemanticIssue( + "ProfileDigestMismatch", + "verifier_profile.profile_digest", + f"recorded {recorded!r} != recomputed {recomputed!r}", + ) + ) + return _as_strings(issues, as_issues=as_issues) + + +def validate_verification_result_v1_semantics( + data: dict[str, Any], + *, + as_issues: bool = False, + context: VaValidationContext | None = None, +) -> list[Any]: + return validate_verification_result_semantics( + data, as_issues=as_issues, context=context + ) + + +def validate_reward_envelope_semantics( + data: dict[str, Any], + *, + as_issues: bool = False, + context: VaValidationContext | None = None, +) -> list[Any]: + issues = _forbid_legacy_signature_field(data, "RewardEvidenceEnvelope.v1") + issues.extend(_require_integrity(data, "RewardEvidenceEnvelope.v1")) + issues.extend(_check_zero_commit(data.get("source_commit"), "source_commit")) + total, total_issues = _parse_decimal(data.get("scalar_total"), "scalar_total") + issues.extend(total_issues) + components = data.get("components") + if isinstance(components, list) and total is not None: + composition = data.get("composition_function") + if composition == "sum": + acc = Decimal("0") + for index, component in enumerate(components): + if not isinstance(component, dict): + continue + value, value_issues = _parse_decimal( + component.get("value"), f"components[{index}].value" + ) + issues.extend(value_issues) + if value is not None: + acc += value + if acc != total: + issues.append( + SemanticIssue( + "RewardCompositionMismatch", + "scalar_total", + f"sum of components {acc} != scalar_total {total}", + ) + ) + claims_issued = data.get("claims_issued") or [] + if isinstance(claims_issued, list) and claims_issued: + refs = data.get("verifier_result_refs") or [] + if not isinstance(refs, list) or not refs: + issues.append( + SemanticIssue( + "ClaimsNeedVerifierRefs", + "verifier_result_refs", + "claims_issued requires at least one verifier_result_ref", + ) + ) + lifecycle = data.get("lifecycle") if isinstance(data.get("lifecycle"), dict) else {} + lifecycle_status = lifecycle.get("status") + migration = lifecycle.get("migration_record_id") + for index, pref in enumerate(data.get("profile_refs") or []): + if not isinstance(pref, dict): + continue + status = pref.get("applicability_status") + if status in {"revoked", "expired"} and lifecycle_status == "active" and not migration: + issues.append( + SemanticIssue( + "RevokedProfileGate", + f"profile_refs[{index}]", + f"{status} profiles cannot support new active rewards without migration_record_id", + ) + ) + elif status == "revoked" and lifecycle_status != "active": + # Still gate revoked refs on non-active envelopes that claim issued positive rewards. + if claims_issued: + issues.append( + SemanticIssue( + "RevokedProfileGate", + f"profile_refs[{index}]", + "revoked profiles cannot authorize reward claims", + ) + ) + mandatory_unresolved = data.get("mandatory_unresolved_claim_ids") or [] + if ( + lifecycle_status == "active" + and isinstance(mandatory_unresolved, list) + and mandatory_unresolved + ): + issues.append( + SemanticIssue( + "ActiveRewardUnresolvedClaims", + "mandatory_unresolved_claim_ids", + "unresolved mandatory claims block active release-grade envelopes", + ) + ) + # Rule 2: reward trajectory digests match supporting results when results supplied. + if context is not None and context.results: + by_id = _index_results(context.results) + reward_traj = data.get("trajectory_digest") + for index, ref in enumerate(data.get("verifier_result_refs") or []): + if not isinstance(ref, dict): + continue + rid = ref.get("artifact_id") + if not isinstance(rid, str) or rid not in by_id: + issues.append( + SemanticIssue( + "RewardResultRefUnresolved", + f"verifier_result_refs[{index}]", + f"result {rid!r} not found in supplied results", + ) + ) + continue + result = by_id[rid] + result_traj = result.get("trajectory_digest") + if ( + isinstance(reward_traj, str) + and isinstance(result_traj, str) + and reward_traj != result_traj + ): + issues.append( + SemanticIssue( + "RewardTrajectoryMismatch", + f"verifier_result_refs[{index}]", + "reward trajectory_digest must match supporting VerificationResult", + ) + ) + # Rule 10: env/policy/model/verifier/campaign versions consistent when cross-checked. + if context is not None and context.campaign is not None: + camp_env = context.campaign.get("env_profile_id") + camp_ver = context.campaign.get("env_profile_version") + if ( + isinstance(camp_env, str) + and isinstance(data.get("env_profile_id"), str) + and camp_env != data.get("env_profile_id") + ): + issues.append( + SemanticIssue( + "CrossArtifactVersionMismatch", + "env_profile_id", + "reward env_profile_id must match campaign env_profile_id", + ) + ) + if ( + isinstance(camp_ver, str) + and isinstance(data.get("env_profile_version"), str) + and camp_ver != data.get("env_profile_version") + ): + issues.append( + SemanticIssue( + "CrossArtifactVersionMismatch", + "env_profile_version", + "reward env_profile_version must match campaign env_profile_version", + ) + ) + if context is not None and context.profiles: + by_profile = _index_profiles(context.profiles) + for p_index, pref in enumerate(data.get("profile_refs") or []): + if not isinstance(pref, dict): + continue + pid = pref.get("verifier_profile_id") + recorded = pref.get("profile_digest") + if not isinstance(pid, str) or pid not in by_profile: + issues.append( + SemanticIssue( + "ProfileRefUnresolved", + f"profile_refs[{p_index}]", + f"profile {pid!r} not found in supplied profiles", + ) + ) + elif isinstance(recorded, str): + recomputed = _profile_body_digest(by_profile[pid]) + if recorded != recomputed: + issues.append( + SemanticIssue( + "ProfileDigestMismatch", + f"profile_refs[{p_index}].profile_digest", + f"recorded {recorded!r} != recomputed {recomputed!r}", + ) + ) + status = by_profile[pid].get("applicability", {}) + if isinstance(status, dict): + app_status = status.get("status") + if ( + app_status in {"revoked", "expired"} + and lifecycle_status == "active" + and not migration + ): + issues.append( + SemanticIssue( + "RevokedProfileGate", + f"profile_refs[{p_index}]", + f"profile body status {app_status!r} blocks active rewards", + ) + ) + return _as_strings(issues, as_issues=as_issues) + + +def validate_campaign_manifest_semantics( + data: dict[str, Any], *, as_issues: bool = False +) -> list[Any]: + issues = _forbid_legacy_signature_field(data, "OptimizationCampaignManifest.v1") + issues.extend(_require_integrity(data, "OptimizationCampaignManifest.v1")) + issues.extend(_check_zero_commit(data.get("source_commit"), "source_commit")) + if not data.get("access_class"): + issues.append( + SemanticIssue("AccessClassRequired", "access_class", "access_class is required") + ) + cohorts = data.get("cohorts") + if isinstance(cohorts, list): + for index, cohort in enumerate(cohorts): + if not isinstance(cohort, dict): + continue + exposure = cohort.get("compute_exposure") + if not isinstance(exposure, dict): + issues.append( + SemanticIssue( + "CohortComputeExposure", + f"cohorts[{index}].compute_exposure", + "compute_exposure is required", + ) + ) + if not cohort.get("access_class"): + issues.append( + SemanticIssue( + "CohortAccessClass", + f"cohorts[{index}].access_class", + "every cohort must declare access_class", + ) + ) + return _as_strings(issues, as_issues=as_issues) + + +def validate_adjudication_record_semantics( + data: dict[str, Any], + *, + release_grade: bool = False, + as_issues: bool = False, +) -> list[Any]: + issues = _forbid_legacy_signature_field(data, "AdjudicationRecord.v1") + issues.extend(_require_integrity(data, "AdjudicationRecord.v1")) + issues.extend(_check_zero_commit(data.get("source_commit"), "source_commit")) + protected = data.get("protected_rationale") + if isinstance(protected, dict): + if not protected.get("commitment_digest"): + issues.append( + SemanticIssue( + "RationaleCommitment", + "protected_rationale.commitment_digest", + "commitment_digest is required", + ) + ) + if release_grade and data.get("independence_declared") is not True: + issues.append( + SemanticIssue( + "IndependenceForReleaseGrade", + "independence_declared", + "release-grade adjudication requires independence_declared=true", + ) + ) + return _as_strings(issues, as_issues=as_issues) + + +def validate_assurance_report_semantics( + data: dict[str, Any], *, as_issues: bool = False +) -> list[Any]: + issues = _forbid_legacy_signature_field(data, "VerifierAssuranceReport.v1") + issues.extend(_require_integrity(data, "VerifierAssuranceReport.v1")) + issues.extend(_check_zero_commit(data.get("source_commit"), "source_commit")) + metrics = data.get("metrics") + if isinstance(metrics, dict): + sample = metrics.get("sample_size") + excluded = metrics.get("excluded_count") + unadj = metrics.get("unadjudicated_count") + if ( + isinstance(sample, int) + and isinstance(excluded, int) + and isinstance(unadj, int) + and excluded + unadj > sample + and sample > 0 + ): + issues.append( + SemanticIssue( + "AggregateCountReconcile", + "metrics", + "excluded_count + unadjudicated_count exceeds sample_size", + ) + ) + for key in ( + "false_accept_rate", + "false_reject_rate", + "abstention_rate", + "adjudication_coverage", + ): + rate = metrics.get(key) + if isinstance(rate, dict): + ci = rate.get("confidence_interval") + if not isinstance(ci, dict) or not ci.get("method"): + issues.append( + SemanticIssue( + "CIMethodsDeclared", + f"metrics.{key}.confidence_interval.method", + "CI method must be declared", + ) + ) + elif not isinstance(ci.get("parameters"), dict): + issues.append( + SemanticIssue( + "CIParametersDeclared", + f"metrics.{key}.confidence_interval.parameters", + "CI parameters must be declared (no silent denominator invention)", + ) + ) + gap = metrics.get("optimization_gap") + gap_nonzero = isinstance(gap, str) and gap not in {"0", "0.0", "0.000000"} + ordinary = metrics.get("ordinary_accept_rate") + optimized = metrics.get("optimized_accept_rate") + both_cohort_rates = ( + isinstance(ordinary, dict) + and isinstance(optimized, dict) + and (ordinary.get("denominator") or 0) > 0 + and (optimized.get("denominator") or 0) > 0 + ) + claims_gap = gap_nonzero or both_cohort_rates + else: + claims_gap = False + + # Rule 12: excluded/unadjudicated remain visible and count-matched. + excluded_items = data.get("excluded_items") + unadj_items = data.get("unadjudicated_items") + if isinstance(metrics, dict): + if isinstance(excluded_items, list) and isinstance(metrics.get("excluded_count"), int): + if len(excluded_items) != metrics["excluded_count"]: + issues.append( + SemanticIssue( + "ExcludedItemsVisible", + "excluded_items", + "excluded_count must equal len(excluded_items)", + ) + ) + elif metrics.get("excluded_count"): + issues.append( + SemanticIssue( + "ExcludedItemsVisible", + "excluded_items", + "excluded_count > 0 requires visible excluded_items", + ) + ) + if isinstance(unadj_items, list) and isinstance(metrics.get("unadjudicated_count"), int): + if len(unadj_items) != metrics["unadjudicated_count"]: + issues.append( + SemanticIssue( + "UnadjudicatedItemsVisible", + "unadjudicated_items", + "unadjudicated_count must equal len(unadjudicated_items)", + ) + ) + elif metrics.get("unadjudicated_count"): + issues.append( + SemanticIssue( + "UnadjudicatedItemsVisible", + "unadjudicated_items", + "unadjudicated_count > 0 requires visible unadjudicated_items", + ) + ) + + if data.get("release_grade") is True and data.get("independent_adjudication") is not True: + issues.append( + SemanticIssue( + "ReleaseGradeAdjudication", + "independent_adjudication", + "release-grade reports require independent_adjudication=true", + ) + ) + + cohorts = data.get("cohorts") or [] + has_ordinary = False + has_optimized = False + for index, cohort in enumerate(cohorts): + if not isinstance(cohort, dict): + continue + kind = cohort.get("cohort_kind") + if kind == "ordinary": + has_ordinary = True + if kind == "optimized": + has_optimized = True + if not cohort.get("access_class"): + issues.append( + SemanticIssue( + "CohortAccessClass", + f"cohorts[{index}].access_class", + "every cohort must declare access_class", + ) + ) + exposure = cohort.get("compute_exposure") + if not isinstance(exposure, dict): + issues.append( + SemanticIssue( + "CohortComputeExposure", + f"cohorts[{index}].compute_exposure", + "every cohort must declare compute_exposure", + ) + ) + accept = cohort.get("accept_count") + reject = cohort.get("reject_count") + indeterminate = cohort.get("indeterminate_count") + included = cohort.get("included_result_count") + if all(isinstance(v, int) for v in (accept, reject, indeterminate, included)): + if accept + reject + indeterminate != included: # type: ignore[operator] + issues.append( + SemanticIssue( + "CohortCountMismatch", + f"cohorts[{index}]", + "aggregate counts must reconcile exactly with included records", + ) + ) + if accept < 0 or reject < 0 or indeterminate < 0: # type: ignore[operator] + issues.append( + SemanticIssue( + "IndeterminateMisclassification", + f"cohorts[{index}]", + "cohort decision counts must be non-negative distinct buckets", + ) + ) + # Rule 15: indeterminate never folded into accept/reject (detect over-claim). + if included > 0 and indeterminate == 0 and accept + reject > included: # type: ignore[operator] + issues.append( + SemanticIssue( + "IndeterminateMisclassification", + f"cohorts[{index}]", + "indeterminate decisions must not be counted as accept/reject", + ) + ) + # Rule 5: optimization-gap needs ordinary + optimized cohorts. + if claims_gap and not (has_ordinary and has_optimized): + issues.append( + SemanticIssue( + "OptimizationGapCohorts", + "cohorts", + "optimization-gap metrics require ordinary and optimized cohorts", + ) + ) + return _as_strings(issues, as_issues=as_issues) + + +def validate_va_semantics( + data: dict[str, Any], + artifact_type: str, + *, + as_issues: bool = False, + context: VaValidationContext | None = None, +) -> list[str] | list[SemanticIssue]: + """Dispatch used by validate_semantics for all VA *.v1 types.""" + return validate_verifier_assurance_semantics( + data, artifact_type, as_issues=as_issues, context=context + ) + + + +def validate_invocation_record_semantics( + data: dict[str, Any], *, as_issues: bool = False +) -> list[Any]: + issues = _forbid_legacy_signature_field(data, "VerifierInvocationRecord.v1") + issues.extend(_require_integrity(data, "VerifierInvocationRecord.v1")) + raw = data.get("raw_backend_result_digest") + normalized = data.get("normalized_result_digest") + if ( + isinstance(raw, str) + and isinstance(normalized, str) + and raw == normalized + and data.get("normalizer_version") + ): + issues.append( + SemanticIssue( + "IdenticalNormalizationDigests", + "normalized_result_digest", + "when normalization is applied, raw and normalized digests must differ", + ) + ) + return _as_strings(issues, as_issues=as_issues) + + +def validate_replay_report_semantics( + data: dict[str, Any], *, as_issues: bool = False +) -> list[Any]: + issues = _forbid_legacy_signature_field(data, "VerifierReplayReport.v1") + issues.extend(_require_integrity(data, "VerifierReplayReport.v1")) + drift = data.get("drift") if isinstance(data.get("drift"), dict) else {} + status = data.get("replay_status") + if status == "matched" and ( + drift.get("raw_digest_match") is False + or drift.get("normalized_digest_match") is False + or data.get("original_raw_digest") != data.get("replay_raw_digest") + or data.get("original_normalized_digest") != data.get("replay_normalized_digest") + ): + issues.append( + SemanticIssue( + "ReplayMatchedWithDrift", + "replay_status", + "matched status requires matching digests and drift flags true", + ) + ) + return _as_strings(issues, as_issues=as_issues) + + +def validate_mutation_manifest_semantics( + data: dict[str, Any], *, as_issues: bool = False +) -> list[Any]: + issues = _forbid_legacy_signature_field(data, "VerifierMutationManifest.v1") + issues.extend(_require_integrity(data, "VerifierMutationManifest.v1")) + if data.get("production_prohibition") is not True: + issues.append( + SemanticIssue( + "ProductionProhibitionRequired", + "production_prohibition", + "production_prohibition must be true", + ) + ) + return _as_strings(issues, as_issues=as_issues) + + +def validate_verifier_assurance_semantics( + data: dict[str, Any], + artifact_type: str, + *, + as_issues: bool = False, + context: VaValidationContext | None = None, +) -> list[str] | list[SemanticIssue]: + if artifact_type == "VerifierProfile.v1": + return validate_verifier_profile_semantics(data, as_issues=as_issues) + if artifact_type == "VerificationResult.v1": + return validate_verification_result_semantics( + data, as_issues=as_issues, context=context + ) + if artifact_type == "VerifierInvocationRecord.v1": + return validate_invocation_record_semantics(data, as_issues=as_issues) + if artifact_type == "VerifierReplayReport.v1": + return validate_replay_report_semantics(data, as_issues=as_issues) + if artifact_type == "VerifierMutationManifest.v1": + return validate_mutation_manifest_semantics(data, as_issues=as_issues) + if artifact_type == "RewardEvidenceEnvelope.v1": + return validate_reward_envelope_semantics( + data, as_issues=as_issues, context=context + ) + if artifact_type == "OptimizationCampaignManifest.v1": + return validate_campaign_manifest_semantics(data, as_issues=as_issues) + if artifact_type == "AdjudicationRecord.v1": + return validate_adjudication_record_semantics(data, as_issues=as_issues) + if artifact_type == "VerifierAssuranceReport.v1": + return validate_assurance_report_semantics(data, as_issues=as_issues) + msg = f"unknown verifier-assurance artifact type: {artifact_type}" + if as_issues: + return [SemanticIssue("UnknownArtifactType", "artifact_type", msg)] + return [msg] + + + + + +def load_va_context_from_dir(case_dir: Path) -> VaValidationContext: + """Load optional profile/results/campaign siblings for multi-file fixtures.""" + profiles: list[dict[str, Any]] = [] + results: list[dict[str, Any]] = [] + rewards: list[dict[str, Any]] = [] + campaign: dict[str, Any] | None = None + profile_path = case_dir / "profile.json" + if profile_path.is_file(): + profiles.append(json.loads(profile_path.read_text(encoding="utf-8"))) + campaign_path = case_dir / "campaign.json" + if campaign_path.is_file(): + campaign = json.loads(campaign_path.read_text(encoding="utf-8")) + results_dir = case_dir / "results" + if results_dir.is_dir(): + for path in sorted(results_dir.glob("*.json")): + item = json.loads(path.read_text(encoding="utf-8")) + if isinstance(item, dict): + results.append(item) + result_path = case_dir / "result.json" + if result_path.is_file(): + item = json.loads(result_path.read_text(encoding="utf-8")) + if isinstance(item, dict): + results.append(item) + reward_path = case_dir / "reward.json" + if reward_path.is_file(): + item = json.loads(reward_path.read_text(encoding="utf-8")) + if isinstance(item, dict): + rewards.append(item) + return VaValidationContext( + profiles=tuple(profiles), + results=tuple(results), + campaign=campaign, + rewards=tuple(rewards), + ) + + +def verifier_assurance_examples_root() -> Path: + return examples_dir() / "verifier_assurance" + + +def list_verifier_assurance_valid_fixtures() -> list[Path]: + root = verifier_assurance_examples_root() + paths: list[Path] = [] + if root.is_dir(): + paths.extend(sorted(p for p in root.glob("*.valid.json") if p.is_file())) + for case_dir in iter_va_example_dirs("valid"): + paths.extend( + p for p in sorted(case_dir.rglob("*.json")) if p.name != "manifest.json" + ) + seen: set[Path] = set() + out: list[Path] = [] + for path in paths: + resolved = path.resolve() + if resolved not in seen: + seen.add(resolved) + out.append(path) + return out + + +def list_verifier_assurance_invalid_fixtures() -> list[Path]: + """Flat invalid JSON files only (manifest-based dirs handled separately).""" + # Invalid VA cases live under examples/verifier_assurance/invalid// + # with manifest.json; there is no flat invalid_* directory for the six-artifact family. + return [] + + +def iter_va_example_dirs(kind: str) -> list[Path]: + candidate = examples_dir() / "verifier_assurance" / kind + if candidate.is_dir(): + return sorted(path for path in candidate.iterdir() if path.is_dir()) + return [] + + +def load_va_fixture_manifest(case_dir: Path) -> dict[str, Any]: + manifest_path = case_dir / "manifest.json" + if not manifest_path.is_file(): + raise ValidationError(f"Missing manifest.json in {case_dir}") + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + if not isinstance(manifest, dict): + raise ValidationError(f"manifest.json root must be an object in {case_dir}") + return manifest + + +def _expected_artifact_type_from_name(path: Path) -> str | None: + name = path.name + for artifact_type in sorted(VA_ARTIFACT_TYPES, key=len, reverse=True): + if name.startswith(artifact_type): + return artifact_type + return None + + +def check_va_valid_fixtures() -> None: + from pcs_core.validate_semantics import validate_artifact + + flat = list_verifier_assurance_valid_fixtures() + if not flat and not iter_va_example_dirs("valid"): + raise ValidationError("missing verifier-assurance valid fixtures") + for path in flat: + if path.is_dir(): + continue + data = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(data, dict): + raise ValidationError(f"{path}: root must be object") + artifact_type = data.get("artifact_type") or _expected_artifact_type_from_name(path) + if not isinstance(artifact_type, str): + raise ValidationError(f"{path}: missing artifact_type") + validate_artifact(data, artifact_type, release_grade=True) + + +def check_va_invalid_fixtures() -> None: + flat_files = list_verifier_assurance_invalid_fixtures() + if flat_files: + for path in flat_files: + data = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(data, dict): + raise ValidationError(f"{path}: root must be object") + artifact_type = data.get("artifact_type") or _expected_artifact_type_from_name( + path + ) + if not isinstance(artifact_type, str) or artifact_type not in VA_ARTIFACT_TYPES: + raise ValidationError(f"{path}: could not resolve VA artifact_type") + schema_errors = validate_schema(data, artifact_type) + semantic_errors = validate_verifier_assurance_semantics(data, artifact_type) + if not schema_errors and not semantic_errors: + raise ValidationError(f"{path}: expected validation failure") + + for case_dir in iter_va_example_dirs("invalid"): + # Skip flat-file directories (no manifest). + if not (case_dir / "manifest.json").is_file(): + continue + from pcs_core.validate_detect import detect_artifact_type + + manifest = load_va_fixture_manifest(case_dir) + expected_error = str(manifest["expected_error"]) + artifact_file = str(manifest.get("artifact_file") or "artifact.json") + artifact_type = str(manifest.get("artifact_type") or "") + path = case_dir / artifact_file + data = json.loads(path.read_text(encoding="utf-8")) + use_type = artifact_type or detect_artifact_type(data) + if not use_type: + raise ValidationError(f"{case_dir.name}: could not detect artifact type") + context = load_va_context_from_dir(case_dir) + schema_errors = validate_schema(data, use_type) + semantic_issues = validate_va_semantics( + data, use_type, as_issues=True, context=context + ) + assert isinstance(semantic_issues, list) + codes = { + (i.code if isinstance(i, SemanticIssue) else str(i)) for i in semantic_issues + } + joined = " ".join(schema_errors) + " " + " ".join(str(i) for i in semantic_issues) + if expected_error not in codes and expected_error not in joined: + raise ValidationError( + f"{case_dir.name}: expected error {expected_error!r}, " + f"got codes={sorted(c for c in codes if c)} schema={schema_errors}" + ) + + +def check_verifier_assurance_valid_fixtures() -> list[str]: + try: + check_va_valid_fixtures() + return [] + except ValidationError as exc: + return [str(exc), *list(exc.errors)] + + +def check_verifier_assurance_invalid_fixtures() -> list[str]: + try: + check_va_invalid_fixtures() + return [] + except ValidationError as exc: + return [str(exc), *list(exc.errors)] + +def profile_digest(profile: dict[str, Any]) -> str: + return canonical_hash(profile) diff --git a/python/tests/test_verifier_assurance.py b/python/tests/test_verifier_assurance.py new file mode 100644 index 00000000..2ee8ebc9 --- /dev/null +++ b/python/tests/test_verifier_assurance.py @@ -0,0 +1,162 @@ +"""Verifier Assurance (PCS-VA) focused tests.""" + +from __future__ import annotations + +import json +from copy import deepcopy +from pathlib import Path + +import pytest + +from pcs_core.hash import canonical_hash +from pcs_core.paths import examples_dir, repo_root +from pcs_core.validate import validate_artifact, validate_file +from pcs_core.verifier_assurance_report import ( + build_assurance_report, + report_body_without_integrity, +) +from pcs_core.verifier_assurance_validate import ( + check_va_invalid_fixtures, + check_va_valid_fixtures, + profile_digest, + validate_verification_result_semantics, + validate_verifier_profile_semantics, +) + + +def test_va_valid_and_invalid_fixtures() -> None: + check_va_valid_fixtures() + check_va_invalid_fixtures() + + +def test_profile_config_substitution_changes_digest() -> None: + path = examples_dir() / "verifier_assurance" / "valid" / "profile_basic" / "profile.json" + profile = json.loads(path.read_text(encoding="utf-8")) + d1 = profile_digest(profile) + altered = deepcopy(profile) + altered["configuration"]["config_digest"] = "sha256:" + ("f" * 64) + altered.pop("integrity", None) + d2 = profile_digest(altered) + assert d1 != d2 + + +def test_fail_closed_timeout_cannot_accept() -> None: + path = examples_dir() / "verifier_assurance" / "valid" / "result_accept" / "result.json" + result = json.loads(path.read_text(encoding="utf-8")) + result["execution_status"] = "timeout" + result["decision"] = "accept" + issues = validate_verification_result_semantics(result, as_issues=True) + codes = {getattr(i, "code", None) for i in issues} + assert "FailClosedDecision" in codes + + +def test_v0_verification_result_still_validates() -> None: + path = examples_dir() / "labtrust" / "invalid_failed_verification_result.json" + # invalid fixture should fail; pick a valid v0 if present + candidates = list((examples_dir()).glob("*verification*valid*.json")) + candidates += list((examples_dir() / "labtrust-release").rglob("*VerificationResult*")) + # At minimum ensure v0 schema remains registered and distinct + from pcs_core.validate_detect import ARTIFACT_SCHEMAS + + assert "VerificationResult.v0" in ARTIFACT_SCHEMAS + assert "VerificationResult.v1" in ARTIFACT_SCHEMAS + assert ARTIFACT_SCHEMAS["VerificationResult.v0"] != ARTIFACT_SCHEMAS["VerificationResult.v1"] + + +def test_report_rebuild_deterministic() -> None: + base = examples_dir() / "verifier_assurance" / "valid" / "report_rebuild" + campaign = json.loads((base / "campaign.json").read_text(encoding="utf-8")) + results = [ + json.loads(p.read_text(encoding="utf-8")) + for p in sorted((base / "results").glob("*.json")) + ] + adjudications = [ + json.loads(p.read_text(encoding="utf-8")) + for p in sorted((base / "adjudications").glob("*.json")) + ] + kwargs = { + "report_id": "rep-det", + "created_at": "2026-07-24T15:00:00Z", + "source_commit": "e068794683959c52a19594a6d271dd5e69f3c999", + "release_grade": True, + "excluded_items": [{"item_id": "ex-1", "reason_code": "out_of_scope"}], + "unadjudicated_items": [], + "applicability_limits": ["synthetic fixture only"], + } + a = build_assurance_report( + campaign=campaign, results=results, adjudications=adjudications, **kwargs + ) + b = build_assurance_report( + campaign=campaign, results=results, adjudications=adjudications, **kwargs + ) + assert report_body_without_integrity(a) == report_body_without_integrity(b) + assert a["integrity"]["artifact_digest"] == b["integrity"]["artifact_digest"] + + +def test_cli_profile_validate(tmp_path: Path) -> None: + from pcs_core.cli import main + + profile = examples_dir() / "verifier_assurance" / "valid" / "profile_basic" / "profile.json" + assert main(["verifier", "profile", "validate", str(profile)]) == 0 + + +def test_profile_digest_mismatch_with_context() -> None: + from pcs_core.verifier_assurance_validate import ( + load_va_context_from_dir, + validate_va_semantics, + ) + + case = examples_dir() / "verifier_assurance" / "invalid" / "profile_digest_mismatch" + data = json.loads((case / "artifact.json").read_text(encoding="utf-8")) + context = load_va_context_from_dir(case) + issues = validate_va_semantics( + data, "VerificationResult.v1", as_issues=True, context=context + ) + codes = {getattr(i, "code", None) for i in issues} + assert "ProfileDigestMismatch" in codes + + +def test_cli_json_diagnostics_structured() -> None: + from pcs_core.cli import main + import io + from contextlib import redirect_stdout + + path = ( + examples_dir() + / "verifier_assurance" + / "invalid" + / "timeout_accept" + / "artifact.json" + ) + buf = io.StringIO() + with redirect_stdout(buf): + code = main(["verifier", "result", "validate", str(path), "--json"]) + assert code == 1 + payload = json.loads(buf.getvalue()) + assert payload["ok"] is False + assert any(err.get("code") == "FailClosedDecision" for err in payload["errors"]) + + +def test_all_fifteen_rules_have_invalid_fixtures() -> None: + """Every semantic matrix rule has at least one dedicated invalid fixture.""" + required = { + "ProfileDigestMismatch": "profile_digest_mismatch", + "RewardTrajectoryMismatch": "reward_trajectory_mismatch", + "AcceptWithMandatoryFailure": "accept_mandatory_failure", + "ReleaseGradeAdjudication": "release_grade_no_adjudication", + "OptimizationGapCohorts": "optimization_gap_missing_cohort", + "CohortAccessClass": "cohort_missing_access", + "CohortCountMismatch": "cohort_count_mismatch", + "RevokedProfileGate": "revoked_profile_active_reward", + "IdenticalNormalizationDigests": "identical_normalization_digests", + "CrossArtifactVersionMismatch": "cross_artifact_version_mismatch", + "RationaleCommitment": "missing_rationale_commitment", + "ExcludedItemsVisible": "excluded_items_invisible", + "CIMethodsDeclared": "missing_ci_method", + "RewardCompositionMismatch": "reward_total_mismatch", + "IndeterminateMisclassification": "indeterminate_misclassification", + } + root = examples_dir() / "verifier_assurance" / "invalid" + for code, dirname in required.items(): + manifest = json.loads((root / dirname / "manifest.json").read_text(encoding="utf-8")) + assert manifest["expected_error"] == code, dirname \ No newline at end of file diff --git a/python/tests/test_verifier_assurance_cross_language.py b/python/tests/test_verifier_assurance_cross_language.py new file mode 100644 index 00000000..26f87ad9 --- /dev/null +++ b/python/tests/test_verifier_assurance_cross_language.py @@ -0,0 +1,173 @@ +"""Cross-language Verifier Assurance (PCS-VA-07) parity. + +Python is the reference for semantics and report verify; Rust/TS must emit the +same error codes on shared fixtures and match shared hash vectors. +""" + +from __future__ import annotations + +import json +import re +import subprocess +from pathlib import Path + +import pytest + +from pcs_core.hash import canonical_hash +from pcs_core.validate import ARTIFACT_SCHEMAS, detect_artifact_type, validate_semantics +from pcs_core.verifier_assurance_report import verify_assurance_report +from pcs_core.verifier_assurance_validate import ( + SemanticIssue, + validate_va_semantics, +) + +REPO = Path(__file__).resolve().parents[2] +VA_ROOT = REPO / "examples" / "verifier_assurance" +TS_SCHEMAS = REPO / "typescript" / "packages" / "core" / "src" / "schema.ts" +RUST_SCHEMAS = REPO / "rust" / "crates" / "pcs-core" / "src" / "validation.rs" + +VA_CORE_TYPES = ( + "VerifierProfile.v1", + "VerificationResult.v1", + "RewardEvidenceEnvelope.v1", + "OptimizationCampaignManifest.v1", + "AdjudicationRecord.v1", + "VerifierAssuranceReport.v1", +) + +INVALID_CASES: tuple[tuple[str, str], ...] = ( + ("invalid/timeout_accept", "FailClosedDecision"), + ("invalid/accept_mandatory_failure", "AcceptWithMandatoryFailure"), + ("invalid/identical_normalization_digests", "IdenticalNormalizationDigests"), + ("invalid/reward_total_mismatch", "RewardCompositionMismatch"), + ("invalid/revoked_profile_active_reward", "RevokedProfileGate"), + ("invalid/missing_rationale_commitment", "RationaleCommitment"), + ("invalid/short_source_commit", "InvalidSourceCommit"), + ("invalid/release_grade_no_adjudication", "ReleaseGradeAdjudication"), + ("invalid/optimization_gap_missing_cohort", "OptimizationGapCohorts"), + ("invalid/cohort_missing_access", "CohortAccessClass"), + ("invalid/cohort_count_mismatch", "CohortCountMismatch"), + ("invalid/excluded_items_invisible", "ExcludedItemsVisible"), + ("invalid/missing_ci_method", "CIMethodsDeclared"), + ("invalid/indeterminate_misclassification", "IndeterminateMisclassification"), + ("invalid/active_reward_unresolved", "ActiveRewardUnresolvedClaims"), + ("invalid/profile_digest_mismatch", "ProfileDigestMismatch"), + ("invalid/reward_trajectory_mismatch", "RewardTrajectoryMismatch"), + ("invalid/cross_artifact_version_mismatch", "CrossArtifactVersionMismatch"), +) + +VALID_SINGLE_ARTIFACTS: tuple[tuple[str, str], ...] = ( + ("valid/profile_basic/profile.json", "VerifierProfile.v1"), + ("valid/result_accept/result.json", "VerificationResult.v1"), + ("valid/reward_scalar/reward.json", "RewardEvidenceEnvelope.v1"), + ("valid/campaign_basic/campaign.json", "OptimizationCampaignManifest.v1"), + ("valid/adjudication_basic/adjudication.json", "AdjudicationRecord.v1"), + ("valid/report_rebuild/report.json", "VerifierAssuranceReport.v1"), +) + + +def _load(rel: str) -> dict: + return json.loads((VA_ROOT / rel).read_text(encoding="utf-8")) + + +def _codes(issues: list) -> set[str]: + out: set[str] = set() + for item in issues: + if isinstance(item, SemanticIssue): + out.add(item.code) + else: + text = str(item) + out.add(text.split(" at ", 1)[0] if " at " in text else text) + return out + + +def _extract_quoted_types(path: Path, marker: str) -> set[str]: + text = path.read_text(encoding="utf-8") + block = text.split(marker, 1)[-1] + patterns = ( + r'"(Verifier[^"]+\.v1)"', + r'"(VerificationResult\.v1)"', + r'"(RewardEvidenceEnvelope\.v1)"', + r'"(OptimizationCampaignManifest\.v1)"', + r'"(AdjudicationRecord\.v1)"', + ) + found: set[str] = set() + for pattern in patterns: + found.update(re.findall(pattern, block)) + return found + + +def test_va_core_types_registered_python() -> None: + for artifact_type in VA_CORE_TYPES: + assert artifact_type in ARTIFACT_SCHEMAS + + +def test_va_core_types_registered_typescript() -> None: + ts_types = _extract_quoted_types(TS_SCHEMAS, "const ARTIFACT_SCHEMAS") + assert set(VA_CORE_TYPES).issubset(ts_types) + + +def test_va_core_types_registered_rust() -> None: + rust_types = _extract_quoted_types(RUST_SCHEMAS, "const ARTIFACT_SCHEMAS") + assert set(VA_CORE_TYPES).issubset(rust_types) + + +@pytest.mark.parametrize("rel,artifact_type", VALID_SINGLE_ARTIFACTS) +def test_va_valid_semantics_and_detection(rel: str, artifact_type: str) -> None: + data = _load(rel) + assert detect_artifact_type(data) == artifact_type + assert validate_semantics(data, artifact_type) == [] + assert validate_va_semantics(data, artifact_type, as_issues=True) == [] + + +@pytest.mark.parametrize("case_dir,expected_code", INVALID_CASES) +def test_va_invalid_semantics_codes(case_dir: str, expected_code: str) -> None: + from pcs_core.verifier_assurance_validate import load_va_context_from_dir + + manifest = _load(f"{case_dir}/manifest.json") + assert manifest["expected_error"] == expected_code + artifact_file = str(manifest.get("artifact_file") or "artifact.json") + artifact_type = str(manifest["artifact_type"]) + data = _load(f"{case_dir}/{artifact_file}") + context = load_va_context_from_dir(VA_ROOT / case_dir) + issues = validate_va_semantics(data, artifact_type, as_issues=True, context=context) + assert expected_code in _codes(issues), issues + + +def test_va_unknown_field_rejected_by_schema() -> None: + data = _load("invalid/unknown_field/artifact.json") + from pcs_core.validate_detect import validate_schema + + errors = validate_schema(data, "VerifierProfile.v1") + assert any("extra_field" in err or "Additional properties" in err for err in errors) + + +def test_va_verify_report_digest_parity() -> None: + report = _load("valid/report_rebuild/report.json") + assert verify_assurance_report(report) == [] + tampered = dict(report) + tampered["report_id"] = "tampered-id" + codes = _codes(verify_assurance_report(tampered)) + assert "ReportDigestMismatch" in codes + + +def test_va_shared_hash_vectors_exist() -> None: + from pcs_core.shared_hash_vectors import VECTOR_SPECS, load_vector + + for artifact_type in VA_CORE_TYPES: + assert artifact_type in VECTOR_SPECS + vector = load_vector(artifact_type) + input_rel = vector.get("input") or vector.get("input_file") + assert isinstance(input_rel, str) + data = json.loads((REPO / input_rel).read_text(encoding="utf-8")) + assert canonical_hash(data) == vector["expected_digest"] + + +def test_rust_va_suite_passes() -> None: + result = subprocess.run( + ["cargo", "test", "--locked", "va_", "--", "--nocapture"], + cwd=REPO / "rust", + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stdout + result.stderr diff --git a/rust/crates/pcs-core/src/lib.rs b/rust/crates/pcs-core/src/lib.rs index 952f4949..56474421 100644 --- a/rust/crates/pcs-core/src/lib.rs +++ b/rust/crates/pcs-core/src/lib.rs @@ -4,6 +4,7 @@ pub mod pf_core_catalog; pub mod schema; pub mod status; pub mod validation; +pub mod verifier_assurance; pub use hash::{ assert_canonical_number_policy, canonical_hash, canonical_hash_legacy, canonical_hash_release, @@ -27,3 +28,8 @@ pub use pf_core_catalog::{CAPABILITY_CATALOG, EFFECT_KINDS, ROLE_CAPABILITY_MAP, pub use validation::{ detect_artifact_type, validate_artifact, validate_semantics, ValidationError, }; +pub use verifier_assurance::{ + attach_nested_integrity, construct_va_artifact, is_va_artifact_type, + validate_assurance_report_semantics, validate_va_semantics, validate_va_semantics_strings, + verify_assurance_report, SemanticIssue, +}; diff --git a/rust/crates/pcs-core/src/validation.rs b/rust/crates/pcs-core/src/validation.rs index f387c188..0e0b1d41 100644 --- a/rust/crates/pcs-core/src/validation.rs +++ b/rust/crates/pcs-core/src/validation.rs @@ -64,6 +64,18 @@ const ARTIFACT_SCHEMAS: &[(&str, &str)] = &[ "PFCoreReleaseBundleManifest.v0.schema.json", ), ("ArtifactIntegrity.v1", "ArtifactIntegrity.v1.schema.json"), + ("VerifierProfile.v1", "VerifierProfile.v1.schema.json"), + ("VerificationResult.v1", "VerificationResult.v1.schema.json"), + ("VerifierInvocationRecord.v1", "VerifierInvocationRecord.v1.schema.json"), + ("VerifierReplayReport.v1", "VerifierReplayReport.v1.schema.json"), + ("VerifierMutationManifest.v1", "VerifierMutationManifest.v1.schema.json"), + ("RewardEvidenceEnvelope.v1", "RewardEvidenceEnvelope.v1.schema.json"), + ( + "OptimizationCampaignManifest.v1", + "OptimizationCampaignManifest.v1.schema.json", + ), + ("AdjudicationRecord.v1", "AdjudicationRecord.v1.schema.json"), + ("VerifierAssuranceReport.v1", "VerifierAssuranceReport.v1.schema.json"), ( "FormatAssertionProbe.v0", "FormatAssertionProbe.v0.schema.json", @@ -170,6 +182,12 @@ const EXPLICIT_ARTIFACT_TYPES: &[&str] = &[ "PFCoreKernelManifest.v0", "PFCoreReleaseBundleManifest.v0", "ArtifactIntegrity.v1", + "VerifierProfile.v1", + "VerificationResult.v1", + "RewardEvidenceEnvelope.v1", + "OptimizationCampaignManifest.v1", + "AdjudicationRecord.v1", + "VerifierAssuranceReport.v1", "FormatAssertionProbe.v0", "ExternalAttestation.v0", ]; @@ -615,6 +633,17 @@ pub fn validate_semantics(value: &Value, artifact_type: &str) -> Result<(), Vali if PROTOCOL_ARTIFACT_TYPES.contains(&artifact_type) { return Ok(()); } + // VA *.v1: mirror Python early-return (dedicated semantic module; skip generic commit walk). + if crate::verifier_assurance::is_va_artifact_type(artifact_type) { + let errors = crate::verifier_assurance::validate_va_semantics_strings(value, artifact_type); + return if errors.is_empty() { + Ok(()) + } else { + Err(ValidationError { + message: errors.join("; "), + }) + }; + } let mut errors = Vec::new(); check_source_commits(value, "", &mut errors, false); @@ -850,7 +879,8 @@ mod tests { let example_name = vector["input"] .as_str() .or_else(|| vector["input_file"].as_str()) - .unwrap(); + .unwrap() + .replace('\\', "/"); let example_path = if example_name.starts_with("examples/") { PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("../../..") diff --git a/rust/crates/pcs-core/src/verifier_assurance.rs b/rust/crates/pcs-core/src/verifier_assurance.rs new file mode 100644 index 00000000..91023450 --- /dev/null +++ b/rust/crates/pcs-core/src/verifier_assurance.rs @@ -0,0 +1,1184 @@ +//! Verifier Assurance (VA) *.v1 semantic validation and report verify. +//! Error codes and message shapes mirror `python/pcs_core/verifier_assurance_validate.py`. + +use serde_json::{json, Map, Value}; + +use crate::hash::{canonical_hash, SIGNATURE_FIELD}; + +const FAIL_CLOSED: &[&str] = &[ + "timeout", + "unavailable", + "malformed_input", + "unsupported_scope", + "error", + "cancelled", + "resource_exhausted", +]; + +const SECRET_KEY_HINTS: &[&str] = &[ + "password", + "passwd", + "secret", + "token", + "api_key", + "apikey", + "authorization", + "private_key", + "access_key", +]; + +fn guarantee_rank(class: &str) -> Option { + match class { + "unchecked_advisory" => Some(0), + "observational" | "runtime_observed" => Some(1), + "empirically_measured" => Some(2), + "human_reviewed" => Some(3), + "certificate_checked" => Some(4), + "formally_checked" => Some(5), + _ => None, + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SemanticIssue { + pub code: String, + pub path: String, + pub message: String, +} + +impl SemanticIssue { + pub fn new(code: impl Into, path: impl Into, message: impl Into) -> Self { + Self { + code: code.into(), + path: path.into(), + message: message.into(), + } + } + + pub fn format(&self) -> String { + format!("{} at {}: {}", self.code, self.path, self.message) + } +} + +fn issue_strings(issues: &[SemanticIssue]) -> Vec { + issues.iter().map(SemanticIssue::format).collect() +} + +fn forbid_legacy_signature(data: &Map, artifact_type: &str) -> Vec { + if data.contains_key(SIGNATURE_FIELD) { + vec![SemanticIssue::new( + "LegacySignatureOrDigest", + SIGNATURE_FIELD, + format!("{artifact_type}: {SIGNATURE_FIELD} is forbidden on VA *.v1 roots"), + )] + } else { + Vec::new() + } +} + +fn require_integrity(data: &Map, artifact_type: &str) -> Vec { + let Some(integrity) = data.get("integrity").and_then(|v| v.as_object()) else { + return vec![SemanticIssue::new( + "MissingIntegrity", + "integrity", + format!("{artifact_type}: nested integrity envelope is required"), + )]; + }; + let mut issues = Vec::new(); + if integrity.get("canonicalization_version").and_then(|v| v.as_str()) != Some("v1") { + issues.push(SemanticIssue::new( + "BadCanonicalizationVersion", + "integrity.canonicalization_version", + "must be v1", + )); + } + let digest = integrity.get("artifact_digest").and_then(|v| v.as_str()); + if digest.map(|d| !d.starts_with("sha256:")).unwrap_or(true) { + issues.push(SemanticIssue::new( + "BadArtifactDigest", + "integrity.artifact_digest", + "must be sha256:<64 hex>", + )); + } + issues +} + +fn check_zero_commit(commit: Option<&str>, path: &str) -> Vec { + if let Some(c) = commit { + if c.len() == 40 && c.chars().all(|ch| ch == '0') { + return vec![SemanticIssue::new( + "PlaceholderCommit", + path, + "placeholder zero git commit rejected", + )]; + } + } + Vec::new() +} + +fn is_full_hex_commit(commit: &str) -> bool { + commit.len() == 40 && commit.chars().all(|c| matches!(c, '0'..='9' | 'a'..='f')) +} + +fn check_secret_leaks(env_entries: Option<&Value>, path: &str) -> Vec { + let Some(obj) = env_entries.and_then(|v| v.as_object()) else { + return Vec::new(); + }; + let mut issues = Vec::new(); + for (key, value) in obj { + let key_l = key.to_lowercase(); + if SECRET_KEY_HINTS.iter().any(|hint| key_l.contains(hint)) { + issues.push(SemanticIssue::new( + "SecretKeyName", + format!("{path}.{key}"), + format!("redacted_environment key {key:?} looks like a secret name"), + )); + } + if let Some(v) = value.as_str() { + let value_l = v.to_lowercase(); + if v.starts_with("sk-") || value_l.contains("begin private key") { + issues.push(SemanticIssue::new( + "SecretValue", + format!("{path}.{key}"), + "value appears to contain a secret", + )); + } + } + } + issues +} + +/// Parse a decimal string into (mantissa, scale) where value = mantissa / 10^scale. +fn parse_decimal(value: &Value, path: &str) -> Result<(i128, u32), SemanticIssue> { + let Some(s) = value.as_str() else { + return Err(SemanticIssue::new( + "DecimalType", + path, + "decimal values must be strings", + )); + }; + parse_decimal_str(s).ok_or_else(|| { + SemanticIssue::new("DecimalParse", path, format!("invalid decimal string {s:?}")) + }) +} + +fn parse_decimal_str(s: &str) -> Option<(i128, u32)> { + let s = s.trim(); + if s.is_empty() { + return None; + } + let negative = s.starts_with('-'); + let body = if negative || s.starts_with('+') { + &s[1..] + } else { + s + }; + if body.is_empty() || !body.chars().all(|c| c.is_ascii_digit() || c == '.') { + return None; + } + if body.matches('.').count() > 1 { + return None; + } + let (int_part, frac_part) = match body.split_once('.') { + Some((i, f)) => (i, f), + None => (body, ""), + }; + if int_part.is_empty() && frac_part.is_empty() { + return None; + } + let int_digits = if int_part.is_empty() { "0" } else { int_part }; + let combined = format!("{int_digits}{frac_part}"); + let mantissa: i128 = combined.parse().ok()?; + let scale = frac_part.len() as u32; + Some((if negative { -mantissa } else { mantissa }, scale)) +} + +fn rescale(mantissa: i128, from_scale: u32, to_scale: u32) -> Option { + if to_scale >= from_scale { + let factor = 10i128.checked_pow(to_scale - from_scale)?; + mantissa.checked_mul(factor) + } else { + let factor = 10i128.checked_pow(from_scale - to_scale)?; + Some(mantissa / factor) + } +} + +fn decimals_equal(a: (i128, u32), b: (i128, u32)) -> bool { + let scale = a.1.max(b.1); + match (rescale(a.0, a.1, scale), rescale(b.0, b.1, scale)) { + (Some(x), Some(y)) => x == y, + _ => false, + } +} + +fn add_decimals(a: (i128, u32), b: (i128, u32)) -> Option<(i128, u32)> { + let scale = a.1.max(b.1); + let x = rescale(a.0, a.1, scale)?; + let y = rescale(b.0, b.1, scale)?; + Some((x.checked_add(y)?, scale)) +} + +pub fn validate_verifier_profile_semantics(data: &Value) -> Vec { + let Some(obj) = data.as_object() else { + return vec![SemanticIssue::new( + "RootType", + "$", + "VerifierProfile.v1 root must be an object", + )]; + }; + let mut issues = forbid_legacy_signature(obj, "VerifierProfile.v1"); + issues.extend(require_integrity(obj, "VerifierProfile.v1")); + let commit = obj.get("source_commit").and_then(|v| v.as_str()); + if commit.map(is_full_hex_commit) != Some(true) { + issues.push(SemanticIssue::new( + "InvalidSourceCommit", + "source_commit", + "source_commit must be a full 40-char lowercase hex SHA", + )); + } + issues.extend(check_zero_commit(commit, "source_commit")); + if let Some(configuration) = obj.get("configuration").and_then(|v| v.as_object()) { + for key in [ + "policy_digest", + "model_digest", + "prompt_digest", + "resource_limit_digest", + ] { + if !configuration.contains_key(key) { + issues.push(SemanticIssue::new( + "MissingNullDigestSlot", + format!("configuration.{key}"), + "inapplicable config digests must be present as explicit null", + )); + } + } + } + if let Some(impl_) = obj.get("implementation").and_then(|v| v.as_object()) { + let digest = impl_.get("implementation_digest"); + let missing = match digest { + None => true, + Some(Value::Null) => true, + Some(Value::String(s)) => s.is_empty(), + Some(_) => false, + }; + if missing { + issues.push(SemanticIssue::new( + "MissingImplementationDigest", + "implementation.implementation_digest", + "implementation_digest is required", + )); + } + } + if let Some(applicability) = obj.get("applicability").and_then(|v| v.as_object()) { + let status = applicability.get("status").and_then(|v| v.as_str()); + if status == Some("revoked") + && applicability + .get("revocation_reason") + .and_then(|v| v.as_str()) + .map(|s| s.is_empty()) + .unwrap_or(true) + { + issues.push(SemanticIssue::new( + "MissingRevocationReason", + "applicability.revocation_reason", + "revoked profiles require revocation_reason", + )); + } + if status == Some("superseded") + && applicability + .get("superseded_by_profile_id") + .and_then(|v| v.as_str()) + .map(|s| s.is_empty()) + .unwrap_or(true) + { + issues.push(SemanticIssue::new( + "MissingSupersededBy", + "applicability.superseded_by_profile_id", + "superseded profiles require superseded_by_profile_id", + )); + } + } + if let Some(redacted) = obj.get("redacted_environment").and_then(|v| v.as_object()) { + issues.extend(check_secret_leaks(redacted.get("entries"), "redacted_environment.entries")); + } + if let (Some(schema_doc), Some(schema_digest)) = ( + obj.get("configuration_schema").filter(|v| v.is_object()), + obj.get("configuration_schema_digest").and_then(|v| v.as_str()), + ) { + let recomputed = canonical_hash(schema_doc); + if recomputed != schema_digest { + issues.push(SemanticIssue::new( + "ConfigSchemaDigestMismatch", + "configuration_schema_digest", + format!("recorded {schema_digest:?} != recomputed {recomputed:?}"), + )); + } + } + issues +} + +pub fn validate_verification_result_semantics(data: &Value) -> Vec { + let Some(obj) = data.as_object() else { + return vec![SemanticIssue::new( + "RootType", + "$", + "VerificationResult.v1 root must be an object", + )]; + }; + let mut issues = forbid_legacy_signature(obj, "VerificationResult.v1"); + issues.extend(require_integrity(obj, "VerificationResult.v1")); + let commit = obj.get("source_commit").and_then(|v| v.as_str()); + if commit.map(is_full_hex_commit) != Some(true) { + issues.push(SemanticIssue::new( + "InvalidSourceCommit", + "source_commit", + "source_commit must be a full 40-char lowercase hex SHA", + )); + } + let decision = obj.get("decision").and_then(|v| v.as_str()); + let execution_status = obj.get("execution_status").and_then(|v| v.as_str()); + if let (Some(status), Some(dec)) = (execution_status, decision) { + if FAIL_CLOSED.contains(&status) && matches!(dec, "accept" | "reject") { + issues.push(SemanticIssue::new( + "FailClosedDecision", + "decision", + format!("execution_status {status:?} cannot yield accept/reject"), + )); + } + } + if obj.get("normalization_applied").and_then(|v| v.as_bool()) == Some(true) { + let raw = obj.get("raw_backend_output_digest").and_then(|v| v.as_str()); + let normalized = obj.get("normalized_result_digest").and_then(|v| v.as_str()); + match (raw, normalized) { + (Some(r), Some(n)) if r == n => { + issues.push(SemanticIssue::new( + "IdenticalNormalizationDigests", + "normalized_result_digest", + "raw and normalized digests must be distinct when normalization occurs", + )); + } + (Some(_), Some(_)) => {} + _ => { + issues.push(SemanticIssue::new( + "MissingNormalizationDigests", + "normalized_result_digest", + "normalization_applied requires raw and normalized digests", + )); + } + } + } + let mut mandatory_failed = false; + if let Some(groups) = obj.get("check_groups").and_then(|v| v.as_array()) { + for (g_index, group) in groups.iter().enumerate() { + let Some(gobj) = group.as_object() else { + continue; + }; + let Some(checks) = gobj.get("checks").and_then(|v| v.as_array()) else { + continue; + }; + for (c_index, check) in checks.iter().enumerate() { + let Some(cobj) = check.as_object() else { + continue; + }; + if cobj.get("mandatory").and_then(|v| v.as_bool()) == Some(true) + && cobj.get("status").and_then(|v| v.as_str()) == Some("failed") + { + mandatory_failed = true; + } + if cobj.get("status").and_then(|v| v.as_str()) == Some("skipped") { + let has_reason = cobj + .get("reason_code") + .and_then(|v| v.as_str()) + .map(|s| !s.is_empty()) + .unwrap_or(false) + || cobj + .get("skip_reason_code") + .and_then(|v| v.as_str()) + .map(|s| !s.is_empty()) + .unwrap_or(false); + if !has_reason { + issues.push(SemanticIssue::new( + "MissingSkipReason", + format!("check_groups[{g_index}].checks[{c_index}].reason_code"), + "skipped checks require reason_code", + )); + } + } + } + } + } + if decision == Some("accept") && mandatory_failed { + issues.push(SemanticIssue::new( + "AcceptWithMandatoryFailure", + "decision", + "accept cannot coexist with a mandatory failed check", + )); + } + if let (Some(declared), Some(result_class)) = ( + obj.get("declared_input_guarantee_class") + .and_then(|v| v.as_str()), + obj.get("guarantee_class").and_then(|v| v.as_str()), + ) { + if let (Some(d_rank), Some(r_rank)) = (guarantee_rank(declared), guarantee_rank(result_class)) + { + if r_rank > d_rank { + issues.push(SemanticIssue::new( + "GuaranteeUpgrade", + "guarantee_class", + format!( + "must not upgrade declared_input_guarantee_class ({declared:?} -> {result_class:?})" + ), + )); + } + } + } + issues +} + +pub fn validate_reward_envelope_semantics(data: &Value) -> Vec { + let Some(obj) = data.as_object() else { + return Vec::new(); + }; + let mut issues = forbid_legacy_signature(obj, "RewardEvidenceEnvelope.v1"); + issues.extend(require_integrity(obj, "RewardEvidenceEnvelope.v1")); + issues.extend(check_zero_commit( + obj.get("source_commit").and_then(|v| v.as_str()), + "source_commit", + )); + let total = match obj.get("scalar_total") { + Some(v) => match parse_decimal(v, "scalar_total") { + Ok(t) => Some(t), + Err(issue) => { + issues.push(issue); + None + } + }, + None => None, + }; + if let (Some(components), Some(total_dec)) = (obj.get("components").and_then(|v| v.as_array()), total) + { + if obj.get("composition_function").and_then(|v| v.as_str()) == Some("sum") { + let mut acc = (0i128, 0u32); + let mut ok = true; + for (index, component) in components.iter().enumerate() { + let Some(cobj) = component.as_object() else { + continue; + }; + match cobj.get("value") { + Some(v) => match parse_decimal(v, &format!("components[{index}].value")) { + Ok(value) => { + if let Some(sum) = add_decimals(acc, value) { + acc = sum; + } else { + ok = false; + } + } + Err(issue) => { + issues.push(issue); + ok = false; + } + }, + None => {} + } + } + if ok && !decimals_equal(acc, total_dec) { + issues.push(SemanticIssue::new( + "RewardCompositionMismatch", + "scalar_total", + format!("sum of components != scalar_total"), + )); + } + } + } + let claims = obj + .get("claims_issued") + .and_then(|v| v.as_array()) + .cloned() + .unwrap_or_default(); + if !claims.is_empty() { + let refs = obj.get("verifier_result_refs").and_then(|v| v.as_array()); + if refs.map(|r| r.is_empty()).unwrap_or(true) { + issues.push(SemanticIssue::new( + "ClaimsNeedVerifierRefs", + "verifier_result_refs", + "claims_issued requires at least one verifier_result_ref", + )); + } + } + if let Some(prefs) = obj.get("profile_refs").and_then(|v| v.as_array()) { + let lifecycle = obj.get("lifecycle").and_then(|v| v.as_object()); + let lifecycle_status = lifecycle.and_then(|l| l.get("status")).and_then(|v| v.as_str()); + let migration = lifecycle + .and_then(|l| l.get("migration_record_id")) + .and_then(|v| v.as_str()) + .map(|s| !s.is_empty()) + .unwrap_or(false); + for (index, pref) in prefs.iter().enumerate() { + let status = pref + .as_object() + .and_then(|p| p.get("applicability_status")) + .and_then(|v| v.as_str()); + if matches!(status, Some("revoked") | Some("expired")) + && lifecycle_status == Some("active") + && !migration + { + issues.push(SemanticIssue::new( + "RevokedProfileGate", + format!("profile_refs[{index}]"), + format!( + "{} profiles cannot support new active rewards without migration_record_id", + status.unwrap_or("revoked") + ), + )); + } else if status == Some("revoked") && lifecycle_status != Some("active") && !claims.is_empty() + { + issues.push(SemanticIssue::new( + "RevokedProfileGate", + format!("profile_refs[{index}]"), + "revoked profiles cannot authorize reward claims", + )); + } + } + } + let mandatory = obj + .get("mandatory_unresolved_claim_ids") + .and_then(|v| v.as_array()) + .cloned() + .unwrap_or_default(); + if obj + .get("lifecycle") + .and_then(|v| v.as_object()) + .and_then(|l| l.get("status")) + .and_then(|v| v.as_str()) + == Some("active") + && !mandatory.is_empty() + { + issues.push(SemanticIssue::new( + "ActiveRewardUnresolvedClaims", + "mandatory_unresolved_claim_ids", + "unresolved mandatory claims block active release-grade envelopes", + )); + } + issues +} + +pub fn validate_campaign_manifest_semantics(data: &Value) -> Vec { + let Some(obj) = data.as_object() else { + return Vec::new(); + }; + let mut issues = forbid_legacy_signature(obj, "OptimizationCampaignManifest.v1"); + issues.extend(require_integrity(obj, "OptimizationCampaignManifest.v1")); + issues.extend(check_zero_commit( + obj.get("source_commit").and_then(|v| v.as_str()), + "source_commit", + )); + if obj + .get("access_class") + .and_then(|v| v.as_str()) + .map(|s| s.is_empty()) + .unwrap_or(true) + { + issues.push(SemanticIssue::new( + "AccessClassRequired", + "access_class", + "access_class is required", + )); + } + if let Some(cohorts) = obj.get("cohorts").and_then(|v| v.as_array()) { + for (index, cohort) in cohorts.iter().enumerate() { + let Some(cobj) = cohort.as_object() else { + continue; + }; + if !cobj.get("compute_exposure").map(|v| v.is_object()).unwrap_or(false) { + issues.push(SemanticIssue::new( + "CohortComputeExposure", + format!("cohorts[{index}].compute_exposure"), + "compute_exposure is required", + )); + } + if cobj + .get("access_class") + .and_then(|v| v.as_str()) + .map(|s| s.is_empty()) + .unwrap_or(true) + { + issues.push(SemanticIssue::new( + "CohortAccessClass", + format!("cohorts[{index}].access_class"), + "every cohort must declare access_class", + )); + } + } + } + issues +} + +pub fn validate_adjudication_record_semantics( + data: &Value, + release_grade: bool, +) -> Vec { + let Some(obj) = data.as_object() else { + return Vec::new(); + }; + let mut issues = forbid_legacy_signature(obj, "AdjudicationRecord.v1"); + issues.extend(require_integrity(obj, "AdjudicationRecord.v1")); + issues.extend(check_zero_commit( + obj.get("source_commit").and_then(|v| v.as_str()), + "source_commit", + )); + if let Some(protected) = obj.get("protected_rationale").and_then(|v| v.as_object()) { + if protected + .get("commitment_digest") + .and_then(|v| v.as_str()) + .map(|s| s.is_empty()) + .unwrap_or(true) + { + issues.push(SemanticIssue::new( + "RationaleCommitment", + "protected_rationale.commitment_digest", + "commitment_digest is required", + )); + } + } + if release_grade && obj.get("independence_declared").and_then(|v| v.as_bool()) != Some(true) { + issues.push(SemanticIssue::new( + "IndependenceForReleaseGrade", + "independence_declared", + "release-grade adjudication requires independence_declared=true", + )); + } + issues +} + +pub fn validate_assurance_report_semantics(data: &Value) -> Vec { + let Some(obj) = data.as_object() else { + return Vec::new(); + }; + let mut issues = forbid_legacy_signature(obj, "VerifierAssuranceReport.v1"); + issues.extend(require_integrity(obj, "VerifierAssuranceReport.v1")); + issues.extend(check_zero_commit( + obj.get("source_commit").and_then(|v| v.as_str()), + "source_commit", + )); + let mut claims_gap = false; + if let Some(metrics) = obj.get("metrics").and_then(|v| v.as_object()) { + let sample = metrics.get("sample_size").and_then(|v| v.as_i64()); + let excluded = metrics.get("excluded_count").and_then(|v| v.as_i64()); + let unadj = metrics.get("unadjudicated_count").and_then(|v| v.as_i64()); + if let (Some(sample), Some(excluded), Some(unadj)) = (sample, excluded, unadj) { + if excluded + unadj > sample && sample > 0 { + issues.push(SemanticIssue::new( + "AggregateCountReconcile", + "metrics", + "excluded_count + unadjudicated_count exceeds sample_size", + )); + } + } + for key in [ + "false_accept_rate", + "false_reject_rate", + "abstention_rate", + "adjudication_coverage", + ] { + if let Some(rate) = metrics.get(key).and_then(|v| v.as_object()) { + let ci = rate.get("confidence_interval").and_then(|v| v.as_object()); + let method_ok = ci + .and_then(|c| c.get("method")) + .and_then(|v| v.as_str()) + .map(|s| !s.is_empty()) + .unwrap_or(false); + if !method_ok { + issues.push(SemanticIssue::new( + "CIMethodsDeclared", + format!("metrics.{key}.confidence_interval.method"), + "CI method must be declared", + )); + } else if ci + .and_then(|c| c.get("parameters")) + .map(|v| v.is_object()) + .unwrap_or(false) + == false + { + issues.push(SemanticIssue::new( + "CIParametersDeclared", + format!("metrics.{key}.confidence_interval.parameters"), + "CI parameters must be declared (no silent denominator invention)", + )); + } + } + } + let gap = metrics.get("optimization_gap").and_then(|v| v.as_str()); + let gap_nonzero = matches!(gap, Some(g) if g != "0" && g != "0.0" && g != "0.000000"); + let ordinary_den = metrics + .get("ordinary_accept_rate") + .and_then(|v| v.as_object()) + .and_then(|o| o.get("denominator")) + .and_then(|v| v.as_i64()) + .unwrap_or(0); + let optimized_den = metrics + .get("optimized_accept_rate") + .and_then(|v| v.as_object()) + .and_then(|o| o.get("denominator")) + .and_then(|v| v.as_i64()) + .unwrap_or(0); + claims_gap = gap_nonzero || (ordinary_den > 0 && optimized_den > 0); + + let excluded_items = obj.get("excluded_items").and_then(|v| v.as_array()); + let unadj_items = obj.get("unadjudicated_items").and_then(|v| v.as_array()); + if let Some(excluded_count) = metrics.get("excluded_count").and_then(|v| v.as_i64()) { + match excluded_items { + Some(items) if items.len() as i64 != excluded_count => { + issues.push(SemanticIssue::new( + "ExcludedItemsVisible", + "excluded_items", + "excluded_count must equal len(excluded_items)", + )); + } + None if excluded_count > 0 => { + issues.push(SemanticIssue::new( + "ExcludedItemsVisible", + "excluded_items", + "excluded_count > 0 requires visible excluded_items", + )); + } + _ => {} + } + } + if let Some(unadj_count) = metrics.get("unadjudicated_count").and_then(|v| v.as_i64()) { + match unadj_items { + Some(items) if items.len() as i64 != unadj_count => { + issues.push(SemanticIssue::new( + "UnadjudicatedItemsVisible", + "unadjudicated_items", + "unadjudicated_count must equal len(unadjudicated_items)", + )); + } + None if unadj_count > 0 => { + issues.push(SemanticIssue::new( + "UnadjudicatedItemsVisible", + "unadjudicated_items", + "unadjudicated_count > 0 requires visible unadjudicated_items", + )); + } + _ => {} + } + } + } + if let Some(cohorts) = obj.get("cohorts").and_then(|v| v.as_array()) { + let mut has_ordinary = false; + let mut has_optimized = false; + for (index, cohort) in cohorts.iter().enumerate() { + let Some(cobj) = cohort.as_object() else { + continue; + }; + match cobj.get("cohort_kind").and_then(|v| v.as_str()) { + Some("ordinary") => has_ordinary = true, + Some("optimized") => has_optimized = true, + _ => {} + } + if cobj + .get("access_class") + .and_then(|v| v.as_str()) + .map(|s| s.is_empty()) + .unwrap_or(true) + { + issues.push(SemanticIssue::new( + "CohortAccessClass", + format!("cohorts[{index}].access_class"), + "every cohort must declare access_class", + )); + } + if !cobj.get("compute_exposure").map(|v| v.is_object()).unwrap_or(false) { + issues.push(SemanticIssue::new( + "CohortComputeExposure", + format!("cohorts[{index}].compute_exposure"), + "every cohort must declare compute_exposure", + )); + } + let accept = cobj.get("accept_count").and_then(|v| v.as_i64()); + let reject = cobj.get("reject_count").and_then(|v| v.as_i64()); + let indeterminate = cobj.get("indeterminate_count").and_then(|v| v.as_i64()); + let included = cobj.get("included_result_count").and_then(|v| v.as_i64()); + if let (Some(a), Some(r), Some(i), Some(n)) = (accept, reject, indeterminate, included) { + if a + r + i != n { + issues.push(SemanticIssue::new( + "CohortCountMismatch", + format!("cohorts[{index}]"), + "aggregate counts must reconcile exactly with included records", + )); + } + if i < 0 || a < 0 || r < 0 { + issues.push(SemanticIssue::new( + "IndeterminateMisclassification", + format!("cohorts[{index}]"), + "cohort decision counts must be non-negative distinct buckets", + )); + } + } + } + if claims_gap && !(has_ordinary && has_optimized) { + issues.push(SemanticIssue::new( + "OptimizationGapCohorts", + "cohorts", + "optimization-gap metrics require ordinary and optimized cohorts", + )); + } + } else if claims_gap { + issues.push(SemanticIssue::new( + "OptimizationGapCohorts", + "cohorts", + "optimization-gap metrics require ordinary and optimized cohorts", + )); + } + if obj.get("release_grade").and_then(|v| v.as_bool()) == Some(true) + && obj.get("independent_adjudication").and_then(|v| v.as_bool()) != Some(true) + { + issues.push(SemanticIssue::new( + "ReleaseGradeAdjudication", + "independent_adjudication", + "release-grade reports require independent_adjudication=true", + )); + } + issues +} + +/// Dispatch used by `validate_semantics` for all VA *.v1 types. +pub fn validate_va_semantics(data: &Value, artifact_type: &str) -> Vec { + match artifact_type { + "VerifierProfile.v1" => validate_verifier_profile_semantics(data), + "VerificationResult.v1" => validate_verification_result_semantics(data), + "RewardEvidenceEnvelope.v1" => validate_reward_envelope_semantics(data), + "OptimizationCampaignManifest.v1" => validate_campaign_manifest_semantics(data), + "AdjudicationRecord.v1" => validate_adjudication_record_semantics(data, false), + "VerifierAssuranceReport.v1" => validate_assurance_report_semantics(data), + other => vec![SemanticIssue::new( + "UnknownArtifactType", + "artifact_type", + format!("unknown verifier-assurance artifact type: {other}"), + )], + } +} + +pub fn validate_va_semantics_strings(data: &Value, artifact_type: &str) -> Vec { + issue_strings(&validate_va_semantics(data, artifact_type)) +} + +pub fn is_va_artifact_type(artifact_type: &str) -> bool { + matches!( + artifact_type, + "VerifierProfile.v1" + | "VerificationResult.v1" + | "RewardEvidenceEnvelope.v1" + | "OptimizationCampaignManifest.v1" + | "AdjudicationRecord.v1" + | "VerifierAssuranceReport.v1" + ) +} + +/// Verify a VerifierAssuranceReport.v1: semantics + integrity digest match. +pub fn verify_assurance_report(report: &Value) -> Vec { + let mut issues = validate_assurance_report_semantics(report); + if let Some(obj) = report.as_object() { + if let Some(integrity) = obj.get("integrity").and_then(|v| v.as_object()) { + let mut body = obj.clone(); + body.remove("integrity"); + let expected = canonical_hash(&Value::Object(body)); + let got = integrity.get("artifact_digest").and_then(|v| v.as_str()); + if got != Some(expected.as_str()) { + issues.push(SemanticIssue::new( + "ReportDigestMismatch", + "integrity.artifact_digest", + "artifact_digest does not match report body", + )); + } + } + } + issues +} + +fn require_keys(obj: &Map, keys: &[&str], artifact: &str) -> Result<(), String> { + for key in keys { + if !obj.contains_key(*key) { + return Err(format!("{artifact}: missing mandatory field {key}")); + } + } + Ok(()) +} + +/// Constructor: rejects missing mandatory top-level fields; does not silently drop unknowns +/// (callers must pass an exact object; extras are preserved for schema to reject). +pub fn construct_va_artifact(artifact_type: &str, fields: Map) -> Result { + let required: &[&str] = match artifact_type { + "VerifierProfile.v1" => &[ + "schema_version", + "artifact_type", + "verifier_profile_id", + "created_at", + "producer", + "producer_version", + "source_repo", + "source_commit", + "implementation", + "configuration", + "mechanism", + "claim_surface", + "applicability", + "assumptions", + "known_blind_spots", + "integrity", + ], + "VerificationResult.v1" => &[ + "schema_version", + "artifact_type", + "verification_result_id", + "created_at", + "producer", + "producer_version", + "source_repo", + "source_commit", + "verifier_profile", + "claim_ids", + "raw_backend_output_digest", + "normalized_result_digest", + "normalization_applied", + "check_groups", + "resource_limits", + "execution_status", + "decision", + "integrity", + ], + "RewardEvidenceEnvelope.v1" => &[ + "schema_version", + "artifact_type", + "reward_envelope_id", + "created_at", + "producer", + "producer_version", + "source_repo", + "source_commit", + "scalar_total", + "components", + "composition_function", + "integrity", + ], + "OptimizationCampaignManifest.v1" => &[ + "schema_version", + "artifact_type", + "campaign_id", + "created_at", + "producer", + "producer_version", + "source_repo", + "source_commit", + "access_class", + "cohorts", + "integrity", + ], + "AdjudicationRecord.v1" => &[ + "schema_version", + "artifact_type", + "adjudication_id", + "created_at", + "producer", + "producer_version", + "source_repo", + "source_commit", + "subject", + "label", + "independence_declared", + "integrity", + ], + "VerifierAssuranceReport.v1" => &[ + "schema_version", + "artifact_type", + "report_id", + "created_at", + "producer", + "producer_version", + "source_repo", + "source_commit", + "campaign_ref", + "release_grade", + "independent_adjudication", + "metrics", + "cohorts", + "integrity", + ], + other => return Err(format!("unsupported VA constructor type: {other}")), + }; + require_keys(&fields, required, artifact_type)?; + if fields.get("artifact_type").and_then(|v| v.as_str()) != Some(artifact_type) { + return Err(format!( + "{artifact_type}: artifact_type must be {artifact_type:?}" + )); + } + // Unknown fields are retained intentionally so schema validation can reject them + // (no silent drop). + Ok(Value::Object(fields)) +} + +/// Attach nested integrity envelope (Python `attach_nested_integrity` equivalent). +pub fn attach_nested_integrity(data: &Value) -> Value { + let mut body = match data { + Value::Object(map) => map.clone(), + other => { + let mut m = Map::new(); + m.insert("_".into(), other.clone()); + m + } + }; + body.remove("integrity"); + let digest = canonical_hash(&Value::Object(body.clone())); + body.insert( + "integrity".into(), + json!({ + "canonicalization_version": "v1", + "artifact_digest": digest, + }), + ); + Value::Object(body) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + use std::fs; + use std::path::PathBuf; + + fn examples_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../../..") + .join("examples") + .join("verifier_assurance") + } + + fn load(rel: &str) -> Value { + let path = examples_dir().join(rel); + serde_json::from_str(&fs::read_to_string(path).unwrap()).unwrap() + } + + #[test] + fn va_valid_fixtures_pass_semantics() { + for rel in [ + "valid/profile_basic/profile.json", + "valid/result_accept/result.json", + "valid/reward_scalar/reward.json", + "valid/campaign_basic/campaign.json", + "valid/adjudication_basic/adjudication.json", + "valid/report_rebuild/report.json", + ] { + let data = load(rel); + let artifact_type = data + .get("artifact_type") + .and_then(|v| v.as_str()) + .unwrap(); + let issues = validate_va_semantics(&data, artifact_type); + assert!(issues.is_empty(), "{rel}: {issues:?}"); + } + } + + #[test] + fn va_invalid_fixtures_emit_expected_codes() { + let cases = [ + ("invalid/timeout_accept", "FailClosedDecision"), + ("invalid/accept_mandatory_failure", "AcceptWithMandatoryFailure"), + ("invalid/identical_normalization_digests", "IdenticalNormalizationDigests"), + ("invalid/reward_total_mismatch", "RewardCompositionMismatch"), + ("invalid/revoked_profile_active_reward", "RevokedProfileGate"), + ("invalid/missing_rationale_commitment", "RationaleCommitment"), + ("invalid/short_source_commit", "InvalidSourceCommit"), + ("invalid/release_grade_no_adjudication", "ReleaseGradeAdjudication"), + ("invalid/optimization_gap_missing_cohort", "OptimizationGapCohorts"), + ("invalid/cohort_missing_access", "CohortAccessClass"), + ("invalid/cohort_count_mismatch", "CohortCountMismatch"), + ("invalid/excluded_items_invisible", "ExcludedItemsVisible"), + ("invalid/missing_ci_method", "CIMethodsDeclared"), + ("invalid/indeterminate_misclassification", "IndeterminateMisclassification"), + ("invalid/active_reward_unresolved", "ActiveRewardUnresolvedClaims"), + ]; + for (dir, code) in cases { + let manifest: Value = load(&format!("{dir}/manifest.json")); + let artifact_file = manifest + .get("artifact_file") + .and_then(|v| v.as_str()) + .unwrap_or("artifact.json"); + let artifact_type = manifest + .get("artifact_type") + .and_then(|v| v.as_str()) + .unwrap(); + let data = load(&format!("{dir}/{artifact_file}")); + let issues = validate_va_semantics(&data, artifact_type); + assert!( + issues.iter().any(|i| i.code == code), + "{dir}: expected {code}, got {issues:?}" + ); + } + } + + #[test] + fn va_verify_report_accepts_valid_and_rejects_digest_tamper() { + let report = load("valid/report_rebuild/report.json"); + assert!(verify_assurance_report(&report).is_empty()); + let mut tampered = report.clone(); + if let Some(obj) = tampered.as_object_mut() { + obj.insert("report_id".into(), json!("tampered")); + } + let issues = verify_assurance_report(&tampered); + assert!( + issues.iter().any(|i| i.code == "ReportDigestMismatch"), + "{issues:?}" + ); + } + + #[test] + fn va_constructor_requires_mandatory_fields() { + let err = construct_va_artifact("VerifierProfile.v1", Map::new()).unwrap_err(); + assert!(err.contains("missing mandatory field")); + let mut fields = Map::new(); + for key in [ + "schema_version", + "artifact_type", + "verifier_profile_id", + "created_at", + "producer", + "producer_version", + "source_repo", + "source_commit", + "implementation", + "configuration", + "mechanism", + "claim_surface", + "applicability", + "assumptions", + "known_blind_spots", + "integrity", + ] { + fields.insert( + key.into(), + if key == "artifact_type" { + json!("VerifierProfile.v1") + } else if key == "assumptions" || key == "known_blind_spots" { + json!([]) + } else { + json!("x") + }, + ); + } + // Retain unknown field — must not be dropped. + fields.insert("extra_field".into(), json!(true)); + let built = construct_va_artifact("VerifierProfile.v1", fields).unwrap(); + assert!(built.get("extra_field").is_some()); + } + + #[test] + fn va_reward_sum_parity() { + let mut reward = load("valid/reward_scalar/reward.json"); + assert!(validate_reward_envelope_semantics(&reward).is_empty()); + if let Some(obj) = reward.as_object_mut() { + obj.insert("scalar_total".into(), json!("9.9")); + } + let issues = validate_reward_envelope_semantics(&reward); + assert!(issues.iter().any(|i| i.code == "RewardCompositionMismatch")); + } +} diff --git a/schemas/AdjudicationRecord.v1.schema.json b/schemas/AdjudicationRecord.v1.schema.json new file mode 100644 index 00000000..4084fa5f --- /dev/null +++ b/schemas/AdjudicationRecord.v1.schema.json @@ -0,0 +1,87 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://pcs.sentinelops.ci/schemas/AdjudicationRecord.v1.schema.json", + "title": "AdjudicationRecord.v1", + "description": "Adjudication record with labels, votes, independence/COI, blinded status, evidence refs, and protected rationale commitment (public may omit rationale content).", + "type": "object", + "required": [ + "schema_version", + "artifact_type", + "adjudication_id", + "created_at", + "producer", + "producer_version", + "source_repo", + "source_commit", + "subject", + "protocol_id", + "protocol_version", + "label", + "votes", + "independence_declared", + "conflict_of_interest_declared", + "blinded", + "label_released_at", + "evidence_refs", + "protected_rationale", + "integrity" + ], + "additionalProperties": false, + "properties": { + "schema_version": { "$ref": "common.defs.json#/$defs/schema_version_v1" }, + "artifact_type": { "const": "AdjudicationRecord.v1" }, + "adjudication_id": { "type": "string", "minLength": 1 }, + "created_at": { "$ref": "common.defs.json#/$defs/iso8601_datetime" }, + "producer": { "type": "string", "minLength": 1 }, + "producer_version": { "type": "string", "minLength": 1 }, + "source_repo": { "type": "string", "format": "uri" }, + "source_commit": { "$ref": "verifier_assurance.defs.json#/$defs/git_commit_sha40" }, + "subject": { "$ref": "verifier_assurance.defs.json#/$defs/artifact_ref" }, + "protocol_id": { "type": "string", "minLength": 1 }, + "protocol_version": { "type": "string", "minLength": 1 }, + "label": { "$ref": "verifier_assurance.defs.json#/$defs/adjudication_label" }, + "votes": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "required": ["voter_pseudonym_id", "role", "vote"], + "additionalProperties": false, + "properties": { + "voter_pseudonym_id": { "type": "string", "minLength": 1 }, + "role": { "type": "string", "minLength": 1 }, + "vote": { "$ref": "verifier_assurance.defs.json#/$defs/adjudication_label" } + } + } + }, + "independence_declared": { "type": "boolean" }, + "conflict_of_interest_declared": { "type": "boolean" }, + "disagreement": { "type": "boolean" }, + "escalation": { "type": "boolean" }, + "blinded": { "type": "boolean" }, + "label_released_at": { "$ref": "common.defs.json#/$defs/iso8601_datetime" }, + "evidence_refs": { + "type": "array", + "items": { "$ref": "verifier_assurance.defs.json#/$defs/artifact_ref" } + }, + "protected_rationale": { + "type": "object", + "required": ["commitment_digest", "location_class"], + "additionalProperties": false, + "properties": { + "commitment_digest": { "$ref": "common.defs.json#/$defs/hex_digest" }, + "location_class": { + "$ref": "verifier_assurance.defs.json#/$defs/protected_evidence_location_class" + }, + "content_omitted": { + "type": "boolean", + "description": "Public records set true; private rationale content must not appear in public fixtures." + } + } + }, + "integrity": { "$ref": "verifier_assurance.defs.json#/$defs/nested_integrity" }, + "signature_or_digest": { + "$ref": "verifier_assurance.defs.json#/$defs/forbid_signature_or_digest" + } + } +} diff --git a/schemas/OptimizationCampaignManifest.v1.schema.json b/schemas/OptimizationCampaignManifest.v1.schema.json new file mode 100644 index 00000000..e28687b5 --- /dev/null +++ b/schemas/OptimizationCampaignManifest.v1.schema.json @@ -0,0 +1,161 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://pcs.sentinelops.ci/schemas/OptimizationCampaignManifest.v1.schema.json", + "title": "OptimizationCampaignManifest.v1", + "description": "Optimization campaign declaration: target profile, env, model/policy/harness IDs, access class, budgets, splits, and containment policies.", + "type": "object", + "required": [ + "schema_version", + "artifact_type", + "campaign_id", + "created_at", + "producer", + "producer_version", + "source_repo", + "source_commit", + "target_verifier_profile", + "env_profile_id", + "env_profile_version", + "model_id", + "policy_id", + "harness_id", + "checkpoint_ids", + "algorithm", + "optimizer_settings", + "episode_count", + "trajectory_count", + "query_count", + "seeds", + "compute_budget", + "access_class", + "visibility", + "splits", + "containment_policy", + "disclosure_policy", + "integrity" + ], + "additionalProperties": false, + "properties": { + "schema_version": { "$ref": "common.defs.json#/$defs/schema_version_v1" }, + "artifact_type": { "const": "OptimizationCampaignManifest.v1" }, + "campaign_id": { "type": "string", "minLength": 1 }, + "created_at": { "$ref": "common.defs.json#/$defs/iso8601_datetime" }, + "producer": { "type": "string", "minLength": 1 }, + "producer_version": { "type": "string", "minLength": 1 }, + "source_repo": { "type": "string", "format": "uri" }, + "source_commit": { "$ref": "verifier_assurance.defs.json#/$defs/git_commit_sha40" }, + "target_verifier_profile": { + "type": "object", + "required": ["verifier_profile_id", "profile_digest"], + "additionalProperties": false, + "properties": { + "verifier_profile_id": { "type": "string", "minLength": 1 }, + "profile_digest": { "$ref": "common.defs.json#/$defs/hex_digest" } + } + }, + "env_profile_id": { "type": "string", "minLength": 1 }, + "env_profile_version": { "type": "string", "minLength": 1 }, + "model_id": { "type": "string", "minLength": 1 }, + "policy_id": { "type": "string", "minLength": 1 }, + "harness_id": { "type": "string", "minLength": 1 }, + "checkpoint_ids": { + "type": "array", + "items": { "type": "string", "minLength": 1 } + }, + "algorithm": { "type": "string", "minLength": 1 }, + "optimizer_settings": { + "type": "object", + "additionalProperties": false, + "properties": { + "learning_rate": { "$ref": "verifier_assurance.defs.json#/$defs/decimal_string" }, + "batch_size": { "type": "integer", "minimum": 1 }, + "notes": { "type": "string" } + } + }, + "episode_count": { "type": "integer", "minimum": 0 }, + "trajectory_count": { "type": "integer", "minimum": 0 }, + "query_count": { "type": "integer", "minimum": 0 }, + "seeds": { + "type": "array", + "items": { "type": "integer" } + }, + "compute_budget": { + "type": "object", + "required": ["accounting_method", "wall_time_seconds", "query_budget"], + "additionalProperties": false, + "properties": { + "accounting_method": { "type": "string", "minLength": 1 }, + "wall_time_seconds": { "type": "integer", "minimum": 0 }, + "query_budget": { "type": "integer", "minimum": 0 }, + "gpu_hours": { "$ref": "verifier_assurance.defs.json#/$defs/non_negative_decimal_string" } + } + }, + "access_class": { "$ref": "verifier_assurance.defs.json#/$defs/access_class" }, + "visibility": { + "type": "object", + "required": ["public_summary", "full_traces_retained"], + "additionalProperties": false, + "properties": { + "public_summary": { "type": "boolean" }, + "full_traces_retained": { "type": "boolean" }, + "notes": { "type": "string" } + } + }, + "splits": { + "type": "object", + "required": ["train", "validation", "holdout"], + "additionalProperties": false, + "properties": { + "train": { "type": "string", "minLength": 1 }, + "validation": { "type": "string", "minLength": 1 }, + "holdout": { "type": "string", "minLength": 1 } + } + }, + "mutation_schedule": { "type": "string" }, + "early_stopping": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { "type": "boolean" }, + "criterion": { "type": "string" } + } + }, + "cohorts": { + "type": "array", + "items": { + "type": "object", + "required": [ + "cohort_id", + "cohort_kind", + "access_class", + "compute_exposure" + ], + "additionalProperties": false, + "properties": { + "cohort_id": { "type": "string", "minLength": 1 }, + "cohort_kind": { + "type": "string", + "enum": ["ordinary", "optimized", "other"] + }, + "access_class": { "$ref": "verifier_assurance.defs.json#/$defs/access_class" }, + "compute_exposure": { + "type": "object", + "required": ["accounting_method", "wall_time_seconds", "query_count"], + "additionalProperties": false, + "properties": { + "accounting_method": { "type": "string", "minLength": 1 }, + "wall_time_seconds": { "type": "integer", "minimum": 0 }, + "query_count": { "type": "integer", "minimum": 0 } + } + } + } + } + }, + "containment_policy": { "type": "string", "minLength": 1 }, + "disclosure_policy": { "type": "string", "minLength": 1 }, + "integrity": { "$ref": "verifier_assurance.defs.json#/$defs/nested_integrity" }, + "signature_or_digest": { + "$ref": "verifier_assurance.defs.json#/$defs/forbid_signature_or_digest" + } + } +} diff --git a/schemas/RewardEvidenceEnvelope.v1.schema.json b/schemas/RewardEvidenceEnvelope.v1.schema.json new file mode 100644 index 00000000..49f656eb --- /dev/null +++ b/schemas/RewardEvidenceEnvelope.v1.schema.json @@ -0,0 +1,130 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://pcs.sentinelops.ci/schemas/RewardEvidenceEnvelope.v1.schema.json", + "title": "RewardEvidenceEnvelope.v1", + "description": "Reward evidence binding episode/step digests, scalar total, named components, verifier-result refs, claims, authority, and lifecycle.", + "type": "object", + "required": [ + "schema_version", + "artifact_type", + "reward_envelope_id", + "created_at", + "producer", + "producer_version", + "source_repo", + "source_commit", + "episode_id", + "step_index", + "env_profile_id", + "env_profile_version", + "state_digest", + "trajectory_digest", + "scalar_total", + "composition_function", + "components", + "verifier_result_refs", + "claims_issued", + "claims_rejected", + "claims_unresolved", + "authority", + "lifecycle", + "integrity" + ], + "additionalProperties": false, + "properties": { + "schema_version": { "$ref": "common.defs.json#/$defs/schema_version_v1" }, + "artifact_type": { "const": "RewardEvidenceEnvelope.v1" }, + "reward_envelope_id": { "type": "string", "minLength": 1 }, + "created_at": { "$ref": "common.defs.json#/$defs/iso8601_datetime" }, + "producer": { "type": "string", "minLength": 1 }, + "producer_version": { "type": "string", "minLength": 1 }, + "source_repo": { "type": "string", "format": "uri" }, + "source_commit": { "$ref": "verifier_assurance.defs.json#/$defs/git_commit_sha40" }, + "episode_id": { "type": "string", "minLength": 1 }, + "step_index": { "type": "integer", "minimum": 0 }, + "env_profile_id": { "type": "string", "minLength": 1 }, + "env_profile_version": { "type": "string", "minLength": 1 }, + "state_digest": { "$ref": "common.defs.json#/$defs/hex_digest" }, + "trajectory_digest": { "$ref": "common.defs.json#/$defs/hex_digest" }, + "scalar_total": { "$ref": "verifier_assurance.defs.json#/$defs/decimal_string" }, + "composition_function": { + "$ref": "verifier_assurance.defs.json#/$defs/reward_composition_function" + }, + "components": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "required": ["component_id", "value", "guarantee_class"], + "additionalProperties": false, + "properties": { + "component_id": { "type": "string", "minLength": 1 }, + "value": { "$ref": "verifier_assurance.defs.json#/$defs/decimal_string" }, + "weight": { "$ref": "verifier_assurance.defs.json#/$defs/decimal_string" }, + "guarantee_class": { "$ref": "verifier_assurance.defs.json#/$defs/guarantee_class" }, + "claim_id": { "type": "string", "minLength": 1 } + } + } + }, + "verifier_result_refs": { + "type": "array", + "items": { "$ref": "verifier_assurance.defs.json#/$defs/artifact_ref" } + }, + "claims_issued": { + "type": "array", + "items": { "type": "string", "minLength": 1 } + }, + "claims_rejected": { + "type": "array", + "items": { "type": "string", "minLength": 1 } + }, + "claims_unresolved": { + "type": "array", + "items": { "type": "string", "minLength": 1 } + }, + "mandatory_unresolved_claim_ids": { + "type": "array", + "items": { "type": "string", "minLength": 1 } + }, + "authority": { + "type": "object", + "required": ["authority_id", "authority_class"], + "additionalProperties": false, + "properties": { + "authority_id": { "type": "string", "minLength": 1 }, + "authority_class": { "type": "string", "minLength": 1 } + } + }, + "lifecycle": { + "type": "object", + "required": ["status"], + "additionalProperties": false, + "properties": { + "status": { "$ref": "verifier_assurance.defs.json#/$defs/lifecycle_status" }, + "supersedes": { "type": "string", "minLength": 1 }, + "superseded_by": { "type": "string", "minLength": 1 }, + "revocation_reason": { "type": "string", "minLength": 1 }, + "migration_record_id": { "type": "string", "minLength": 1 } + } + }, + "profile_refs": { + "type": "array", + "items": { + "type": "object", + "required": ["verifier_profile_id", "profile_digest", "applicability_status"], + "additionalProperties": false, + "properties": { + "verifier_profile_id": { "type": "string", "minLength": 1 }, + "profile_digest": { "$ref": "common.defs.json#/$defs/hex_digest" }, + "applicability_status": { + "$ref": "verifier_assurance.defs.json#/$defs/applicability_status" + } + } + } + }, + "integrity": { "$ref": "verifier_assurance.defs.json#/$defs/nested_integrity" }, + "signature_or_digest": { + "$ref": "verifier_assurance.defs.json#/$defs/forbid_signature_or_digest" + } + } +} diff --git a/schemas/VerificationResult.v1.schema.json b/schemas/VerificationResult.v1.schema.json new file mode 100644 index 00000000..1330cdff --- /dev/null +++ b/schemas/VerificationResult.v1.schema.json @@ -0,0 +1,154 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://pcs.sentinelops.ci/schemas/VerificationResult.v1.schema.json", + "title": "VerificationResult.v1", + "description": "Portable verifier-assurance decision record for a single claim evaluation. Distinct from VerificationResult.v0 (LabTrust/PF import checks with status+checks[]). Do not auto-upgrade v0 to v1. Nested integrity only; signature_or_digest forbidden.", + "type": "object", + "required": [ + "schema_version", + "artifact_type", + "verification_result_id", + "created_at", + "producer", + "producer_version", + "source_repo", + "source_commit", + "verifier_profile", + "claim_ids", + "raw_backend_output_digest", + "normalized_result_digest", + "normalization_applied", + "check_groups", + "resource_limits", + "execution_status", + "decision", + "integrity" + ], + "additionalProperties": false, + "properties": { + "schema_version": { "$ref": "common.defs.json#/$defs/schema_version_v1" }, + "artifact_type": { "const": "VerificationResult.v1" }, + "verification_result_id": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "created_at": { "$ref": "common.defs.json#/$defs/iso8601_datetime" }, + "producer": { "type": "string", "minLength": 1, "maxLength": 256 }, + "producer_version": { "type": "string", "minLength": 1, "maxLength": 128 }, + "source_repo": { "type": "string", "format": "uri" }, + "source_commit": { + "$ref": "verifier_assurance.defs.json#/$defs/git_commit_sha40" + }, + "verifier_profile": { + "type": "object", + "required": ["verifier_profile_id", "profile_digest"], + "additionalProperties": false, + "properties": { + "verifier_profile_id": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "profile_digest": { "$ref": "common.defs.json#/$defs/hex_digest" } + } + }, + "claim_ids": { + "type": "array", + "minItems": 1, + "items": { "type": "string", "minLength": 1 }, + "uniqueItems": true + }, + "trajectory_digest": { "$ref": "common.defs.json#/$defs/hex_digest" }, + "initial_state_digest": { "$ref": "common.defs.json#/$defs/hex_digest" }, + "terminal_state_digest": { "$ref": "common.defs.json#/$defs/hex_digest" }, + "input_bundle_digest": { "$ref": "common.defs.json#/$defs/hex_digest" }, + "raw_backend_output_digest": { + "$ref": "common.defs.json#/$defs/hex_digest" + }, + "normalized_result_digest": { + "$ref": "common.defs.json#/$defs/hex_digest" + }, + "normalization_applied": { "type": "boolean" }, + "check_groups": { + "type": "array", + "description": "Separate process / authority / side_effect / state / evidence check groups. Skipped checks retain status and optional reason_code.", + "items": { + "type": "object", + "required": ["kind", "checks"], + "additionalProperties": false, + "properties": { + "kind": { + "$ref": "verifier_assurance.defs.json#/$defs/check_group_kind" + }, + "checks": { + "type": "array", + "items": { + "type": "object", + "required": ["check_id", "mandatory", "status"], + "additionalProperties": false, + "properties": { + "check_id": { "type": "string", "minLength": 1 }, + "mandatory": { "type": "boolean" }, + "status": { + "$ref": "verifier_assurance.defs.json#/$defs/check_status" + }, + "reason_code": { "type": "string", "minLength": 1 } + } + } + } + } + } + }, + "resource_limits": { + "type": "object", + "additionalProperties": false, + "properties": { + "wall_time_ms": { "type": "integer", "minimum": 0 }, + "memory_bytes": { "type": "integer", "minimum": 0 }, + "max_output_bytes": { "type": "integer", "minimum": 0 }, + "notes": { "type": "string", "minLength": 1 } + } + }, + "execution_status": { + "$ref": "verifier_assurance.defs.json#/$defs/execution_status" + }, + "decision": { + "$ref": "verifier_assurance.defs.json#/$defs/verification_decision" + }, + "guarantee_class": { + "$ref": "verifier_assurance.defs.json#/$defs/guarantee_class" + }, + "declared_input_guarantee_class": { + "$ref": "verifier_assurance.defs.json#/$defs/guarantee_class" + }, + "normalizer_version": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "description": "Normalizer implementation version producing normalized_result_digest." + }, + "invocation_ref": { + "$ref": "verifier_assurance.defs.json#/$defs/invocation_ref" + }, + "assumptions": { + "type": "array", + "items": { + "$ref": "verifier_assurance.defs.json#/$defs/assumption_entry" + } + }, + "limits": { + "type": "array", + "items": { + "$ref": "verifier_assurance.defs.json#/$defs/limit_entry" + } + }, + "cohort_id": { "type": "string", "minLength": 1, "maxLength": 256 }, + "integrity": { + "$ref": "verifier_assurance.defs.json#/$defs/nested_integrity" + }, + "signature_or_digest": { + "$ref": "verifier_assurance.defs.json#/$defs/forbid_signature_or_digest" + } + } +} diff --git a/schemas/VerifierAssuranceReport.v1.schema.json b/schemas/VerifierAssuranceReport.v1.schema.json new file mode 100644 index 00000000..5c6f8f70 --- /dev/null +++ b/schemas/VerifierAssuranceReport.v1.schema.json @@ -0,0 +1,171 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://pcs.sentinelops.ci/schemas/VerifierAssuranceReport.v1.schema.json", + "title": "VerifierAssuranceReport.v1", + "description": "Deterministic assurance report aggregating FAR/FRR/abstention, adjudication coverage, optimization gap, and related metrics with declared CI methods.", + "type": "object", + "required": [ + "schema_version", + "artifact_type", + "report_id", + "created_at", + "producer", + "producer_version", + "source_repo", + "source_commit", + "campaign_ref", + "release_grade", + "metrics", + "cohorts", + "excluded_items", + "unadjudicated_items", + "applicability_limits", + "integrity" + ], + "additionalProperties": false, + "properties": { + "schema_version": { "$ref": "common.defs.json#/$defs/schema_version_v1" }, + "artifact_type": { "const": "VerifierAssuranceReport.v1" }, + "report_id": { "type": "string", "minLength": 1 }, + "created_at": { "$ref": "common.defs.json#/$defs/iso8601_datetime" }, + "producer": { "type": "string", "minLength": 1 }, + "producer_version": { "type": "string", "minLength": 1 }, + "source_repo": { "type": "string", "format": "uri" }, + "source_commit": { "$ref": "verifier_assurance.defs.json#/$defs/git_commit_sha40" }, + "campaign_ref": { "$ref": "verifier_assurance.defs.json#/$defs/artifact_ref" }, + "preregistration_ref": { "type": "string", "minLength": 1 }, + "release_grade": { "type": "boolean" }, + "independent_adjudication": { "type": "boolean" }, + "metrics": { + "type": "object", + "required": [ + "false_accept_rate", + "false_reject_rate", + "abstention_rate", + "adjudication_coverage", + "ordinary_accept_rate", + "optimized_accept_rate", + "optimization_gap", + "time_to_first_exploit_seconds", + "queries_to_first_exploit", + "exploit_family_counts", + "inter_verifier_disagreement_rate", + "reward_invalidation_rate", + "latency_ms_p50", + "cost_decimal", + "sample_size", + "excluded_count", + "unadjudicated_count" + ], + "additionalProperties": false, + "properties": { + "false_accept_rate": { "$ref": "verifier_assurance.defs.json#/$defs/rate_with_ci" }, + "false_reject_rate": { "$ref": "verifier_assurance.defs.json#/$defs/rate_with_ci" }, + "abstention_rate": { "$ref": "verifier_assurance.defs.json#/$defs/rate_with_ci" }, + "adjudication_coverage": { "$ref": "verifier_assurance.defs.json#/$defs/rate_with_ci" }, + "ordinary_accept_rate": { "$ref": "verifier_assurance.defs.json#/$defs/rate_with_ci" }, + "optimized_accept_rate": { "$ref": "verifier_assurance.defs.json#/$defs/rate_with_ci" }, + "optimization_gap": { "$ref": "verifier_assurance.defs.json#/$defs/decimal_string" }, + "time_to_first_exploit_seconds": { + "oneOf": [ + { "type": "integer", "minimum": 0 }, + { "type": "null" } + ] + }, + "queries_to_first_exploit": { + "oneOf": [ + { "type": "integer", "minimum": 0 }, + { "type": "null" } + ] + }, + "exploit_family_counts": { + "type": "object", + "additionalProperties": { "type": "integer", "minimum": 0 } + }, + "inter_verifier_disagreement_rate": { + "$ref": "verifier_assurance.defs.json#/$defs/rate_with_ci" + }, + "reward_invalidation_rate": { + "$ref": "verifier_assurance.defs.json#/$defs/rate_with_ci" + }, + "latency_ms_p50": { "type": "integer", "minimum": 0 }, + "cost_decimal": { "$ref": "verifier_assurance.defs.json#/$defs/non_negative_decimal_string" }, + "sample_size": { "type": "integer", "minimum": 0 }, + "excluded_count": { "type": "integer", "minimum": 0 }, + "unadjudicated_count": { "type": "integer", "minimum": 0 } + } + }, + "cohorts": { + "type": "array", + "items": { + "type": "object", + "required": [ + "cohort_id", + "cohort_kind", + "access_class", + "compute_exposure", + "included_result_count", + "accept_count", + "reject_count", + "indeterminate_count" + ], + "additionalProperties": false, + "properties": { + "cohort_id": { "type": "string", "minLength": 1 }, + "cohort_kind": { + "type": "string", + "enum": ["ordinary", "optimized", "other"] + }, + "access_class": { "$ref": "verifier_assurance.defs.json#/$defs/access_class" }, + "compute_exposure": { + "type": "object", + "required": ["accounting_method", "wall_time_seconds", "query_count"], + "additionalProperties": false, + "properties": { + "accounting_method": { "type": "string", "minLength": 1 }, + "wall_time_seconds": { "type": "integer", "minimum": 0 }, + "query_count": { "type": "integer", "minimum": 0 } + } + }, + "included_result_count": { "type": "integer", "minimum": 0 }, + "accept_count": { "type": "integer", "minimum": 0 }, + "reject_count": { "type": "integer", "minimum": 0 }, + "indeterminate_count": { "type": "integer", "minimum": 0 }, + "stratification": { "type": "string" } + } + } + }, + "excluded_items": { + "type": "array", + "items": { + "type": "object", + "required": ["item_id", "reason_code"], + "additionalProperties": false, + "properties": { + "item_id": { "type": "string", "minLength": 1 }, + "reason_code": { "type": "string", "minLength": 1 } + } + } + }, + "unadjudicated_items": { + "type": "array", + "items": { + "type": "object", + "required": ["item_id", "reason_code"], + "additionalProperties": false, + "properties": { + "item_id": { "type": "string", "minLength": 1 }, + "reason_code": { "type": "string", "minLength": 1 } + } + } + }, + "applicability_limits": { + "type": "array", + "items": { "type": "string", "minLength": 1 } + }, + "integrity": { "$ref": "verifier_assurance.defs.json#/$defs/nested_integrity" }, + "signature_or_digest": { + "$ref": "verifier_assurance.defs.json#/$defs/forbid_signature_or_digest" + } + } +} diff --git a/schemas/VerifierInvocationRecord.v1.schema.json b/schemas/VerifierInvocationRecord.v1.schema.json new file mode 100644 index 00000000..c1476a50 --- /dev/null +++ b/schemas/VerifierInvocationRecord.v1.schema.json @@ -0,0 +1,210 @@ +{ + "$id": "https://pcs.sentinelops.ci/schemas/VerifierInvocationRecord.v1.schema.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "description": "Portable single-run invocation evidence for verifier assurance.", + "properties": { + "artifact_type": { + "const": "VerifierInvocationRecord.v1" + }, + "assumptions": { + "items": { + "$ref": "verifier_assurance.defs.json#/$defs/assumption_entry" + }, + "type": "array" + }, + "canonicalization_version": { + "$ref": "common.defs.json#/$defs/canonicalization_version" + }, + "command": { + "additionalProperties": false, + "properties": { + "argv": { + "items": { + "minLength": 1, + "type": "string" + }, + "minItems": 1, + "type": "array" + }, + "cwd": { + "minLength": 1, + "type": "string" + }, + "env_digest": { + "$ref": "common.defs.json#/$defs/hex_digest" + }, + "limits": { + "additionalProperties": false, + "properties": { + "max_memory_bytes": { + "minimum": 1, + "type": "integer" + }, + "max_output_bytes": { + "minimum": 1, + "type": "integer" + }, + "timeout_ms": { + "minimum": 1, + "type": "integer" + } + }, + "type": "object" + } + }, + "required": [ + "argv", + "cwd", + "env_digest" + ], + "type": "object" + }, + "compiled_obligation": { + "additionalProperties": false, + "properties": { + "digest": { + "$ref": "common.defs.json#/$defs/hex_digest" + }, + "media_type": { + "maxLength": 256, + "minLength": 1, + "type": "string" + }, + "name": { + "maxLength": 256, + "minLength": 1, + "type": "string" + }, + "path": { + "$ref": "common.defs.json#/$defs/relative_posix_path" + } + }, + "required": [ + "digest" + ], + "type": "object" + }, + "completed_at": { + "$ref": "common.defs.json#/$defs/iso8601_datetime" + }, + "created_at": { + "$ref": "common.defs.json#/$defs/iso8601_datetime" + }, + "exit": { + "additionalProperties": false, + "properties": { + "code": { + "type": "integer" + }, + "kind": { + "$ref": "verifier_assurance.defs.json#/$defs/invocation_exit_kind" + }, + "message": { + "minLength": 1, + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + "failure_reason": { + "$ref": "verifier_assurance.defs.json#/$defs/failure_reason" + }, + "guarantee_class": { + "$ref": "verifier_assurance.defs.json#/$defs/guarantee_class" + }, + "indeterminate_reason": { + "$ref": "verifier_assurance.defs.json#/$defs/indeterminate_reason" + }, + "input_digest": { + "$ref": "common.defs.json#/$defs/hex_digest" + }, + "integrity": { + "$ref": "verifier_assurance.defs.json#/$defs/nested_integrity" + }, + "invocation_id": { + "maxLength": 256, + "minLength": 1, + "type": "string" + }, + "limits": { + "items": { + "$ref": "verifier_assurance.defs.json#/$defs/limit_entry" + }, + "type": "array" + }, + "normalized_result_digest": { + "$ref": "common.defs.json#/$defs/hex_digest" + }, + "normalizer_version": { + "maxLength": 128, + "minLength": 1, + "type": "string" + }, + "producer": { + "maxLength": 256, + "minLength": 1, + "type": "string" + }, + "producer_version": { + "maxLength": 128, + "minLength": 1, + "type": "string" + }, + "profile_ref": { + "$ref": "verifier_assurance.defs.json#/$defs/profile_ref" + }, + "raw_backend_result_digest": { + "$ref": "common.defs.json#/$defs/hex_digest" + }, + "result_ref": { + "$ref": "verifier_assurance.defs.json#/$defs/result_ref" + }, + "schema_version": { + "$ref": "common.defs.json#/$defs/schema_version_v1" + }, + "signature_or_digest": { + "$ref": "verifier_assurance.defs.json#/$defs/forbid_signature_or_digest" + }, + "source_commit": { + "$ref": "verifier_assurance.defs.json#/$defs/git_commit_sha40" + }, + "source_repo": { + "format": "uri", + "type": "string" + }, + "started_at": { + "$ref": "common.defs.json#/$defs/iso8601_datetime" + }, + "stderr_digest": { + "$ref": "common.defs.json#/$defs/hex_digest" + }, + "stdout_digest": { + "$ref": "common.defs.json#/$defs/hex_digest" + } + }, + "required": [ + "schema_version", + "artifact_type", + "canonicalization_version", + "invocation_id", + "profile_ref", + "input_digest", + "command", + "started_at", + "completed_at", + "exit", + "stdout_digest", + "stderr_digest", + "raw_backend_result_digest", + "normalized_result_digest", + "producer", + "producer_version", + "integrity" + ], + "title": "VerifierInvocationRecord.v1", + "type": "object" +} diff --git a/schemas/VerifierMutationManifest.v1.schema.json b/schemas/VerifierMutationManifest.v1.schema.json new file mode 100644 index 00000000..81f8c08c --- /dev/null +++ b/schemas/VerifierMutationManifest.v1.schema.json @@ -0,0 +1,91 @@ +{ + "$id": "https://pcs.sentinelops.ci/schemas/VerifierMutationManifest.v1.schema.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "description": "Typed immutable-profile mutation record.", + "properties": { + "artifact_type": { + "const": "VerifierMutationManifest.v1" + }, + "base_profile_ref": { + "$ref": "verifier_assurance.defs.json#/$defs/profile_ref" + }, + "canonicalization_version": { + "$ref": "common.defs.json#/$defs/canonicalization_version" + }, + "created_at": { + "$ref": "common.defs.json#/$defs/iso8601_datetime" + }, + "expected_effect": { + "minLength": 1, + "type": "string" + }, + "integrity": { + "$ref": "verifier_assurance.defs.json#/$defs/nested_integrity" + }, + "mutated_profile_ref": { + "$ref": "verifier_assurance.defs.json#/$defs/profile_ref" + }, + "mutation_class": { + "$ref": "verifier_assurance.defs.json#/$defs/mutation_class" + }, + "mutation_id": { + "maxLength": 256, + "minLength": 1, + "type": "string" + }, + "parameters": { + "type": "object" + }, + "producer": { + "maxLength": 256, + "minLength": 1, + "type": "string" + }, + "producer_version": { + "maxLength": 128, + "minLength": 1, + "type": "string" + }, + "production_prohibition": { + "const": true + }, + "rationale": { + "minLength": 1, + "type": "string" + }, + "schema_version": { + "$ref": "common.defs.json#/$defs/schema_version_v1" + }, + "signature_or_digest": { + "$ref": "verifier_assurance.defs.json#/$defs/forbid_signature_or_digest" + }, + "source_commit": { + "$ref": "verifier_assurance.defs.json#/$defs/git_commit_sha40" + }, + "source_repo": { + "format": "uri", + "type": "string" + }, + "supported_by_adapter": { + "type": "boolean" + } + }, + "required": [ + "schema_version", + "artifact_type", + "canonicalization_version", + "mutation_id", + "base_profile_ref", + "mutated_profile_ref", + "mutation_class", + "expected_effect", + "production_prohibition", + "producer", + "producer_version", + "created_at", + "integrity" + ], + "title": "VerifierMutationManifest.v1", + "type": "object" +} diff --git a/schemas/VerifierProfile.v1.schema.json b/schemas/VerifierProfile.v1.schema.json new file mode 100644 index 00000000..9d781261 --- /dev/null +++ b/schemas/VerifierProfile.v1.schema.json @@ -0,0 +1,247 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://pcs.sentinelops.ci/schemas/VerifierProfile.v1.schema.json", + "title": "VerifierProfile.v1", + "description": "Immutable verifier identity and configuration for portable verifier assurance. Distinct from WorkflowProfile.v0. Integrity is nested (ArtifactIntegrity.v1 pattern); signature_or_digest is forbidden.", + "type": "object", + "required": [ + "schema_version", + "artifact_type", + "verifier_profile_id", + "created_at", + "producer", + "producer_version", + "source_repo", + "source_commit", + "implementation", + "configuration", + "mechanism", + "claim_surface", + "applicability", + "assumptions", + "known_blind_spots", + "integrity" + ], + "additionalProperties": false, + "properties": { + "schema_version": { "$ref": "common.defs.json#/$defs/schema_version_v1" }, + "artifact_type": { "const": "VerifierProfile.v1" }, + "verifier_profile_id": { "type": "string", "minLength": 1, "maxLength": 256 }, + "created_at": { "$ref": "common.defs.json#/$defs/iso8601_datetime" }, + "producer": { "type": "string", "minLength": 1, "maxLength": 256 }, + "producer_version": { "type": "string", "minLength": 1, "maxLength": 128 }, + "source_repo": { "type": "string", "format": "uri" }, + "source_commit": { + "$ref": "verifier_assurance.defs.json#/$defs/git_commit_sha40" + }, + "implementation": { + "type": "object", + "required": ["name", "version", "implementation_digest"], + "additionalProperties": false, + "properties": { + "name": { "type": "string", "minLength": 1, "maxLength": 256 }, + "version": { "type": "string", "minLength": 1, "maxLength": 128 }, + "language": { "type": "string", "minLength": 1, "maxLength": 64 }, + "entry_point": { "type": "string", "minLength": 1 }, + "implementation_digest": { + "$ref": "common.defs.json#/$defs/hex_digest" + }, + "package_digest": { "$ref": "common.defs.json#/$defs/hex_digest" }, + "container_digest": { "$ref": "common.defs.json#/$defs/hex_digest" }, + "runtime_digest": { "$ref": "common.defs.json#/$defs/hex_digest" }, + "dependency_lock_digest": { + "$ref": "common.defs.json#/$defs/hex_digest" + }, + "normalization_implementation_version": { + "type": "string", + "minLength": 1, + "maxLength": 128 + } + } + }, + "configuration": { + "type": "object", + "description": "Material configuration digests. Inapplicable slots must be explicit JSON null (not omitted).", + "required": [ + "config_digest", + "policy_digest", + "model_digest", + "prompt_digest", + "resource_limit_digest" + ], + "additionalProperties": false, + "properties": { + "config_digest": { "$ref": "common.defs.json#/$defs/hex_digest" }, + "policy_digest": { + "$ref": "verifier_assurance.defs.json#/$defs/digest_or_null" + }, + "model_digest": { + "$ref": "verifier_assurance.defs.json#/$defs/digest_or_null" + }, + "prompt_digest": { + "$ref": "verifier_assurance.defs.json#/$defs/digest_or_null" + }, + "resource_limit_digest": { + "$ref": "verifier_assurance.defs.json#/$defs/digest_or_null" + }, + "rubric_digest": { + "$ref": "verifier_assurance.defs.json#/$defs/digest_or_null" + }, + "test_suite_digest": { + "$ref": "verifier_assurance.defs.json#/$defs/digest_or_null" + }, + "threshold_digest": { + "$ref": "verifier_assurance.defs.json#/$defs/digest_or_null" + }, + "ensemble_digest": { + "$ref": "verifier_assurance.defs.json#/$defs/digest_or_null" + } + } + }, + "mechanism": { + "type": "object", + "required": [ + "mechanism_class", + "determinism", + "allows_abstention", + "description" + ], + "additionalProperties": false, + "properties": { + "mechanism_class": { + "$ref": "verifier_assurance.defs.json#/$defs/mechanism_class" + }, + "determinism": { + "$ref": "verifier_assurance.defs.json#/$defs/decision_determinism" + }, + "allows_abstention": { "type": "boolean" }, + "description": { "type": "string", "minLength": 1 }, + "requires_authoritative_state": { "type": "boolean" }, + "requires_complete_trajectory": { "type": "boolean" }, + "requires_external_execution": { "type": "boolean" }, + "requires_human_judgment": { "type": "boolean" } + } + }, + "claim_surface": { + "type": "object", + "required": [ + "supported_claim_ids", + "guarantee_class", + "out_of_scope_claim_ids" + ], + "additionalProperties": false, + "properties": { + "supported_claim_ids": { + "type": "array", + "items": { "type": "string", "minLength": 1 } + }, + "guarantee_class": { + "$ref": "verifier_assurance.defs.json#/$defs/guarantee_class" + }, + "out_of_scope_claim_ids": { + "type": "array", + "items": { "type": "string", "minLength": 1 } + }, + "decision_space": { + "type": "array", + "items": { + "$ref": "verifier_assurance.defs.json#/$defs/verification_decision" + }, + "minItems": 1, + "uniqueItems": true + } + } + }, + "applicability": { + "type": "object", + "required": ["status"], + "additionalProperties": false, + "properties": { + "status": { + "$ref": "verifier_assurance.defs.json#/$defs/applicability_status" + }, + "valid_from": { "$ref": "common.defs.json#/$defs/iso8601_datetime" }, + "valid_until": { "$ref": "common.defs.json#/$defs/iso8601_datetime" }, + "environment_families": { + "type": "array", + "items": { "type": "string", "minLength": 1 } + }, + "environment_versions": { + "type": "array", + "items": { "type": "string", "minLength": 1 } + }, + "task_families": { + "type": "array", + "items": { "type": "string", "minLength": 1 } + }, + "claim_ids": { + "type": "array", + "items": { "type": "string", "minLength": 1 } + }, + "supported_input_schema_versions": { + "type": "array", + "items": { "type": "string", "minLength": 1 } + }, + "excluded_conditions": { + "type": "array", + "items": { "type": "string", "minLength": 1 } + }, + "supersedes_profile_id": { "type": "string", "minLength": 1 }, + "superseded_by_profile_id": { "type": "string", "minLength": 1 }, + "revoked_at": { "$ref": "common.defs.json#/$defs/iso8601_datetime" }, + "revocation_reason": { "type": "string", "minLength": 1 } + } + }, + "assumptions": { + "type": "array", + "items": { "type": "string", "minLength": 1 } + }, + "known_blind_spots": { + "type": "array", + "items": { "type": "string", "minLength": 1 } + }, + "configuration_schema_digest": { + "$ref": "common.defs.json#/$defs/hex_digest", + "description": "Optional digest of the schema that validates canonical_configuration." + }, + "configuration_schema": { + "type": "object", + "description": "Optional inline configuration schema. When present, Canonical JSON digest must equal configuration_schema_digest." + }, + "canonical_configuration": { + "type": "object", + "description": "Optional canonical configuration object (secrets redacted). Prefer digesting via configuration.config_digest." + }, + "redacted_environment": { + "$ref": "verifier_assurance.defs.json#/$defs/redacted_environment" + }, + "external_dependencies": { + "type": "array", + "items": { + "$ref": "verifier_assurance.defs.json#/$defs/external_dependency" + } + }, + "execution_controls": { + "type": "object", + "additionalProperties": false, + "properties": { + "seed": { "type": "string", "minLength": 1, "maxLength": 256 }, + "temperature": { + "$ref": "verifier_assurance.defs.json#/$defs/decimal_string" + }, + "timeout_ms": { "type": "integer", "minimum": 1 }, + "max_memory_bytes": { "type": "integer", "minimum": 1 }, + "notes": { "type": "string", "minLength": 1 } + } + }, + "limitations_notice": { + "$ref": "common.defs.json#/$defs/limitations_notice" + }, + "integrity": { + "$ref": "verifier_assurance.defs.json#/$defs/nested_integrity" + }, + "signature_or_digest": { + "$ref": "verifier_assurance.defs.json#/$defs/forbid_signature_or_digest" + } + } +} diff --git a/schemas/VerifierReplayReport.v1.schema.json b/schemas/VerifierReplayReport.v1.schema.json new file mode 100644 index 00000000..f1f40a9c --- /dev/null +++ b/schemas/VerifierReplayReport.v1.schema.json @@ -0,0 +1,125 @@ +{ + "$id": "https://pcs.sentinelops.ci/schemas/VerifierReplayReport.v1.schema.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "description": "Replay comparison report for verifier assurance.", + "properties": { + "artifact_type": { + "const": "VerifierReplayReport.v1" + }, + "canonicalization_version": { + "$ref": "common.defs.json#/$defs/canonicalization_version" + }, + "created_at": { + "$ref": "common.defs.json#/$defs/iso8601_datetime" + }, + "determinism": { + "$ref": "verifier_assurance.defs.json#/$defs/determinism_mode" + }, + "drift": { + "additionalProperties": false, + "properties": { + "claim_match": { + "type": "boolean" + }, + "normalized_digest_match": { + "type": "boolean" + }, + "notes": { + "minLength": 1, + "type": "string" + }, + "raw_digest_match": { + "type": "boolean" + } + }, + "required": [ + "raw_digest_match", + "normalized_digest_match" + ], + "type": "object" + }, + "failure_reason": { + "$ref": "verifier_assurance.defs.json#/$defs/failure_reason" + }, + "indeterminate_reason": { + "$ref": "verifier_assurance.defs.json#/$defs/indeterminate_reason" + }, + "integrity": { + "$ref": "verifier_assurance.defs.json#/$defs/nested_integrity" + }, + "original_invocation_ref": { + "$ref": "verifier_assurance.defs.json#/$defs/invocation_ref" + }, + "original_normalized_digest": { + "$ref": "common.defs.json#/$defs/hex_digest" + }, + "original_raw_digest": { + "$ref": "common.defs.json#/$defs/hex_digest" + }, + "producer": { + "maxLength": 256, + "minLength": 1, + "type": "string" + }, + "producer_version": { + "maxLength": 128, + "minLength": 1, + "type": "string" + }, + "profile_ref": { + "$ref": "verifier_assurance.defs.json#/$defs/profile_ref" + }, + "replay_id": { + "maxLength": 256, + "minLength": 1, + "type": "string" + }, + "replay_invocation_ref": { + "$ref": "verifier_assurance.defs.json#/$defs/invocation_ref" + }, + "replay_normalized_digest": { + "$ref": "common.defs.json#/$defs/hex_digest" + }, + "replay_raw_digest": { + "$ref": "common.defs.json#/$defs/hex_digest" + }, + "replay_status": { + "$ref": "verifier_assurance.defs.json#/$defs/replay_status" + }, + "schema_version": { + "$ref": "common.defs.json#/$defs/schema_version_v1" + }, + "signature_or_digest": { + "$ref": "verifier_assurance.defs.json#/$defs/forbid_signature_or_digest" + }, + "source_commit": { + "$ref": "verifier_assurance.defs.json#/$defs/git_commit_sha40" + }, + "source_repo": { + "format": "uri", + "type": "string" + } + }, + "required": [ + "schema_version", + "artifact_type", + "canonicalization_version", + "replay_id", + "original_invocation_ref", + "profile_ref", + "replay_status", + "determinism", + "original_raw_digest", + "replay_raw_digest", + "original_normalized_digest", + "replay_normalized_digest", + "drift", + "producer", + "producer_version", + "created_at", + "integrity" + ], + "title": "VerifierReplayReport.v1", + "type": "object" +} diff --git a/schemas/verifier_assurance.defs.json b/schemas/verifier_assurance.defs.json new file mode 100644 index 00000000..43d1b732 --- /dev/null +++ b/schemas/verifier_assurance.defs.json @@ -0,0 +1,576 @@ +{ + "$defs": { + "access_class": { + "description": "Optimizer / evaluator access to the verifier under test.", + "enum": [ + "black_box", + "gray_box", + "white_box", + "adaptive", + "transfer" + ], + "type": "string" + }, + "adjudication_label": { + "enum": [ + "valid", + "invalid", + "indeterminate_missing_evidence", + "indeterminate_domain_disagreement" + ], + "type": "string" + }, + "applicability_status": { + "enum": [ + "active", + "expired", + "superseded", + "revoked" + ], + "type": "string" + }, + "artifact_ref": { + "additionalProperties": false, + "properties": { + "artifact_digest": { + "$ref": "common.defs.json#/$defs/hex_digest" + }, + "artifact_id": { + "minLength": 1, + "type": "string" + }, + "artifact_type": { + "minLength": 1, + "type": "string" + } + }, + "required": [ + "artifact_type", + "artifact_id", + "artifact_digest" + ], + "type": "object" + }, + "assumption_entry": { + "additionalProperties": false, + "properties": { + "assumption_id": { + "maxLength": 256, + "minLength": 1, + "type": "string" + }, + "critical": { + "type": "boolean" + }, + "statement": { + "minLength": 1, + "type": "string" + } + }, + "required": [ + "assumption_id", + "statement" + ], + "type": "object" + }, + "check_group_kind": { + "enum": [ + "process", + "authority", + "side_effect", + "state", + "evidence" + ], + "type": "string" + }, + "check_status": { + "enum": [ + "passed", + "failed", + "skipped", + "error" + ], + "type": "string" + }, + "ci_method": { + "description": "Declared confidence-interval method. No silent default that invents denominators.", + "enum": [ + "wilson", + "clopper_pearson", + "bootstrap", + "other" + ], + "type": "string" + }, + "confidence_interval": { + "additionalProperties": false, + "properties": { + "lower": { + "$ref": "#/$defs/rate_decimal_string" + }, + "method": { + "$ref": "#/$defs/ci_method" + }, + "parameters": { + "description": "Method parameters (e.g. alpha). Must be declared explicitly; empty object only when method truly has none.", + "type": "object" + }, + "sample_size": { + "minimum": 0, + "type": "integer" + }, + "upper": { + "$ref": "#/$defs/rate_decimal_string" + } + }, + "required": [ + "method", + "parameters", + "lower", + "upper", + "sample_size" + ], + "type": "object" + }, + "decimal_string": { + "description": "Decimal number encoded as a string for Canonical JSON v1 (no JSON floats).", + "pattern": "^-?(?:0|[1-9][0-9]*)(?:\\.[0-9]+)?(?:[eE][+-]?[0-9]+)?$", + "type": "string" + }, + "decision_determinism": { + "enum": [ + "deterministic", + "stochastic" + ], + "type": "string" + }, + "determinism_mode": { + "enum": [ + "deterministic", + "stochastic", + "mixed" + ], + "type": "string" + }, + "digest_or_null": { + "description": "Hex digest slot that must be present; use JSON null when the mechanism is inapplicable (do not omit).", + "oneOf": [ + { + "$ref": "common.defs.json#/$defs/hex_digest" + }, + { + "type": "null" + } + ] + }, + "execution_status": { + "enum": [ + "completed", + "timeout", + "unavailable", + "error", + "cancelled", + "resource_exhausted" + ], + "type": "string" + }, + "external_dependency": { + "additionalProperties": false, + "properties": { + "dependency_id": { + "maxLength": 256, + "minLength": 1, + "type": "string" + }, + "digest": { + "$ref": "common.defs.json#/$defs/hex_digest" + }, + "identity": { + "minLength": 1, + "type": "string" + }, + "kind": { + "enum": [ + "binary", + "container", + "library", + "service", + "dataset", + "model", + "toolchain", + "other" + ], + "type": "string" + }, + "optional": { + "type": "boolean" + }, + "version": { + "maxLength": 128, + "minLength": 1, + "type": "string" + } + }, + "required": [ + "dependency_id", + "kind", + "identity" + ], + "type": "object" + }, + "failure_reason": { + "enum": [ + "obligation_unsatisfied", + "policy_denied", + "counterexample_found", + "test_failed", + "state_diff_mismatch", + "integrity_mismatch", + "unsupported_mutation", + "profile_revoked", + "internal_error", + "other" + ], + "type": "string" + }, + "forbid_signature_or_digest": { + "description": "Forbidden on VA *.v1 roots; use nested integrity.artifact_digest (+ optional signature).", + "not": {} + }, + "git_commit_sha40": { + "$ref": "common.defs.json#/$defs/git_commit" + }, + "guarantee_class": { + "description": "PCS guarantee label for VA evidence. Normalization/export must never upgrade this class. observational replaces runtime_observed as the VA observational label (do not treat runtime_observed as a formal upgrade path).", + "enum": [ + "observational", + "runtime_observed", + "empirically_measured", + "human_reviewed", + "certificate_checked", + "formally_checked", + "unchecked_advisory" + ], + "type": "string" + }, + "indeterminate_reason": { + "description": "Typed indeterminate reasons for invocation/replay layers (fail-closed; never acceptance).", + "enum": [ + "missing_checker", + "timeout", + "parser_failure", + "unsupported_input", + "external_service_error", + "resource_exhausted", + "spawn_error", + "incomplete_trajectory", + "missing_authoritative_state", + "declared_nondeterminism", + "other" + ], + "type": "string" + }, + "invocation_exit_kind": { + "enum": [ + "exited", + "timeout", + "missing_checker", + "spawn_error", + "signal", + "cancelled", + "resource_exhausted", + "error", + "other" + ], + "type": "string" + }, + "invocation_ref": { + "additionalProperties": false, + "properties": { + "artifact_type": { + "const": "VerifierInvocationRecord.v1" + }, + "invocation_digest": { + "$ref": "common.defs.json#/$defs/hex_digest" + }, + "invocation_id": { + "maxLength": 256, + "minLength": 1, + "type": "string" + } + }, + "required": [ + "invocation_id", + "invocation_digest" + ], + "type": "object" + }, + "lifecycle_status": { + "enum": [ + "draft", + "active", + "superseded", + "revoked", + "archived" + ], + "type": "string" + }, + "limit_entry": { + "additionalProperties": false, + "properties": { + "limit_id": { + "maxLength": 256, + "minLength": 1, + "type": "string" + }, + "statement": { + "minLength": 1, + "type": "string" + } + }, + "required": [ + "limit_id", + "statement" + ], + "type": "object" + }, + "mechanism_class": { + "description": "Controlled verifier mechanism class (not a formal-strength upgrade).", + "enum": [ + "static_analysis", + "dynamic_analysis", + "policy_engine", + "test_suite", + "model_judge", + "human_review", + "formal_proof", + "replay", + "hybrid", + "other" + ], + "type": "string" + }, + "mutation_class": { + "enum": [ + "remove_success_predicate", + "remove_process_predicate", + "remove_authority_predicate", + "change_threshold", + "reduce_test_subset", + "alter_timeout", + "suppress_error", + "change_rubric", + "change_prompt", + "ensemble_quorum", + "hidden_state_access", + "policy_bundle", + "abstention", + "other" + ], + "type": "string" + }, + "named_digest": { + "additionalProperties": false, + "properties": { + "digest": { + "$ref": "common.defs.json#/$defs/hex_digest" + }, + "media_type": { + "maxLength": 256, + "minLength": 1, + "type": "string" + }, + "name": { + "maxLength": 256, + "minLength": 1, + "type": "string" + }, + "path": { + "$ref": "common.defs.json#/$defs/relative_posix_path" + } + }, + "required": [ + "name", + "digest" + ], + "type": "object" + }, + "nested_integrity": { + "additionalProperties": false, + "description": "ArtifactIntegrity.v1 field pattern nested on VA roots (canonicalization_version + artifact_digest; optional signature).", + "properties": { + "artifact_digest": { + "$ref": "common.defs.json#/$defs/hex_digest" + }, + "canonicalization_version": { + "$ref": "common.defs.json#/$defs/canonicalization_version" + }, + "signature": { + "$ref": "common.defs.json#/$defs/ed25519_signature_envelope" + } + }, + "required": [ + "canonicalization_version", + "artifact_digest" + ], + "type": "object" + }, + "non_negative_decimal_string": { + "description": "Non-negative decimal string (Canonical JSON v1; no JSON floats).", + "pattern": "^(?:0|[1-9][0-9]*)(?:\\.[0-9]+)?(?:[eE][+-]?[0-9]+)?$", + "type": "string" + }, + "profile_ref": { + "additionalProperties": false, + "properties": { + "artifact_type": { + "const": "VerifierProfile.v1" + }, + "profile_digest": { + "$ref": "common.defs.json#/$defs/hex_digest" + }, + "verifier_profile_id": { + "maxLength": 256, + "minLength": 1, + "type": "string" + } + }, + "required": [ + "artifact_type", + "verifier_profile_id", + "profile_digest" + ], + "type": "object" + }, + "protected_evidence_location_class": { + "enum": [ + "public_artifact", + "access_controlled_store", + "partner_private_store", + "holdout_store", + "omitted" + ], + "type": "string" + }, + "rate_decimal_string": { + "description": "Rate in [0, 1] encoded as a decimal string (Canonical JSON v1; no JSON floats).", + "pattern": "^(?:0(?:\\.[0-9]+)?|1(?:\\.0+)?)(?:[eE][+-]?[0-9]+)?$", + "type": "string" + }, + "rate_with_ci": { + "additionalProperties": false, + "properties": { + "confidence_interval": { + "$ref": "#/$defs/confidence_interval" + }, + "denominator": { + "minimum": 0, + "type": "integer" + }, + "numerator": { + "minimum": 0, + "type": "integer" + }, + "rate": { + "$ref": "#/$defs/rate_decimal_string" + } + }, + "required": [ + "rate", + "numerator", + "denominator", + "confidence_interval" + ], + "type": "object" + }, + "redacted_environment": { + "additionalProperties": false, + "properties": { + "entries": { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + "environment_digest": { + "$ref": "common.defs.json#/$defs/hex_digest" + }, + "redacted_keys": { + "items": { + "minLength": 1, + "type": "string" + }, + "type": "array", + "uniqueItems": true + }, + "redaction_policy_id": { + "maxLength": 256, + "minLength": 1, + "type": "string" + } + }, + "required": [ + "redaction_policy_id", + "entries" + ], + "type": "object" + }, + "replay_status": { + "enum": [ + "matched", + "drifted", + "indeterminate", + "failed" + ], + "type": "string" + }, + "result_ref": { + "additionalProperties": false, + "properties": { + "artifact_type": { + "const": "VerificationResult.v1" + }, + "result_digest": { + "$ref": "common.defs.json#/$defs/hex_digest" + }, + "verification_result_id": { + "maxLength": 256, + "minLength": 1, + "type": "string" + } + }, + "required": [ + "artifact_type", + "verification_result_id", + "result_digest" + ], + "type": "object" + }, + "reward_composition_function": { + "enum": [ + "sum", + "weighted_sum", + "min", + "max", + "custom" + ], + "type": "string" + }, + "verification_decision": { + "description": "Closed decision vocabulary for VerificationResult.v1. Fail closed: timeouts/unavailable/malformed/unsupported map to indeterminate_*, never accept/reject.", + "enum": [ + "accept", + "reject", + "indeterminate_insufficient_evidence", + "indeterminate_execution_error", + "indeterminate_out_of_scope", + "indeterminate_configuration_drift" + ], + "type": "string" + } + }, + "$id": "https://pcs.sentinelops.ci/schemas/verifier_assurance.defs.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "Shared $defs for Verifier Assurance (VA) *.v1 artifacts. Digests use common.defs.json#/$defs/hex_digest. Rates and rewards use decimal strings (no JSON floats). Do not upgrade observational / runtime-observed evidence to formal guarantee classes.", + "title": "PCS Verifier Assurance Common Definitions" +} diff --git a/scripts/gen_va_fixtures.py b/scripts/gen_va_fixtures.py new file mode 100644 index 00000000..a83e2ada --- /dev/null +++ b/scripts/gen_va_fixtures.py @@ -0,0 +1,635 @@ +"""Generate Verifier Assurance example fixtures.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from pcs_core.hash import CANONICALIZATION_VERSION, canonical_hash +from pcs_core.paths import repo_root +from pcs_core.verifier_assurance_report import build_assurance_report + +D = "sha256:" + ("a" * 64) +D2 = "sha256:" + ("b" * 64) +D3 = "sha256:" + ("c" * 64) +D4 = "sha256:" + ("d" * 64) +D5 = "sha256:" + ("e" * 64) +COMMIT = "e068794683959c52a19594a6d271dd5e69f3c999" +REPO = "https://github.com/SentinelOps-CI/pcs-core" + + +def integrity_for(body: dict) -> dict: + payload = {k: v for k, v in body.items() if k != "integrity"} + return { + "canonicalization_version": CANONICALIZATION_VERSION, + "artifact_digest": canonical_hash(payload), + } + + +def finalize(body: dict) -> dict: + out = {k: v for k, v in body.items() if k != "integrity"} + out["integrity"] = integrity_for(out) + return out + + +def write(path: Path, data: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def write_manifest(path: Path, expected_error: str, artifact_type: str, artifact_file: str = "artifact.json") -> None: + write( + path, + { + "expected_error": expected_error, + "artifact_type": artifact_type, + "artifact_file": artifact_file, + }, + ) + + +def base_profile(**overrides) -> dict: + body = { + "schema_version": "v1", + "artifact_type": "VerifierProfile.v1", + "verifier_profile_id": "vp-ovk-static-001", + "created_at": "2026-07-24T12:00:00Z", + "producer": "OVK", + "producer_version": "0.1.0", + "source_repo": REPO, + "source_commit": COMMIT, + "implementation": { + "name": "ovk-static-checker", + "version": "1.0.0", + "implementation_digest": D, + "language": "rust", + }, + "configuration": { + "config_digest": D2, + "policy_digest": None, + "model_digest": None, + "prompt_digest": None, + "resource_limit_digest": D3, + }, + "mechanism": { + "mechanism_class": "static_analysis", + "determinism": "deterministic", + "allows_abstention": True, + "description": "Static claim checker", + }, + "claim_surface": { + "supported_claim_ids": ["claim.safety.no_egress"], + "guarantee_class": "certificate_checked", + "out_of_scope_claim_ids": ["claim.formal.full_correctness"], + }, + "applicability": {"status": "active", "valid_from": "2026-07-01T00:00:00Z"}, + "assumptions": ["host OS is Linux"], + "known_blind_spots": ["does not observe runtime network"], + } + body.update(overrides) + return finalize(body) + + +def base_result(profile: dict, **overrides) -> dict: + body = { + "schema_version": "v1", + "artifact_type": "VerificationResult.v1", + "verification_result_id": "vr-001", + "created_at": "2026-07-24T12:05:00Z", + "producer": "OVK", + "producer_version": "0.1.0", + "source_repo": REPO, + "source_commit": COMMIT, + "verifier_profile": { + "verifier_profile_id": profile["verifier_profile_id"], + "profile_digest": canonical_hash(profile), + }, + "claim_ids": ["claim.safety.no_egress"], + "trajectory_digest": D4, + "initial_state_digest": D, + "terminal_state_digest": D2, + "input_bundle_digest": D3, + "raw_backend_output_digest": D, + "normalized_result_digest": D2, + "normalization_applied": True, + "check_groups": [ + { + "kind": "process", + "checks": [ + {"check_id": "process.complete", "mandatory": True, "status": "passed"} + ], + }, + { + "kind": "evidence", + "checks": [ + {"check_id": "evidence.present", "mandatory": True, "status": "passed"} + ], + }, + ], + "resource_limits": {"wall_time_ms": 1000, "memory_bytes": 1048576}, + "execution_status": "completed", + "decision": "accept", + } + body.update(overrides) + return finalize(body) + + +def base_reward(**overrides) -> dict: + body = { + "schema_version": "v1", + "artifact_type": "RewardEvidenceEnvelope.v1", + "reward_envelope_id": "rew-001", + "created_at": "2026-07-24T12:10:00Z", + "producer": "LabTrust-Gym", + "producer_version": "0.1.0", + "source_repo": REPO, + "source_commit": COMMIT, + "episode_id": "ep-1", + "step_index": 0, + "env_profile_id": "env-lab", + "env_profile_version": "1.0.0", + "state_digest": D, + "trajectory_digest": D4, + "scalar_total": "1.5", + "composition_function": "sum", + "components": [ + { + "component_id": "safety", + "value": "1.0", + "guarantee_class": "certificate_checked", + "claim_id": "claim.safety.no_egress", + }, + { + "component_id": "quality", + "value": "0.5", + "guarantee_class": "empirically_measured", + }, + ], + "verifier_result_refs": [ + { + "artifact_type": "VerificationResult.v1", + "artifact_id": "vr-001", + "artifact_digest": D5, + } + ], + "claims_issued": ["claim.safety.no_egress"], + "claims_rejected": [], + "claims_unresolved": [], + "mandatory_unresolved_claim_ids": [], + "authority": {"authority_id": "labtrust-reward", "authority_class": "producer"}, + "lifecycle": {"status": "active"}, + "profile_refs": [ + { + "verifier_profile_id": "vp-ovk-static-001", + "profile_digest": D, + "applicability_status": "active", + } + ], + } + body.update(overrides) + return finalize(body) + + +def base_campaign(**overrides) -> dict: + body = { + "schema_version": "v1", + "artifact_type": "OptimizationCampaignManifest.v1", + "campaign_id": "camp-001", + "created_at": "2026-07-24T11:00:00Z", + "producer": "pcs-core", + "producer_version": "0.1.0", + "source_repo": REPO, + "source_commit": COMMIT, + "target_verifier_profile": { + "verifier_profile_id": "vp-ovk-static-001", + "profile_digest": D, + }, + "env_profile_id": "env-lab", + "env_profile_version": "1.0.0", + "model_id": "model-a", + "policy_id": "policy-a", + "harness_id": "harness-a", + "checkpoint_ids": ["ckpt-0"], + "algorithm": "random-search", + "optimizer_settings": {"learning_rate": "0.01", "batch_size": 8}, + "episode_count": 10, + "trajectory_count": 10, + "query_count": 100, + "seeds": [1, 2, 3], + "compute_budget": { + "accounting_method": "wall_clock", + "wall_time_seconds": 3600, + "query_budget": 1000, + "gpu_hours": "0", + }, + "access_class": "black_box", + "visibility": {"public_summary": True, "full_traces_retained": False}, + "splits": {"train": "train-v1", "validation": "val-v1", "holdout": "hold-v1"}, + "cohorts": [ + { + "cohort_id": "cohort-ordinary", + "cohort_kind": "ordinary", + "access_class": "black_box", + "compute_exposure": { + "accounting_method": "wall_clock", + "wall_time_seconds": 600, + "query_count": 50, + }, + }, + { + "cohort_id": "cohort-optimized", + "cohort_kind": "optimized", + "access_class": "adaptive", + "compute_exposure": { + "accounting_method": "wall_clock", + "wall_time_seconds": 3000, + "query_count": 950, + }, + }, + ], + "containment_policy": "offline-only", + "disclosure_policy": "public-summary", + } + body.update(overrides) + return finalize(body) + + +def base_adjudication(**overrides) -> dict: + body = { + "schema_version": "v1", + "artifact_type": "AdjudicationRecord.v1", + "adjudication_id": "adj-001", + "created_at": "2026-07-24T13:00:00Z", + "producer": "pcs-core", + "producer_version": "0.1.0", + "source_repo": REPO, + "source_commit": COMMIT, + "subject": { + "artifact_type": "VerificationResult.v1", + "artifact_id": "vr-001", + "artifact_digest": D5, + }, + "protocol_id": "adj-protocol", + "protocol_version": "1.0.0", + "label": "valid", + "votes": [{"voter_pseudonym_id": "voter-a", "role": "primary", "vote": "valid"}], + "independence_declared": True, + "conflict_of_interest_declared": False, + "disagreement": False, + "escalation": False, + "blinded": True, + "label_released_at": "2026-07-24T14:00:00Z", + "evidence_refs": [], + "protected_rationale": { + "commitment_digest": D, + "location_class": "access_controlled_store", + "content_omitted": True, + }, + } + body.update(overrides) + return finalize(body) + + +def main() -> None: + root = repo_root() / "examples" / "verifier_assurance" + profile = base_profile() + write(root / "valid" / "profile_basic" / "profile.json", profile) + write(root / "valid" / "result_accept" / "result.json", base_result(profile)) + write(root / "valid" / "reward_scalar" / "reward.json", base_reward()) + campaign = base_campaign() + write(root / "valid" / "campaign_basic" / "campaign.json", campaign) + write(root / "valid" / "adjudication_basic" / "adjudication.json", base_adjudication()) + + r1 = base_result(profile, verification_result_id="vr-001", cohort_id="cohort-ordinary") + r2 = base_result( + profile, + verification_result_id="vr-002", + cohort_id="cohort-optimized", + decision="reject", + check_groups=[ + { + "kind": "process", + "checks": [ + {"check_id": "process.complete", "mandatory": True, "status": "failed"} + ], + } + ], + ) + a1 = base_adjudication() + a2 = base_adjudication( + adjudication_id="adj-002", + subject={ + "artifact_type": "VerificationResult.v1", + "artifact_id": "vr-002", + "artifact_digest": D5, + }, + label="invalid", + votes=[{"voter_pseudonym_id": "voter-a", "role": "primary", "vote": "invalid"}], + ) + report = build_assurance_report( + campaign=campaign, + results=[r1, r2], + adjudications=[a1, a2], + report_id="rep-001", + created_at="2026-07-24T15:00:00Z", + source_commit=COMMIT, + release_grade=True, + excluded_items=[{"item_id": "ex-1", "reason_code": "out_of_scope"}], + unadjudicated_items=[], + applicability_limits=["synthetic fixture only"], + ) + report_dir = root / "valid" / "report_rebuild" + write(report_dir / "campaign.json", campaign) + write(report_dir / "results" / "vr-001.json", r1) + write(report_dir / "results" / "vr-002.json", r2) + write(report_dir / "adjudications" / "adj-001.json", a1) + write(report_dir / "adjudications" / "adj-002.json", a2) + write(report_dir / "report.json", report) + + # Invalid fixtures + inv = root / "invalid" + + bad_timeout = base_result(profile, execution_status="timeout", decision="accept") + write(inv / "timeout_accept" / "artifact.json", bad_timeout) + write_manifest(inv / "timeout_accept" / "manifest.json", "FailClosedDecision", "VerificationResult.v1") + + bad_accept = base_result( + profile, + decision="accept", + check_groups=[ + { + "kind": "process", + "checks": [ + {"check_id": "process.complete", "mandatory": True, "status": "failed"} + ], + } + ], + ) + write(inv / "accept_mandatory_failure" / "artifact.json", bad_accept) + write_manifest( + inv / "accept_mandatory_failure" / "manifest.json", + "AcceptWithMandatoryFailure", + "VerificationResult.v1", + ) + + bad_norm = base_result( + profile, + normalization_applied=True, + raw_backend_output_digest=D, + normalized_result_digest=D, + ) + write(inv / "identical_normalization_digests" / "artifact.json", bad_norm) + write_manifest( + inv / "identical_normalization_digests" / "manifest.json", + "IdenticalNormalizationDigests", + "VerificationResult.v1", + ) + + bad_reward = base_reward(scalar_total="9.9") + write(inv / "reward_total_mismatch" / "artifact.json", bad_reward) + write_manifest( + inv / "reward_total_mismatch" / "manifest.json", + "RewardCompositionMismatch", + "RewardEvidenceEnvelope.v1", + ) + + bad_revoked = base_reward( + lifecycle={"status": "active"}, + profile_refs=[ + { + "verifier_profile_id": "vp-ovk-static-001", + "profile_digest": D, + "applicability_status": "revoked", + } + ], + ) + write(inv / "revoked_profile_active_reward" / "artifact.json", bad_revoked) + write_manifest( + inv / "revoked_profile_active_reward" / "manifest.json", + "RevokedProfileGate", + "RewardEvidenceEnvelope.v1", + ) + + bad_adj = base_adjudication(protected_rationale={"location_class": "partner_vault"}) + write(inv / "missing_rationale_commitment" / "artifact.json", bad_adj) + write_manifest( + inv / "missing_rationale_commitment" / "manifest.json", + "RationaleCommitment", + "AdjudicationRecord.v1", + ) + + short_commit = base_profile(source_commit="abc1234") + write(inv / "short_source_commit" / "artifact.json", short_commit) + write_manifest( + inv / "short_source_commit" / "manifest.json", + "InvalidSourceCommit", + "VerifierProfile.v1", + ) + + unknown = base_profile() + unknown["extra_field"] = "nope" + unknown.pop("integrity", None) + unknown = finalize(unknown) + write(inv / "unknown_field" / "artifact.json", unknown) + write_manifest( + inv / "unknown_field" / "manifest.json", + "Additional properties are not allowed ('extra_field' was unexpected)", + "VerifierProfile.v1", + ) + + # Rule 1: profile digest mismatch (multi-file) + digest_case = inv / "profile_digest_mismatch" + write(digest_case / "profile.json", profile) + bad_result = base_result( + profile, + verifier_profile={ + "verifier_profile_id": profile["verifier_profile_id"], + "profile_digest": "sha256:" + ("f" * 64), + }, + ) + write(digest_case / "artifact.json", bad_result) + write_manifest(digest_case / "manifest.json", "ProfileDigestMismatch", "VerificationResult.v1") + + # Rule 2: reward trajectory mismatch (multi-file) + traj_case = inv / "reward_trajectory_mismatch" + good_result = base_result(profile) + write(traj_case / "result.json", good_result) + bad_traj_reward = base_reward( + trajectory_digest="sha256:" + ("f" * 64), + verifier_result_refs=[ + { + "artifact_type": "VerificationResult.v1", + "artifact_id": "vr-001", + "artifact_digest": D5, + } + ], + ) + write(traj_case / "artifact.json", bad_traj_reward) + write_manifest( + traj_case / "manifest.json", "RewardTrajectoryMismatch", "RewardEvidenceEnvelope.v1" + ) + + # Rule 4: release-grade without independent adjudication + bad_rg = dict(report) + bad_rg["release_grade"] = True + bad_rg["independent_adjudication"] = False + bad_rg.pop("integrity", None) + bad_rg = finalize(bad_rg) + write(inv / "release_grade_no_adjudication" / "artifact.json", bad_rg) + write_manifest( + inv / "release_grade_no_adjudication" / "manifest.json", + "ReleaseGradeAdjudication", + "VerifierAssuranceReport.v1", + ) + + # Rule 5: optimization gap without both cohorts + bad_gap = dict(report) + bad_gap["cohorts"] = [report["cohorts"][0]] + bad_gap["metrics"] = dict(report["metrics"]) + bad_gap["metrics"]["optimization_gap"] = "0.500000" + bad_gap.pop("integrity", None) + bad_gap = finalize(bad_gap) + write(inv / "optimization_gap_missing_cohort" / "artifact.json", bad_gap) + write_manifest( + inv / "optimization_gap_missing_cohort" / "manifest.json", + "OptimizationGapCohorts", + "VerifierAssuranceReport.v1", + ) + + # Rule 6: cohort missing access/compute + bad_cohort = base_campaign( + cohorts=[ + { + "cohort_id": "cohort-ordinary", + "cohort_kind": "ordinary", + "compute_exposure": { + "accounting_method": "wall_clock", + "wall_time_seconds": 600, + "query_count": 50, + }, + } + ] + ) + write(inv / "cohort_missing_access" / "artifact.json", bad_cohort) + write_manifest( + inv / "cohort_missing_access" / "manifest.json", + "CohortAccessClass", + "OptimizationCampaignManifest.v1", + ) + + # Rule 7: cohort count mismatch + bad_counts = dict(report) + bad_counts["cohorts"] = [ + { + **report["cohorts"][0], + "accept_count": 99, + "reject_count": 0, + "indeterminate_count": 0, + "included_result_count": 1, + }, + report["cohorts"][1], + ] + bad_counts.pop("integrity", None) + bad_counts = finalize(bad_counts) + write(inv / "cohort_count_mismatch" / "artifact.json", bad_counts) + write_manifest( + inv / "cohort_count_mismatch" / "manifest.json", + "CohortCountMismatch", + "VerifierAssuranceReport.v1", + ) + + # Rule 10: cross-artifact env version mismatch + ver_case = inv / "cross_artifact_version_mismatch" + write(ver_case / "campaign.json", campaign) + write(ver_case / "result.json", good_result) + bad_env_reward = base_reward(env_profile_id="env-other", env_profile_version="9.9.9") + write(ver_case / "artifact.json", bad_env_reward) + write_manifest( + ver_case / "manifest.json", + "CrossArtifactVersionMismatch", + "RewardEvidenceEnvelope.v1", + ) + + # Rule 12: excluded items not visible + bad_excl = dict(report) + bad_excl["excluded_items"] = [] + bad_excl["metrics"] = dict(report["metrics"]) + bad_excl["metrics"]["excluded_count"] = 3 + bad_excl.pop("integrity", None) + bad_excl = finalize(bad_excl) + write(inv / "excluded_items_invisible" / "artifact.json", bad_excl) + write_manifest( + inv / "excluded_items_invisible" / "manifest.json", + "ExcludedItemsVisible", + "VerifierAssuranceReport.v1", + ) + + # Rule 13: missing CI method + bad_ci = dict(report) + bad_ci["metrics"] = dict(report["metrics"]) + far = dict(report["metrics"]["false_accept_rate"]) + far["confidence_interval"] = { + "lower": "0", + "upper": "0", + "sample_size": 2, + "parameters": {"alpha": "0.05"}, + } + bad_ci["metrics"]["false_accept_rate"] = far + bad_ci.pop("integrity", None) + bad_ci = finalize(bad_ci) + write(inv / "missing_ci_method" / "artifact.json", bad_ci) + write_manifest( + inv / "missing_ci_method" / "manifest.json", + "CIMethodsDeclared", + "VerifierAssuranceReport.v1", + ) + + # Rule 15: negative indeterminate bucket + bad_indet = dict(report) + bad_indet["cohorts"] = [ + { + **report["cohorts"][0], + "accept_count": 2, + "reject_count": 0, + "indeterminate_count": -1, + "included_result_count": 1, + }, + report["cohorts"][1], + ] + bad_indet.pop("integrity", None) + bad_indet = finalize(bad_indet) + write(inv / "indeterminate_misclassification" / "artifact.json", bad_indet) + write_manifest( + inv / "indeterminate_misclassification" / "manifest.json", + "IndeterminateMisclassification", + "VerifierAssuranceReport.v1", + ) + + # Active reward with mandatory unresolved claims + bad_unresolved = base_reward( + lifecycle={"status": "active"}, + mandatory_unresolved_claim_ids=["claim.safety.no_egress"], + ) + write(inv / "active_reward_unresolved" / "artifact.json", bad_unresolved) + write_manifest( + inv / "active_reward_unresolved" / "manifest.json", + "ActiveRewardUnresolvedClaims", + "RewardEvidenceEnvelope.v1", + ) + + # Producer dialect: float reward (schema reject) + float_reward = base_reward() + float_reward["scalar_total"] = 1.5 # type: ignore[assignment] + float_reward.pop("integrity", None) + write(inv / "float_reward" / "artifact.json", float_reward) + write_manifest( + inv / "float_reward" / "manifest.json", + "is not of type 'string'", + "RewardEvidenceEnvelope.v1", + ) + + print("Wrote VA fixtures under", root) + + +if __name__ == "__main__": + main() diff --git a/test_vectors/hash/adjudicationrecord_v1.vector.json b/test_vectors/hash/adjudicationrecord_v1.vector.json new file mode 100644 index 00000000..6237ff58 --- /dev/null +++ b/test_vectors/hash/adjudicationrecord_v1.vector.json @@ -0,0 +1,6 @@ +{ + "artifact_type": "AdjudicationRecord.v1", + "input_file": "examples/verifier_assurance/valid/adjudication_basic/adjudication.json", + "expected_digest": "sha256:3b6b935a9f226e588419007dfd1445ef2dc1b8707b6c60d7982a052acc5368c3", + "canonical_json": "{\"adjudication_id\":\"adj-001\",\"artifact_type\":\"AdjudicationRecord.v1\",\"blinded\":true,\"conflict_of_interest_declared\":false,\"created_at\":\"2026-07-24T13:00:00Z\",\"disagreement\":false,\"escalation\":false,\"evidence_refs\":[],\"independence_declared\":true,\"integrity\":{\"artifact_digest\":\"sha256:86f586ae5b592104734272f11cdfa141a104fd2c11f4485b6c2f0e7ab35a5dc4\",\"canonicalization_version\":\"v1\"},\"label\":\"valid\",\"label_released_at\":\"2026-07-24T14:00:00Z\",\"producer\":\"pcs-core\",\"producer_version\":\"0.1.0\",\"protected_rationale\":{\"commitment_digest\":\"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"content_omitted\":true,\"location_class\":\"access_controlled_store\"},\"protocol_id\":\"adj-protocol\",\"protocol_version\":\"1.0.0\",\"schema_version\":\"v1\",\"source_commit\":\"e068794683959c52a19594a6d271dd5e69f3c999\",\"source_repo\":\"https://github.com/SentinelOps-CI/pcs-core\",\"subject\":{\"artifact_digest\":\"sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee\",\"artifact_id\":\"vr-001\",\"artifact_type\":\"VerificationResult.v1\"},\"votes\":[{\"role\":\"primary\",\"vote\":\"valid\",\"voter_pseudonym_id\":\"voter-a\"}]}" +} diff --git a/test_vectors/hash/optimizationcampaignmanifest_v1.vector.json b/test_vectors/hash/optimizationcampaignmanifest_v1.vector.json new file mode 100644 index 00000000..c0c8d28c --- /dev/null +++ b/test_vectors/hash/optimizationcampaignmanifest_v1.vector.json @@ -0,0 +1,6 @@ +{ + "artifact_type": "OptimizationCampaignManifest.v1", + "input_file": "examples/verifier_assurance/valid/campaign_basic/campaign.json", + "expected_digest": "sha256:fd790a71e8eeefcd40d6b2b3b1642b3f93b93669d518564a62fc4f9f2d93ff8f", + "canonical_json": "{\"access_class\":\"black_box\",\"algorithm\":\"random-search\",\"artifact_type\":\"OptimizationCampaignManifest.v1\",\"campaign_id\":\"camp-001\",\"checkpoint_ids\":[\"ckpt-0\"],\"cohorts\":[{\"access_class\":\"black_box\",\"cohort_id\":\"cohort-ordinary\",\"cohort_kind\":\"ordinary\",\"compute_exposure\":{\"accounting_method\":\"wall_clock\",\"query_count\":50,\"wall_time_seconds\":600}},{\"access_class\":\"adaptive\",\"cohort_id\":\"cohort-optimized\",\"cohort_kind\":\"optimized\",\"compute_exposure\":{\"accounting_method\":\"wall_clock\",\"query_count\":950,\"wall_time_seconds\":3000}}],\"compute_budget\":{\"accounting_method\":\"wall_clock\",\"gpu_hours\":\"0\",\"query_budget\":1000,\"wall_time_seconds\":3600},\"containment_policy\":\"offline-only\",\"created_at\":\"2026-07-24T11:00:00Z\",\"disclosure_policy\":\"public-summary\",\"env_profile_id\":\"env-lab\",\"env_profile_version\":\"1.0.0\",\"episode_count\":10,\"harness_id\":\"harness-a\",\"integrity\":{\"artifact_digest\":\"sha256:1adebaaa9854973c44977fb1bcf22c4792f827623804be8ac67279189811d4b8\",\"canonicalization_version\":\"v1\"},\"model_id\":\"model-a\",\"optimizer_settings\":{\"batch_size\":8,\"learning_rate\":\"0.01\"},\"policy_id\":\"policy-a\",\"producer\":\"pcs-core\",\"producer_version\":\"0.1.0\",\"query_count\":100,\"schema_version\":\"v1\",\"seeds\":[1,2,3],\"source_commit\":\"e068794683959c52a19594a6d271dd5e69f3c999\",\"source_repo\":\"https://github.com/SentinelOps-CI/pcs-core\",\"splits\":{\"holdout\":\"hold-v1\",\"train\":\"train-v1\",\"validation\":\"val-v1\"},\"target_verifier_profile\":{\"profile_digest\":\"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"verifier_profile_id\":\"vp-ovk-static-001\"},\"trajectory_count\":10,\"visibility\":{\"full_traces_retained\":false,\"public_summary\":true}}" +} diff --git a/test_vectors/hash/rewardevidenceenvelope_v1.vector.json b/test_vectors/hash/rewardevidenceenvelope_v1.vector.json new file mode 100644 index 00000000..adee1155 --- /dev/null +++ b/test_vectors/hash/rewardevidenceenvelope_v1.vector.json @@ -0,0 +1,6 @@ +{ + "artifact_type": "RewardEvidenceEnvelope.v1", + "input_file": "examples/verifier_assurance/valid/reward_scalar/reward.json", + "expected_digest": "sha256:c8e4f980c201623cc795592cd1302f29a04f95b7960c8cc719403f115bad215b", + "canonical_json": "{\"artifact_type\":\"RewardEvidenceEnvelope.v1\",\"authority\":{\"authority_class\":\"producer\",\"authority_id\":\"labtrust-reward\"},\"claims_issued\":[\"claim.safety.no_egress\"],\"claims_rejected\":[],\"claims_unresolved\":[],\"components\":[{\"claim_id\":\"claim.safety.no_egress\",\"component_id\":\"safety\",\"guarantee_class\":\"certificate_checked\",\"value\":\"1.0\"},{\"component_id\":\"quality\",\"guarantee_class\":\"empirically_measured\",\"value\":\"0.5\"}],\"composition_function\":\"sum\",\"created_at\":\"2026-07-24T12:10:00Z\",\"env_profile_id\":\"env-lab\",\"env_profile_version\":\"1.0.0\",\"episode_id\":\"ep-1\",\"integrity\":{\"artifact_digest\":\"sha256:5acb6ac017e8ff3e05eeb7a7453fec985b0424a05137191a4b505cccef2b7889\",\"canonicalization_version\":\"v1\"},\"lifecycle\":{\"status\":\"active\"},\"mandatory_unresolved_claim_ids\":[],\"producer\":\"LabTrust-Gym\",\"producer_version\":\"0.1.0\",\"profile_refs\":[{\"applicability_status\":\"active\",\"profile_digest\":\"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"verifier_profile_id\":\"vp-ovk-static-001\"}],\"reward_envelope_id\":\"rew-001\",\"scalar_total\":\"1.5\",\"schema_version\":\"v1\",\"source_commit\":\"e068794683959c52a19594a6d271dd5e69f3c999\",\"source_repo\":\"https://github.com/SentinelOps-CI/pcs-core\",\"state_digest\":\"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"step_index\":0,\"trajectory_digest\":\"sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd\",\"verifier_result_refs\":[{\"artifact_digest\":\"sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee\",\"artifact_id\":\"vr-001\",\"artifact_type\":\"VerificationResult.v1\"}]}" +} diff --git a/test_vectors/hash/verificationresult_v1.vector.json b/test_vectors/hash/verificationresult_v1.vector.json new file mode 100644 index 00000000..49839594 --- /dev/null +++ b/test_vectors/hash/verificationresult_v1.vector.json @@ -0,0 +1,6 @@ +{ + "artifact_type": "VerificationResult.v1", + "input_file": "examples/verifier_assurance/valid/result_accept/result.json", + "expected_digest": "sha256:df3368113125f16f753697307e13cefb484817978cc076e9c2fe44c8d40e4d11", + "canonical_json": "{\"artifact_type\":\"VerificationResult.v1\",\"check_groups\":[{\"checks\":[{\"check_id\":\"process.complete\",\"mandatory\":true,\"status\":\"passed\"}],\"kind\":\"process\"},{\"checks\":[{\"check_id\":\"evidence.present\",\"mandatory\":true,\"status\":\"passed\"}],\"kind\":\"evidence\"}],\"claim_ids\":[\"claim.safety.no_egress\"],\"created_at\":\"2026-07-24T12:05:00Z\",\"decision\":\"accept\",\"execution_status\":\"completed\",\"initial_state_digest\":\"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"input_bundle_digest\":\"sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc\",\"integrity\":{\"artifact_digest\":\"sha256:7fa5afbf3d931d448c724fb0542101f7c8bc5f14cd19456c3344d3d8d23a0ad2\",\"canonicalization_version\":\"v1\"},\"normalization_applied\":true,\"normalized_result_digest\":\"sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\",\"producer\":\"OVK\",\"producer_version\":\"0.1.0\",\"raw_backend_output_digest\":\"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"resource_limits\":{\"memory_bytes\":1048576,\"wall_time_ms\":1000},\"schema_version\":\"v1\",\"source_commit\":\"e068794683959c52a19594a6d271dd5e69f3c999\",\"source_repo\":\"https://github.com/SentinelOps-CI/pcs-core\",\"terminal_state_digest\":\"sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\",\"trajectory_digest\":\"sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd\",\"verification_result_id\":\"vr-001\",\"verifier_profile\":{\"profile_digest\":\"sha256:c515b34f1f61d9b8d139d2a14a92409b2d45ffee16d16bb4c89913db1ebd53b0\",\"verifier_profile_id\":\"vp-ovk-static-001\"}}" +} diff --git a/test_vectors/hash/verifierassurancereport_v1.vector.json b/test_vectors/hash/verifierassurancereport_v1.vector.json new file mode 100644 index 00000000..290ea914 --- /dev/null +++ b/test_vectors/hash/verifierassurancereport_v1.vector.json @@ -0,0 +1,6 @@ +{ + "artifact_type": "VerifierAssuranceReport.v1", + "input_file": "examples/verifier_assurance/valid/report_rebuild/report.json", + "expected_digest": "sha256:49ff7573b1523e3596a8e2be0c5a7165273cb7cded0d373916349321946f763c", + "canonical_json": "{\"applicability_limits\":[\"synthetic fixture only\"],\"artifact_type\":\"VerifierAssuranceReport.v1\",\"campaign_ref\":{\"artifact_digest\":\"sha256:fd790a71e8eeefcd40d6b2b3b1642b3f93b93669d518564a62fc4f9f2d93ff8f\",\"artifact_id\":\"camp-001\",\"artifact_type\":\"OptimizationCampaignManifest.v1\"},\"cohorts\":[{\"accept_count\":1,\"access_class\":\"black_box\",\"cohort_id\":\"cohort-ordinary\",\"cohort_kind\":\"ordinary\",\"compute_exposure\":{\"accounting_method\":\"wall_clock\",\"query_count\":50,\"wall_time_seconds\":600},\"included_result_count\":1,\"indeterminate_count\":0,\"reject_count\":0},{\"accept_count\":0,\"access_class\":\"adaptive\",\"cohort_id\":\"cohort-optimized\",\"cohort_kind\":\"optimized\",\"compute_exposure\":{\"accounting_method\":\"wall_clock\",\"query_count\":950,\"wall_time_seconds\":3000},\"included_result_count\":1,\"indeterminate_count\":0,\"reject_count\":1}],\"created_at\":\"2026-07-24T15:00:00Z\",\"excluded_items\":[{\"item_id\":\"ex-1\",\"reason_code\":\"out_of_scope\"}],\"independent_adjudication\":true,\"integrity\":{\"artifact_digest\":\"sha256:b7396e3817f27ab95f8b28bf89faea4d7c670727233ac4eb6c54a2778d23ac84\",\"canonicalization_version\":\"v1\"},\"metrics\":{\"abstention_rate\":{\"confidence_interval\":{\"lower\":\"0.000000\",\"method\":\"wilson\",\"parameters\":{\"alpha\":\"0.05\",\"declared\":true},\"sample_size\":2,\"upper\":\"0.000000\"},\"denominator\":2,\"numerator\":0,\"rate\":\"0.000000\"},\"adjudication_coverage\":{\"confidence_interval\":{\"lower\":\"1.000000\",\"method\":\"wilson\",\"parameters\":{\"alpha\":\"0.05\",\"declared\":true},\"sample_size\":2,\"upper\":\"1.000000\"},\"denominator\":2,\"numerator\":2,\"rate\":\"1.000000\"},\"cost_decimal\":\"0\",\"excluded_count\":1,\"exploit_family_counts\":{},\"false_accept_rate\":{\"confidence_interval\":{\"lower\":\"0.000000\",\"method\":\"wilson\",\"parameters\":{\"alpha\":\"0.05\",\"declared\":true},\"sample_size\":2,\"upper\":\"0.000000\"},\"denominator\":2,\"numerator\":0,\"rate\":\"0.000000\"},\"false_reject_rate\":{\"confidence_interval\":{\"lower\":\"0.000000\",\"method\":\"wilson\",\"parameters\":{\"alpha\":\"0.05\",\"declared\":true},\"sample_size\":2,\"upper\":\"0.000000\"},\"denominator\":2,\"numerator\":0,\"rate\":\"0.000000\"},\"inter_verifier_disagreement_rate\":{\"confidence_interval\":{\"lower\":\"0.000000\",\"method\":\"wilson\",\"parameters\":{\"alpha\":\"0.05\",\"declared\":true},\"sample_size\":2,\"upper\":\"0.000000\"},\"denominator\":2,\"numerator\":0,\"rate\":\"0.000000\"},\"latency_ms_p50\":0,\"optimization_gap\":\"-1.000000\",\"optimized_accept_rate\":{\"confidence_interval\":{\"lower\":\"0.000000\",\"method\":\"wilson\",\"parameters\":{\"alpha\":\"0.05\",\"declared\":true},\"sample_size\":1,\"upper\":\"0.000000\"},\"denominator\":1,\"numerator\":0,\"rate\":\"0.000000\"},\"ordinary_accept_rate\":{\"confidence_interval\":{\"lower\":\"1.000000\",\"method\":\"wilson\",\"parameters\":{\"alpha\":\"0.05\",\"declared\":true},\"sample_size\":1,\"upper\":\"1.000000\"},\"denominator\":1,\"numerator\":1,\"rate\":\"1.000000\"},\"queries_to_first_exploit\":null,\"reward_invalidation_rate\":{\"confidence_interval\":{\"lower\":\"0.000000\",\"method\":\"wilson\",\"parameters\":{\"alpha\":\"0.05\",\"declared\":true},\"sample_size\":2,\"upper\":\"0.000000\"},\"denominator\":2,\"numerator\":0,\"rate\":\"0.000000\"},\"sample_size\":2,\"time_to_first_exploit_seconds\":null,\"unadjudicated_count\":0},\"producer\":\"pcs-core\",\"producer_version\":\"0.1.0\",\"release_grade\":true,\"report_id\":\"rep-001\",\"schema_version\":\"v1\",\"source_commit\":\"e068794683959c52a19594a6d271dd5e69f3c999\",\"source_repo\":\"https://github.com/SentinelOps-CI/pcs-core\",\"unadjudicated_items\":[]}" +} diff --git a/test_vectors/hash/verifierinvocationrecord_v1.vector.json b/test_vectors/hash/verifierinvocationrecord_v1.vector.json new file mode 100644 index 00000000..294d5e9f --- /dev/null +++ b/test_vectors/hash/verifierinvocationrecord_v1.vector.json @@ -0,0 +1,6 @@ +{ + "artifact_type": "VerifierInvocationRecord.v1", + "input_file": "examples/verifier_assurance/valid/invocation_basic/invocation.json", + "expected_digest": "sha256:67b706c3107fde36b4c6ccbd502d331aca7a688f45277e2edd8b2f6af070d8f4", + "canonical_json": "{\"artifact_type\":\"VerifierInvocationRecord.v1\",\"assumptions\":[],\"canonicalization_version\":\"v1\",\"command\":{\"argv\":[\"opa\",\"eval\",\"-d\",\"policy.rego\",\"data.allow\"],\"cwd\":\".\",\"env_digest\":\"sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc\",\"limits\":{\"timeout_ms\":30000}},\"compiled_obligation\":{\"digest\":\"sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc\",\"media_type\":\"application/json\",\"path\":\"compiled_obligation.json\"},\"completed_at\":\"2026-07-24T16:01:01Z\",\"exit\":{\"code\":0,\"kind\":\"exited\"},\"guarantee_class\":\"runtime_observed\",\"input_digest\":\"sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\",\"integrity\":{\"artifact_digest\":\"sha256:a8761bfcdda71d47ac64d26513d378d258404f9042d4ad1a8816dcfc935a6ac1\",\"canonicalization_version\":\"v1\"},\"invocation_id\":\"vi-ovk-pin-001\",\"limits\":[],\"normalized_result_digest\":\"sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\",\"normalizer_version\":\"ovk.normalize.v1\",\"producer\":\"OVK\",\"producer_version\":\"0.4.0\",\"profile_ref\":{\"artifact_type\":\"VerifierProfile.v1\",\"profile_digest\":\"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"verifier_profile_id\":\"vp-opa-demo-001\"},\"raw_backend_result_digest\":\"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"schema_version\":\"v1\",\"source_commit\":\"e068794683959c52a19594a6d271dd5e69f3c999\",\"source_repo\":\"https://github.com/SentinelOps-CI/pcs-core\",\"started_at\":\"2026-07-24T16:01:00Z\",\"stderr_digest\":\"sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee\",\"stdout_digest\":\"sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd\"}" +} diff --git a/test_vectors/hash/verifiermutationmanifest_v1.vector.json b/test_vectors/hash/verifiermutationmanifest_v1.vector.json new file mode 100644 index 00000000..61265fdc --- /dev/null +++ b/test_vectors/hash/verifiermutationmanifest_v1.vector.json @@ -0,0 +1,6 @@ +{ + "artifact_type": "VerifierMutationManifest.v1", + "input_file": "examples/verifier_assurance/valid/mutation_timeout/mutation.json", + "expected_digest": "sha256:2ea6399e73595607de81e3f39f03c9c556bc671ac48813a9a1ab6791cf56e9d2", + "canonical_json": "{\"artifact_type\":\"VerifierMutationManifest.v1\",\"base_profile_ref\":{\"artifact_type\":\"VerifierProfile.v1\",\"profile_digest\":\"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"verifier_profile_id\":\"vp-opa-demo-001\"},\"canonicalization_version\":\"v1\",\"created_at\":\"2026-07-24T16:03:00Z\",\"expected_effect\":\"Changes execution timeout.\",\"integrity\":{\"artifact_digest\":\"sha256:35d8c217a200611e57fdc6039018429aa27e1169717678bed7c205d1dd95f025\",\"canonicalization_version\":\"v1\"},\"mutated_profile_ref\":{\"artifact_type\":\"VerifierProfile.v1\",\"profile_digest\":\"sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\",\"verifier_profile_id\":\"vp-opa-demo-001-mut-alter_timeout\"},\"mutation_class\":\"alter_timeout\",\"mutation_id\":\"mut-alter_timeout-001\",\"parameters\":{\"timeout_ms\":1000},\"producer\":\"OVK\",\"producer_version\":\"0.4.0\",\"production_prohibition\":true,\"schema_version\":\"v1\",\"source_commit\":\"e068794683959c52a19594a6d271dd5e69f3c999\",\"source_repo\":\"https://github.com/SentinelOps-CI/pcs-core\",\"supported_by_adapter\":true}" +} diff --git a/test_vectors/hash/verifierprofile_v1.vector.json b/test_vectors/hash/verifierprofile_v1.vector.json new file mode 100644 index 00000000..a53ea77f --- /dev/null +++ b/test_vectors/hash/verifierprofile_v1.vector.json @@ -0,0 +1,6 @@ +{ + "artifact_type": "VerifierProfile.v1", + "input_file": "examples/verifier_assurance/valid/profile_basic/profile.json", + "expected_digest": "sha256:c515b34f1f61d9b8d139d2a14a92409b2d45ffee16d16bb4c89913db1ebd53b0", + "canonical_json": "{\"applicability\":{\"status\":\"active\",\"valid_from\":\"2026-07-01T00:00:00Z\"},\"artifact_type\":\"VerifierProfile.v1\",\"assumptions\":[\"host OS is Linux\"],\"claim_surface\":{\"guarantee_class\":\"certificate_checked\",\"out_of_scope_claim_ids\":[\"claim.formal.full_correctness\"],\"supported_claim_ids\":[\"claim.safety.no_egress\"]},\"configuration\":{\"config_digest\":\"sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\",\"model_digest\":null,\"policy_digest\":null,\"prompt_digest\":null,\"resource_limit_digest\":\"sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc\"},\"created_at\":\"2026-07-24T12:00:00Z\",\"implementation\":{\"implementation_digest\":\"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"language\":\"rust\",\"name\":\"ovk-static-checker\",\"version\":\"1.0.0\"},\"integrity\":{\"artifact_digest\":\"sha256:3af8ff33a41e4a1af25cd184b4f610f7d80da942100fb01667419195b3c5ff91\",\"canonicalization_version\":\"v1\"},\"known_blind_spots\":[\"does not observe runtime network\"],\"mechanism\":{\"allows_abstention\":true,\"description\":\"Static claim checker\",\"determinism\":\"deterministic\",\"mechanism_class\":\"static_analysis\"},\"producer\":\"OVK\",\"producer_version\":\"0.1.0\",\"schema_version\":\"v1\",\"source_commit\":\"e068794683959c52a19594a6d271dd5e69f3c999\",\"source_repo\":\"https://github.com/SentinelOps-CI/pcs-core\",\"verifier_profile_id\":\"vp-ovk-static-001\"}" +} diff --git a/test_vectors/hash/verifierreplayreport_v1.vector.json b/test_vectors/hash/verifierreplayreport_v1.vector.json new file mode 100644 index 00000000..558e269c --- /dev/null +++ b/test_vectors/hash/verifierreplayreport_v1.vector.json @@ -0,0 +1,6 @@ +{ + "artifact_type": "VerifierReplayReport.v1", + "input_file": "examples/verifier_assurance/valid/replay_matched/replay.json", + "expected_digest": "sha256:17d0d748480fb3155408cd46f6d32841232e603aed0a605823a371f702c721b5", + "canonical_json": "{\"artifact_type\":\"VerifierReplayReport.v1\",\"canonicalization_version\":\"v1\",\"created_at\":\"2026-07-24T16:02:00Z\",\"determinism\":\"deterministic\",\"drift\":{\"claim_match\":true,\"normalized_digest_match\":true,\"raw_digest_match\":true},\"integrity\":{\"artifact_digest\":\"sha256:f2af1afd1ee217db70273ebce5d695af47bfbb66e975b4411b5860b3b79f85af\",\"canonicalization_version\":\"v1\"},\"original_invocation_ref\":{\"artifact_type\":\"VerifierInvocationRecord.v1\",\"invocation_digest\":\"sha256:a8761bfcdda71d47ac64d26513d378d258404f9042d4ad1a8816dcfc935a6ac1\",\"invocation_id\":\"vi-ovk-pin-001\"},\"original_normalized_digest\":\"sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\",\"original_raw_digest\":\"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"producer\":\"OVK\",\"producer_version\":\"0.4.0\",\"profile_ref\":{\"artifact_type\":\"VerifierProfile.v1\",\"profile_digest\":\"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"verifier_profile_id\":\"vp-opa-demo-001\"},\"replay_id\":\"rr-ovk-pin-001\",\"replay_normalized_digest\":\"sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\",\"replay_raw_digest\":\"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"replay_status\":\"matched\",\"schema_version\":\"v1\",\"source_commit\":\"e068794683959c52a19594a6d271dd5e69f3c999\",\"source_repo\":\"https://github.com/SentinelOps-CI/pcs-core\"}" +} diff --git a/typescript/packages/core/src/index.ts b/typescript/packages/core/src/index.ts index b3d10048..eeb2c416 100644 --- a/typescript/packages/core/src/index.ts +++ b/typescript/packages/core/src/index.ts @@ -4,3 +4,4 @@ export * from "./pfCore.js"; export * from "./schema.js"; export * from "./status.js"; export * from "./validate.js"; +export * from "./verifierAssurance.js"; diff --git a/typescript/packages/core/src/schema.ts b/typescript/packages/core/src/schema.ts index c2f7dbd9..a719ef19 100644 --- a/typescript/packages/core/src/schema.ts +++ b/typescript/packages/core/src/schema.ts @@ -85,6 +85,15 @@ const ARTIFACT_SCHEMAS: Record = { "PFCoreKernelManifest.v0": "PFCoreKernelManifest.v0.schema.json", "PFCoreReleaseBundleManifest.v0": "PFCoreReleaseBundleManifest.v0.schema.json", "ArtifactIntegrity.v1": "ArtifactIntegrity.v1.schema.json", + "VerifierProfile.v1": "VerifierProfile.v1.schema.json", + "VerificationResult.v1": "VerificationResult.v1.schema.json", + "VerifierInvocationRecord.v1": "VerifierInvocationRecord.v1.schema.json", + "VerifierReplayReport.v1": "VerifierReplayReport.v1.schema.json", + "VerifierMutationManifest.v1": "VerifierMutationManifest.v1.schema.json", + "RewardEvidenceEnvelope.v1": "RewardEvidenceEnvelope.v1.schema.json", + "OptimizationCampaignManifest.v1": "OptimizationCampaignManifest.v1.schema.json", + "AdjudicationRecord.v1": "AdjudicationRecord.v1.schema.json", + "VerifierAssuranceReport.v1": "VerifierAssuranceReport.v1.schema.json", "FormatAssertionProbe.v0": "FormatAssertionProbe.v0.schema.json", "ExternalAttestation.v0": "ExternalAttestation.v0.schema.json", }; diff --git a/typescript/packages/core/src/tests/examples.test.ts b/typescript/packages/core/src/tests/examples.test.ts index 5234ebcc..00249d46 100644 --- a/typescript/packages/core/src/tests/examples.test.ts +++ b/typescript/packages/core/src/tests/examples.test.ts @@ -41,6 +41,11 @@ import { resolveCertificateModeDefault, resolveToolMapping, } from "../pfCore.js"; +import { + constructVaArtifact, + validateVaSemantics, + verifyAssuranceReport, +} from "../verifierAssurance.js"; import { detectArtifactType, validateArtifact, ValidationError, type ArtifactType } from "../validate.js"; const examplesDir = join(dirname(fileURLToPath(import.meta.url)), "../../../../../examples"); @@ -464,7 +469,7 @@ test("shared hash vectors match test_vectors/hash fixtures", () => { expected_digest: string; canonical_json: string; }; - const inputPath = vector.input ?? vector.input_file ?? ""; + const inputPath = (vector.input ?? vector.input_file ?? "").replace(/\\/g, "/"); const data = load(inputPath.replace(/^examples\//, "")); assert.equal(Buffer.from(canonicalJsonBytes(data)).toString("utf8"), vector.canonical_json); assert.equal(canonicalHash(data), vector.expected_digest); @@ -580,3 +585,84 @@ test("canonical_json_v1 accept and release-reject vectors", () => { assert.ok(canonicalHashLegacy(data).startsWith("sha256:"), caseRow.case_id); } }); + +test("verifier assurance valid fixtures pass semantics", () => { + const cases: Array<[string, string]> = [ + ["verifier_assurance/valid/profile_basic/profile.json", "VerifierProfile.v1"], + ["verifier_assurance/valid/result_accept/result.json", "VerificationResult.v1"], + ["verifier_assurance/valid/reward_scalar/reward.json", "RewardEvidenceEnvelope.v1"], + ["verifier_assurance/valid/campaign_basic/campaign.json", "OptimizationCampaignManifest.v1"], + ["verifier_assurance/valid/adjudication_basic/adjudication.json", "AdjudicationRecord.v1"], + ["verifier_assurance/valid/report_rebuild/report.json", "VerifierAssuranceReport.v1"], + ]; + for (const [rel, expected] of cases) { + const data = load(rel); + assert.equal(detectArtifactType(data), expected, rel); + assert.deepEqual(validateVaSemantics(data, expected), []); + validateArtifact(data, expected as ArtifactType); + } +}); + +test("verifier assurance invalid fixtures emit expected codes", () => { + const cases: Array<[string, string]> = [ + ["timeout_accept", "FailClosedDecision"], + ["accept_mandatory_failure", "AcceptWithMandatoryFailure"], + ["identical_normalization_digests", "IdenticalNormalizationDigests"], + ["reward_total_mismatch", "RewardCompositionMismatch"], + ["revoked_profile_active_reward", "RevokedProfileGate"], + ["missing_rationale_commitment", "RationaleCommitment"], + ["short_source_commit", "InvalidSourceCommit"], + ["release_grade_no_adjudication", "ReleaseGradeAdjudication"], + ["optimization_gap_missing_cohort", "OptimizationGapCohorts"], + ["cohort_missing_access", "CohortAccessClass"], + ["cohort_count_mismatch", "CohortCountMismatch"], + ["excluded_items_invisible", "ExcludedItemsVisible"], + ["missing_ci_method", "CIMethodsDeclared"], + ["indeterminate_misclassification", "IndeterminateMisclassification"], + ["active_reward_unresolved", "ActiveRewardUnresolvedClaims"], + ]; + for (const [dir, code] of cases) { + const manifest = load(`verifier_assurance/invalid/${dir}/manifest.json`); + const artifactFile = String(manifest.artifact_file ?? "artifact.json"); + const artifactType = String(manifest.artifact_type); + const data = load(`verifier_assurance/invalid/${dir}/${artifactFile}`); + const issues = validateVaSemantics(data, artifactType); + assert.ok( + issues.some((i) => i.code === code), + `${dir}: expected ${code}, got ${JSON.stringify(issues)}`, + ); + } +}); + +test("verifier assurance verify report digest parity", () => { + const report = load("verifier_assurance/valid/report_rebuild/report.json"); + assert.deepEqual(verifyAssuranceReport(report), []); + const tampered = { ...report, report_id: "tampered-id" }; + const issues = verifyAssuranceReport(tampered); + assert.ok(issues.some((i) => i.code === "ReportDigestMismatch"), JSON.stringify(issues)); +}); + +test("verifier assurance constructor retains unknown fields", () => { + assert.throws(() => constructVaArtifact("VerifierProfile.v1", {})); + const fields: Record = { + schema_version: "v1", + artifact_type: "VerifierProfile.v1", + verifier_profile_id: "vp", + created_at: "2026-07-24T00:00:00Z", + producer: "pcs-core", + producer_version: "0.1.0", + source_repo: "https://example.invalid", + source_commit: "e068794683959c52a19594a6d271dd5e69f3c999", + implementation: {}, + configuration: {}, + mechanism: {}, + claim_surface: {}, + applicability: {}, + assumptions: [], + known_blind_spots: [], + integrity: { canonicalization_version: "v1", artifact_digest: "sha256:" + "a".repeat(64) }, + extra_field: true, + }; + const built = constructVaArtifact("VerifierProfile.v1", fields); + assert.equal(built.extra_field, true); +}); diff --git a/typescript/packages/core/src/validate.ts b/typescript/packages/core/src/validate.ts index cc8f60b8..3e9c096f 100644 --- a/typescript/packages/core/src/validate.ts +++ b/typescript/packages/core/src/validate.ts @@ -10,6 +10,10 @@ import { import { ARTIFACT_STATUSES, TRACE_CERTIFICATE_STATUSES } from "./status.js"; import { isZeroSourceCommit } from "./hash.js"; import { validateSchema } from "./schema.js"; +import { + isVaArtifactType, + validateVaSemanticsStrings, +} from "./verifierAssurance.js"; export class ValidationError extends Error { readonly errors: string[]; @@ -60,6 +64,12 @@ export type ArtifactType = | "PFCoreKernelManifest.v0" | "PFCoreReleaseBundleManifest.v0" | "ArtifactIntegrity.v1" + | "VerifierProfile.v1" + | "VerificationResult.v1" + | "RewardEvidenceEnvelope.v1" + | "OptimizationCampaignManifest.v1" + | "AdjudicationRecord.v1" + | "VerifierAssuranceReport.v1" | "FormatAssertionProbe.v0" | "ExternalAttestation.v0" | "ReleaseManifest.v0" @@ -127,6 +137,12 @@ const EXPLICIT_ARTIFACT_TYPES = new Set([ "PFCoreKernelManifest.v0", "PFCoreReleaseBundleManifest.v0", "ArtifactIntegrity.v1", + "VerifierProfile.v1", + "VerificationResult.v1", + "RewardEvidenceEnvelope.v1", + "OptimizationCampaignManifest.v1", + "AdjudicationRecord.v1", + "VerifierAssuranceReport.v1", "FormatAssertionProbe.v0", "ExternalAttestation.v0", ]); @@ -520,6 +536,14 @@ export function validateArtifact( } return; } + // VA *.v1: mirror Python early-return (dedicated semantic module). + if (isVaArtifactType(type)) { + errors.push(...validateVaSemanticsStrings(data, type)); + if (errors.length > 0) { + throw new ValidationError(`Validation failed for ${type}`, errors); + } + return; + } checkSourceCommits(data, "", errors, false); if (type === "RuntimeReceipt.v0") { diff --git a/typescript/packages/core/src/verifierAssurance.ts b/typescript/packages/core/src/verifierAssurance.ts new file mode 100644 index 00000000..58edc540 --- /dev/null +++ b/typescript/packages/core/src/verifierAssurance.ts @@ -0,0 +1,999 @@ +/** + * Verifier Assurance (VA) *.v1 semantic validation and report verify. + * Error codes mirror python/pcs_core/verifier_assurance_validate.py. + */ + +import { canonicalHash, SIGNATURE_FIELD } from "./hash.js"; + +export interface SemanticIssue { + code: string; + path: string; + message: string; +} + +export function formatSemanticIssue(issue: SemanticIssue): string { + return `${issue.code} at ${issue.path}: ${issue.message}`; +} + +const FAIL_CLOSED = new Set([ + "timeout", + "unavailable", + "malformed_input", + "unsupported_scope", + "error", + "cancelled", + "resource_exhausted", +]); + +const SECRET_KEY_HINTS = [ + "password", + "passwd", + "secret", + "token", + "api_key", + "apikey", + "authorization", + "private_key", + "access_key", +]; + +const GUARANTEE_RANK: Record = { + unchecked_advisory: 0, + observational: 1, + runtime_observed: 1, + empirically_measured: 2, + human_reviewed: 3, + certificate_checked: 4, + formally_checked: 5, +}; + +const VA_ARTIFACT_TYPES = new Set([ + "VerifierProfile.v1", + "VerificationResult.v1", + "RewardEvidenceEnvelope.v1", + "OptimizationCampaignManifest.v1", + "AdjudicationRecord.v1", + "VerifierAssuranceReport.v1", +]); + +export function isVaArtifactType(artifactType: string): boolean { + return VA_ARTIFACT_TYPES.has(artifactType); +} + +function issue(code: string, path: string, message: string): SemanticIssue { + return { code, path, message }; +} + +function forbidLegacySignature( + data: Record, + artifactType: string, +): SemanticIssue[] { + if (SIGNATURE_FIELD in data) { + return [ + issue( + "LegacySignatureOrDigest", + SIGNATURE_FIELD, + `${artifactType}: ${SIGNATURE_FIELD} is forbidden on VA *.v1 roots`, + ), + ]; + } + return []; +} + +function requireIntegrity( + data: Record, + artifactType: string, +): SemanticIssue[] { + const integrity = data.integrity; + if (!integrity || typeof integrity !== "object" || Array.isArray(integrity)) { + return [ + issue( + "MissingIntegrity", + "integrity", + `${artifactType}: nested integrity envelope is required`, + ), + ]; + } + const obj = integrity as Record; + const issues: SemanticIssue[] = []; + if (obj.canonicalization_version !== "v1") { + issues.push( + issue("BadCanonicalizationVersion", "integrity.canonicalization_version", "must be v1"), + ); + } + const digest = obj.artifact_digest; + if (typeof digest !== "string" || !digest.startsWith("sha256:")) { + issues.push( + issue("BadArtifactDigest", "integrity.artifact_digest", "must be sha256:<64 hex>"), + ); + } + return issues; +} + +function checkZeroCommit(commit: unknown, path: string): SemanticIssue[] { + if ( + typeof commit === "string" && + commit.length === 40 && + /^0+$/.test(commit) + ) { + return [issue("PlaceholderCommit", path, "placeholder zero git commit rejected")]; + } + return []; +} + +function isFullHexCommit(commit: string): boolean { + return commit.length === 40 && /^[0-9a-f]{40}$/.test(commit); +} + +function checkSecretLeaks(envEntries: unknown, path: string): SemanticIssue[] { + if (!envEntries || typeof envEntries !== "object" || Array.isArray(envEntries)) { + return []; + } + const issues: SemanticIssue[] = []; + for (const [key, value] of Object.entries(envEntries as Record)) { + const keyL = key.toLowerCase(); + if (SECRET_KEY_HINTS.some((hint) => keyL.includes(hint))) { + issues.push( + issue( + "SecretKeyName", + `${path}.${key}`, + `redacted_environment key ${JSON.stringify(key)} looks like a secret name`, + ), + ); + } + if (typeof value === "string") { + const valueL = value.toLowerCase(); + if (value.startsWith("sk-") || valueL.includes("begin private key")) { + issues.push(issue("SecretValue", `${path}.${key}`, "value appears to contain a secret")); + } + } + } + return issues; +} + +type Decimal = { mantissa: bigint; scale: number }; + +function parseDecimalStr(s: string): Decimal | null { + const trimmed = s.trim(); + if (!trimmed) return null; + const negative = trimmed.startsWith("-"); + const body = + negative || trimmed.startsWith("+") ? trimmed.slice(1) : trimmed; + if (!body || !/^\d*\.?\d*$/.test(body) || (body.match(/\./g) ?? []).length > 1) { + return null; + } + const [intPart, fracPart = ""] = body.split("."); + if (!intPart && !fracPart) return null; + const combined = `${intPart || "0"}${fracPart}`; + try { + let mantissa = BigInt(combined); + if (negative) mantissa = -mantissa; + return { mantissa, scale: fracPart.length }; + } catch { + return null; + } +} + +function parseDecimal( + value: unknown, + path: string, +): { ok: Decimal } | { err: SemanticIssue } { + if (typeof value !== "string") { + return { err: issue("DecimalType", path, "decimal values must be strings") }; + } + const parsed = parseDecimalStr(value); + if (!parsed) { + return { + err: issue("DecimalParse", path, `invalid decimal string ${JSON.stringify(value)}`), + }; + } + return { ok: parsed }; +} + +function rescale(d: Decimal, toScale: number): bigint { + if (toScale >= d.scale) { + return d.mantissa * 10n ** BigInt(toScale - d.scale); + } + return d.mantissa / 10n ** BigInt(d.scale - toScale); +} + +function decimalsEqual(a: Decimal, b: Decimal): boolean { + const scale = Math.max(a.scale, b.scale); + return rescale(a, scale) === rescale(b, scale); +} + +function addDecimals(a: Decimal, b: Decimal): Decimal { + const scale = Math.max(a.scale, b.scale); + return { mantissa: rescale(a, scale) + rescale(b, scale), scale }; +} + +export function validateVerifierProfileSemantics( + data: Record, +): SemanticIssue[] { + const issues = [ + ...forbidLegacySignature(data, "VerifierProfile.v1"), + ...requireIntegrity(data, "VerifierProfile.v1"), + ]; + const commit = data.source_commit; + if (typeof commit !== "string" || !isFullHexCommit(commit)) { + issues.push( + issue( + "InvalidSourceCommit", + "source_commit", + "source_commit must be a full 40-char lowercase hex SHA", + ), + ); + } + issues.push(...checkZeroCommit(commit, "source_commit")); + const configuration = data.configuration; + if (configuration && typeof configuration === "object" && !Array.isArray(configuration)) { + const cfg = configuration as Record; + for (const key of [ + "policy_digest", + "model_digest", + "prompt_digest", + "resource_limit_digest", + ]) { + if (!(key in cfg)) { + issues.push( + issue( + "MissingNullDigestSlot", + `configuration.${key}`, + "inapplicable config digests must be present as explicit null", + ), + ); + } + } + } + const impl = data.implementation; + if (impl && typeof impl === "object" && !Array.isArray(impl)) { + const digest = (impl as Record).implementation_digest; + if (digest === undefined || digest === null || digest === "") { + issues.push( + issue( + "MissingImplementationDigest", + "implementation.implementation_digest", + "implementation_digest is required", + ), + ); + } + } + const applicability = data.applicability; + if (applicability && typeof applicability === "object" && !Array.isArray(applicability)) { + const app = applicability as Record; + if (app.status === "revoked" && !app.revocation_reason) { + issues.push( + issue( + "MissingRevocationReason", + "applicability.revocation_reason", + "revoked profiles require revocation_reason", + ), + ); + } + if (app.status === "superseded" && !app.superseded_by_profile_id) { + issues.push( + issue( + "MissingSupersededBy", + "applicability.superseded_by_profile_id", + "superseded profiles require superseded_by_profile_id", + ), + ); + } + } + const redacted = data.redacted_environment; + if (redacted && typeof redacted === "object" && !Array.isArray(redacted)) { + issues.push( + ...checkSecretLeaks( + (redacted as Record).entries, + "redacted_environment.entries", + ), + ); + } + const schemaDoc = data.configuration_schema; + const schemaDigest = data.configuration_schema_digest; + if ( + schemaDoc && + typeof schemaDoc === "object" && + !Array.isArray(schemaDoc) && + typeof schemaDigest === "string" + ) { + try { + const recomputed = canonicalHash(schemaDoc as Record); + if (recomputed !== schemaDigest) { + issues.push( + issue( + "ConfigSchemaDigestMismatch", + "configuration_schema_digest", + `recorded ${JSON.stringify(schemaDigest)} != recomputed ${JSON.stringify(recomputed)}`, + ), + ); + } + } catch (err) { + issues.push( + issue("ConfigSchemaUnhashable", "configuration_schema", String(err)), + ); + } + } + return issues; +} + +export function validateVerificationResultSemantics( + data: Record, +): SemanticIssue[] { + const issues = [ + ...forbidLegacySignature(data, "VerificationResult.v1"), + ...requireIntegrity(data, "VerificationResult.v1"), + ]; + const commit = data.source_commit; + if (typeof commit !== "string" || !isFullHexCommit(commit)) { + issues.push( + issue( + "InvalidSourceCommit", + "source_commit", + "source_commit must be a full 40-char lowercase hex SHA", + ), + ); + } + const decision = data.decision; + const executionStatus = data.execution_status; + if ( + typeof executionStatus === "string" && + FAIL_CLOSED.has(executionStatus) && + (decision === "accept" || decision === "reject") + ) { + issues.push( + issue( + "FailClosedDecision", + "decision", + `execution_status ${JSON.stringify(executionStatus)} cannot yield accept/reject`, + ), + ); + } + if (data.normalization_applied === true) { + const raw = data.raw_backend_output_digest; + const normalized = data.normalized_result_digest; + if (typeof raw !== "string" || typeof normalized !== "string") { + issues.push( + issue( + "MissingNormalizationDigests", + "normalized_result_digest", + "normalization_applied requires raw and normalized digests", + ), + ); + } else if (raw === normalized) { + issues.push( + issue( + "IdenticalNormalizationDigests", + "normalized_result_digest", + "raw and normalized digests must be distinct when normalization occurs", + ), + ); + } + } + let mandatoryFailed = false; + const groups = Array.isArray(data.check_groups) ? data.check_groups : []; + groups.forEach((group, gIndex) => { + if (!group || typeof group !== "object" || Array.isArray(group)) return; + const checks = Array.isArray((group as Record).checks) + ? ((group as Record).checks as unknown[]) + : []; + checks.forEach((check, cIndex) => { + if (!check || typeof check !== "object" || Array.isArray(check)) return; + const c = check as Record; + if (c.mandatory === true && c.status === "failed") mandatoryFailed = true; + if (c.status === "skipped" && !(c.reason_code || c.skip_reason_code)) { + issues.push( + issue( + "MissingSkipReason", + `check_groups[${gIndex}].checks[${cIndex}].reason_code`, + "skipped checks require reason_code", + ), + ); + } + }); + }); + if (decision === "accept" && mandatoryFailed) { + issues.push( + issue( + "AcceptWithMandatoryFailure", + "decision", + "accept cannot coexist with a mandatory failed check", + ), + ); + } + const declared = data.declared_input_guarantee_class; + const resultClass = data.guarantee_class; + if (typeof declared === "string" && typeof resultClass === "string") { + const dRank = GUARANTEE_RANK[declared]; + const rRank = GUARANTEE_RANK[resultClass]; + if (dRank !== undefined && rRank !== undefined && rRank > dRank) { + issues.push( + issue( + "GuaranteeUpgrade", + "guarantee_class", + `must not upgrade declared_input_guarantee_class (${JSON.stringify(declared)} -> ${JSON.stringify(resultClass)})`, + ), + ); + } + } + return issues; +} + +export function validateRewardEnvelopeSemantics( + data: Record, +): SemanticIssue[] { + const issues = [ + ...forbidLegacySignature(data, "RewardEvidenceEnvelope.v1"), + ...requireIntegrity(data, "RewardEvidenceEnvelope.v1"), + ...checkZeroCommit(data.source_commit, "source_commit"), + ]; + const totalResult = parseDecimal(data.scalar_total, "scalar_total"); + let total: Decimal | null = null; + if ("err" in totalResult) { + issues.push(totalResult.err); + } else { + total = totalResult.ok; + } + const components = data.components; + if (Array.isArray(components) && total && data.composition_function === "sum") { + let acc: Decimal = { mantissa: 0n, scale: 0 }; + let ok = true; + components.forEach((component, index) => { + if (!component || typeof component !== "object" || Array.isArray(component)) return; + const value = (component as Record).value; + if (value === undefined) return; + const parsed = parseDecimal(value, `components[${index}].value`); + if ("err" in parsed) { + issues.push(parsed.err); + ok = false; + return; + } + acc = addDecimals(acc, parsed.ok); + }); + if (ok && !decimalsEqual(acc, total)) { + issues.push( + issue( + "RewardCompositionMismatch", + "scalar_total", + "sum of components != scalar_total", + ), + ); + } + } + const claims = Array.isArray(data.claims_issued) ? data.claims_issued : []; + if (claims.length > 0) { + const refs = data.verifier_result_refs; + if (!Array.isArray(refs) || refs.length === 0) { + issues.push( + issue( + "ClaimsNeedVerifierRefs", + "verifier_result_refs", + "claims_issued requires at least one verifier_result_ref", + ), + ); + } + } + const prefs = Array.isArray(data.profile_refs) ? data.profile_refs : []; + const lifecycle = + data.lifecycle && typeof data.lifecycle === "object" && !Array.isArray(data.lifecycle) + ? (data.lifecycle as Record) + : {}; + const lifecycleStatus = lifecycle.status; + const migration = + typeof lifecycle.migration_record_id === "string" && + lifecycle.migration_record_id.length > 0; + prefs.forEach((pref, index) => { + if (!pref || typeof pref !== "object" || Array.isArray(pref)) return; + const status = (pref as Record).applicability_status; + if ( + (status === "revoked" || status === "expired") && + lifecycleStatus === "active" && + !migration + ) { + issues.push( + issue( + "RevokedProfileGate", + `profile_refs[${index}]`, + `${status} profiles cannot support new active rewards without migration_record_id`, + ), + ); + } else if (status === "revoked" && lifecycleStatus !== "active" && claims.length > 0) { + issues.push( + issue( + "RevokedProfileGate", + `profile_refs[${index}]`, + "revoked profiles cannot authorize reward claims", + ), + ); + } + }); + const mandatory = Array.isArray(data.mandatory_unresolved_claim_ids) + ? data.mandatory_unresolved_claim_ids + : []; + if (lifecycleStatus === "active" && mandatory.length > 0) { + issues.push( + issue( + "ActiveRewardUnresolvedClaims", + "mandatory_unresolved_claim_ids", + "unresolved mandatory claims block active release-grade envelopes", + ), + ); + } + return issues; +} + +export function validateCampaignManifestSemantics( + data: Record, +): SemanticIssue[] { + const issues = [ + ...forbidLegacySignature(data, "OptimizationCampaignManifest.v1"), + ...requireIntegrity(data, "OptimizationCampaignManifest.v1"), + ...checkZeroCommit(data.source_commit, "source_commit"), + ]; + if (!data.access_class) { + issues.push(issue("AccessClassRequired", "access_class", "access_class is required")); + } + const cohorts = Array.isArray(data.cohorts) ? data.cohorts : []; + cohorts.forEach((cohort, index) => { + if (!cohort || typeof cohort !== "object" || Array.isArray(cohort)) return; + const exposure = (cohort as Record).compute_exposure; + if (!exposure || typeof exposure !== "object" || Array.isArray(exposure)) { + issues.push( + issue( + "CohortComputeExposure", + `cohorts[${index}].compute_exposure`, + "compute_exposure is required", + ), + ); + } + if (!(cohort as Record).access_class) { + issues.push( + issue( + "CohortAccessClass", + `cohorts[${index}].access_class`, + "every cohort must declare access_class", + ), + ); + } + }); + return issues; +} + +export function validateAdjudicationRecordSemantics( + data: Record, + releaseGrade = false, +): SemanticIssue[] { + const issues = [ + ...forbidLegacySignature(data, "AdjudicationRecord.v1"), + ...requireIntegrity(data, "AdjudicationRecord.v1"), + ...checkZeroCommit(data.source_commit, "source_commit"), + ]; + const protectedRationale = data.protected_rationale; + if ( + protectedRationale && + typeof protectedRationale === "object" && + !Array.isArray(protectedRationale) + ) { + if (!(protectedRationale as Record).commitment_digest) { + issues.push( + issue( + "RationaleCommitment", + "protected_rationale.commitment_digest", + "commitment_digest is required", + ), + ); + } + } + if (releaseGrade && data.independence_declared !== true) { + issues.push( + issue( + "IndependenceForReleaseGrade", + "independence_declared", + "release-grade adjudication requires independence_declared=true", + ), + ); + } + return issues; +} + +export function validateAssuranceReportSemantics( + data: Record, +): SemanticIssue[] { + const issues = [ + ...forbidLegacySignature(data, "VerifierAssuranceReport.v1"), + ...requireIntegrity(data, "VerifierAssuranceReport.v1"), + ...checkZeroCommit(data.source_commit, "source_commit"), + ]; + let claimsGap = false; + const metrics = data.metrics; + if (metrics && typeof metrics === "object" && !Array.isArray(metrics)) { + const m = metrics as Record; + const sample = m.sample_size; + const excluded = m.excluded_count; + const unadj = m.unadjudicated_count; + if ( + typeof sample === "number" && + typeof excluded === "number" && + typeof unadj === "number" && + excluded + unadj > sample && + sample > 0 + ) { + issues.push( + issue( + "AggregateCountReconcile", + "metrics", + "excluded_count + unadjudicated_count exceeds sample_size", + ), + ); + } + for (const key of [ + "false_accept_rate", + "false_reject_rate", + "abstention_rate", + "adjudication_coverage", + ]) { + const rate = m[key]; + if (rate && typeof rate === "object" && !Array.isArray(rate)) { + const ci = (rate as Record).confidence_interval; + const ciObj = + ci && typeof ci === "object" && !Array.isArray(ci) + ? (ci as Record) + : undefined; + const method = ciObj?.method; + if (typeof method !== "string" || !method) { + issues.push( + issue( + "CIMethodsDeclared", + `metrics.${key}.confidence_interval.method`, + "CI method must be declared", + ), + ); + } else if ( + !ciObj?.parameters || + typeof ciObj.parameters !== "object" || + Array.isArray(ciObj.parameters) + ) { + issues.push( + issue( + "CIParametersDeclared", + `metrics.${key}.confidence_interval.parameters`, + "CI parameters must be declared (no silent denominator invention)", + ), + ); + } + } + } + const gap = m.optimization_gap; + const gapNonzero = + typeof gap === "string" && gap !== "0" && gap !== "0.0" && gap !== "0.000000"; + const ordinary = m.ordinary_accept_rate; + const optimized = m.optimized_accept_rate; + const ordinaryDen = + ordinary && typeof ordinary === "object" && !Array.isArray(ordinary) + ? Number((ordinary as Record).denominator ?? 0) + : 0; + const optimizedDen = + optimized && typeof optimized === "object" && !Array.isArray(optimized) + ? Number((optimized as Record).denominator ?? 0) + : 0; + claimsGap = gapNonzero || (ordinaryDen > 0 && optimizedDen > 0); + + const excludedItems = data.excluded_items; + const unadjItems = data.unadjudicated_items; + if (typeof excluded === "number") { + if (Array.isArray(excludedItems)) { + if (excludedItems.length !== excluded) { + issues.push( + issue( + "ExcludedItemsVisible", + "excluded_items", + "excluded_count must equal len(excluded_items)", + ), + ); + } + } else if (excluded > 0) { + issues.push( + issue( + "ExcludedItemsVisible", + "excluded_items", + "excluded_count > 0 requires visible excluded_items", + ), + ); + } + } + if (typeof unadj === "number") { + if (Array.isArray(unadjItems)) { + if (unadjItems.length !== unadj) { + issues.push( + issue( + "UnadjudicatedItemsVisible", + "unadjudicated_items", + "unadjudicated_count must equal len(unadjudicated_items)", + ), + ); + } + } else if (unadj > 0) { + issues.push( + issue( + "UnadjudicatedItemsVisible", + "unadjudicated_items", + "unadjudicated_count > 0 requires visible unadjudicated_items", + ), + ); + } + } + } + const cohorts = Array.isArray(data.cohorts) ? data.cohorts : []; + let hasOrdinary = false; + let hasOptimized = false; + cohorts.forEach((cohort, index) => { + if (!cohort || typeof cohort !== "object" || Array.isArray(cohort)) return; + const c = cohort as Record; + if (c.cohort_kind === "ordinary") hasOrdinary = true; + if (c.cohort_kind === "optimized") hasOptimized = true; + if (!c.access_class) { + issues.push( + issue( + "CohortAccessClass", + `cohorts[${index}].access_class`, + "every cohort must declare access_class", + ), + ); + } + const exposure = c.compute_exposure; + if (!exposure || typeof exposure !== "object" || Array.isArray(exposure)) { + issues.push( + issue( + "CohortComputeExposure", + `cohorts[${index}].compute_exposure`, + "every cohort must declare compute_exposure", + ), + ); + } + const accept = c.accept_count; + const reject = c.reject_count; + const indeterminate = c.indeterminate_count; + const included = c.included_result_count; + if ( + typeof accept === "number" && + typeof reject === "number" && + typeof indeterminate === "number" && + typeof included === "number" + ) { + if (accept + reject + indeterminate !== included) { + issues.push( + issue( + "CohortCountMismatch", + `cohorts[${index}]`, + "aggregate counts must reconcile exactly with included records", + ), + ); + } + if (accept < 0 || reject < 0 || indeterminate < 0) { + issues.push( + issue( + "IndeterminateMisclassification", + `cohorts[${index}]`, + "cohort decision counts must be non-negative distinct buckets", + ), + ); + } + } + }); + if (claimsGap && !(hasOrdinary && hasOptimized)) { + issues.push( + issue( + "OptimizationGapCohorts", + "cohorts", + "optimization-gap metrics require ordinary and optimized cohorts", + ), + ); + } + if (data.release_grade === true && data.independent_adjudication !== true) { + issues.push( + issue( + "ReleaseGradeAdjudication", + "independent_adjudication", + "release-grade reports require independent_adjudication=true", + ), + ); + } + return issues; +} + +export function validateVaSemantics( + data: Record, + artifactType: string, +): SemanticIssue[] { + switch (artifactType) { + case "VerifierProfile.v1": + return validateVerifierProfileSemantics(data); + case "VerificationResult.v1": + return validateVerificationResultSemantics(data); + case "RewardEvidenceEnvelope.v1": + return validateRewardEnvelopeSemantics(data); + case "OptimizationCampaignManifest.v1": + return validateCampaignManifestSemantics(data); + case "AdjudicationRecord.v1": + return validateAdjudicationRecordSemantics(data, false); + case "VerifierAssuranceReport.v1": + return validateAssuranceReportSemantics(data); + default: + return [ + issue( + "UnknownArtifactType", + "artifact_type", + `unknown verifier-assurance artifact type: ${artifactType}`, + ), + ]; + } +} + +export function validateVaSemanticsStrings( + data: Record, + artifactType: string, +): string[] { + return validateVaSemantics(data, artifactType).map(formatSemanticIssue); +} + +/** Verify report: semantics + integrity digest match (Python verify_assurance_report). */ +export function verifyAssuranceReport(report: Record): SemanticIssue[] { + const issues = validateAssuranceReportSemantics(report); + const integrity = report.integrity; + if (integrity && typeof integrity === "object" && !Array.isArray(integrity)) { + const body: Record = {}; + for (const [k, v] of Object.entries(report)) { + if (k !== "integrity") body[k] = v; + } + const expected = canonicalHash(body); + const got = (integrity as Record).artifact_digest; + if (got !== expected) { + issues.push( + issue( + "ReportDigestMismatch", + "integrity.artifact_digest", + "artifact_digest does not match report body", + ), + ); + } + } + return issues; +} + +const REQUIRED_FIELDS: Record = { + "VerifierProfile.v1": [ + "schema_version", + "artifact_type", + "verifier_profile_id", + "created_at", + "producer", + "producer_version", + "source_repo", + "source_commit", + "implementation", + "configuration", + "mechanism", + "claim_surface", + "applicability", + "assumptions", + "known_blind_spots", + "integrity", + ], + "VerificationResult.v1": [ + "schema_version", + "artifact_type", + "verification_result_id", + "created_at", + "producer", + "producer_version", + "source_repo", + "source_commit", + "verifier_profile", + "claim_ids", + "raw_backend_output_digest", + "normalized_result_digest", + "normalization_applied", + "check_groups", + "resource_limits", + "execution_status", + "decision", + "integrity", + ], + "RewardEvidenceEnvelope.v1": [ + "schema_version", + "artifact_type", + "reward_envelope_id", + "created_at", + "producer", + "producer_version", + "source_repo", + "source_commit", + "scalar_total", + "components", + "composition_function", + "integrity", + ], + "OptimizationCampaignManifest.v1": [ + "schema_version", + "artifact_type", + "campaign_id", + "created_at", + "producer", + "producer_version", + "source_repo", + "source_commit", + "access_class", + "cohorts", + "integrity", + ], + "AdjudicationRecord.v1": [ + "schema_version", + "artifact_type", + "adjudication_id", + "created_at", + "producer", + "producer_version", + "source_repo", + "source_commit", + "subject", + "label", + "independence_declared", + "integrity", + ], + "VerifierAssuranceReport.v1": [ + "schema_version", + "artifact_type", + "report_id", + "created_at", + "producer", + "producer_version", + "source_repo", + "source_commit", + "campaign_ref", + "release_grade", + "independent_adjudication", + "metrics", + "cohorts", + "integrity", + ], +}; + +/** + * Constructor requiring mandatory fields. Unknown fields are retained (no silent drop) + * so schema validation can reject additionalProperties. + */ +export function constructVaArtifact( + artifactType: string, + fields: Record, +): Record { + const required = REQUIRED_FIELDS[artifactType]; + if (!required) { + throw new Error(`unsupported VA constructor type: ${artifactType}`); + } + for (const key of required) { + if (!(key in fields)) { + throw new Error(`${artifactType}: missing mandatory field ${key}`); + } + } + if (fields.artifact_type !== artifactType) { + throw new Error(`${artifactType}: artifact_type must be ${JSON.stringify(artifactType)}`); + } + return { ...fields }; +} + +export function attachNestedIntegrity( + data: Record, +): Record { + const body: Record = {}; + for (const [k, v] of Object.entries(data)) { + if (k !== "integrity") body[k] = v; + } + const digest = canonicalHash(body); + return { + ...body, + integrity: { + canonicalization_version: "v1", + artifact_digest: digest, + }, + }; +}