From c686d536c648b372ac289801ba8c847849740318 Mon Sep 17 00:00:00 2001 From: Jerry Xiao Date: Fri, 24 Jul 2026 06:10:51 +0000 Subject: [PATCH 01/16] feat(experiment-runner): deterministic trial engine with immutable evaluator and journal replay Add the experiment engine phase: typed experiment contracts, a pure immutable evaluator, a deterministic measurement model, and a trial runner that measures baseline and candidate, gates them through the evaluator, and journals each trial as a self-describing record that replays and re-evaluates from the journal alone. - contracts: experiment spec, decision bounds, metric sample/summary, and verdict wire types, with schemas/{experiment,verdict}.schema.json kept in sync (contract tests enforce field and enum parity). - control-plane: additive read/journal APIs (snapshot, record_trial, read_trial, trial_ids) over a new experiment_trials table; no change to the existing journal schema or lifecycle. - experiment-runner: immutable evaluator, deterministic model, trial runner, and replay; integration tests for a promoted trial, a rejected-and-preserved trial, journal-only replay reproducing the verdict, and the MVP acceptance criterion. Safe-alpha limitation: mock capabilities are leased and the broker lifecycle always rolls back, so the verdict gates whether the candidate is applied at all rather than whether it persists; durable keep-or-rollback awaits the privileged broker. --- Cargo.lock | 9 + apps/experiment-runner/Cargo.toml | 11 + apps/experiment-runner/src/evaluator.rs | 215 +++++++++++++++ apps/experiment-runner/src/lib.rs | 22 ++ apps/experiment-runner/src/main.rs | 65 ++++- apps/experiment-runner/src/model.rs | 114 ++++++++ apps/experiment-runner/src/runner.rs | 211 +++++++++++++++ apps/experiment-runner/tests/integration.rs | 154 +++++++++++ crates/contracts/src/lib.rs | 282 +++++++++++++++++++- crates/control-plane/src/lib.rs | 93 +++++++ docs/ARCHITECTURE.md | 6 +- schemas/experiment.schema.json | 53 ++++ schemas/verdict.schema.json | 44 +++ 13 files changed, 1273 insertions(+), 6 deletions(-) create mode 100644 apps/experiment-runner/src/evaluator.rs create mode 100644 apps/experiment-runner/src/lib.rs create mode 100644 apps/experiment-runner/src/model.rs create mode 100644 apps/experiment-runner/src/runner.rs create mode 100644 apps/experiment-runner/tests/integration.rs create mode 100644 schemas/experiment.schema.json create mode 100644 schemas/verdict.schema.json diff --git a/Cargo.lock b/Cargo.lock index d45f471..031e646 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -112,6 +112,15 @@ dependencies = [ [[package]] name = "fpsmaxxing-experiment-runner" version = "0.1.0" +dependencies = [ + "fpsmaxxing-contracts", + "fpsmaxxing-control-plane", + "fpsmaxxing-mock-provider", + "serde", + "serde_json", + "tempfile", + "thiserror", +] [[package]] name = "fpsmaxxing-gateway" diff --git a/apps/experiment-runner/Cargo.toml b/apps/experiment-runner/Cargo.toml index dc2b8ad..bf6500a 100644 --- a/apps/experiment-runner/Cargo.toml +++ b/apps/experiment-runner/Cargo.toml @@ -7,5 +7,16 @@ repository.workspace = true rust-version.workspace = true publish = false +[dependencies] +fpsmaxxing-contracts.workspace = true +fpsmaxxing-control-plane = { path = "../../crates/control-plane" } +fpsmaxxing-mock-provider = { path = "../../sidecars/mock-provider" } +serde.workspace = true +serde_json.workspace = true +thiserror.workspace = true + +[dev-dependencies] +tempfile = "3" + [lints] workspace = true diff --git a/apps/experiment-runner/src/evaluator.rs b/apps/experiment-runner/src/evaluator.rs new file mode 100644 index 0000000..c8736b4 --- /dev/null +++ b/apps/experiment-runner/src/evaluator.rs @@ -0,0 +1,215 @@ +//! The immutable experiment evaluator. +//! +//! [`evaluate`] is a pure function: given recorded baseline and candidate +//! samples plus fixed [`DecisionBounds`] it returns a deterministic +//! [`Verdict`]. It performs no I/O, reads no clock, holds no state, and never +//! mutates its inputs, so a trial can be re-evaluated from the journal alone +//! and always yield an identical verdict. +//! +//! # Decision rule (fixed and ordered) +//! +//! 1. **Minimum samples.** Both the baseline and candidate sets must hold at +//! least `bounds.min_samples` samples, otherwise the verdict is a +//! [`Reject`](Decision::Reject) with +//! [`InsufficientSamples`](VerdictReason::InsufficientSamples). +//! 2. **Constraint bounds.** The candidate's worst temperature, then worst +//! power draw, then total errors must each stay within the inclusive +//! ceilings. The first ceiling exceeded rejects, in that order. +//! 3. **Improvement threshold.** The candidate mean FPS must beat the baseline +//! mean FPS by at least `bounds.min_fps_improvement`. If it does, the +//! verdict is a [`Promote`](Decision::Promote); otherwise a +//! [`Reject`](Decision::Reject) with +//! [`InsufficientImprovement`](VerdictReason::InsufficientImprovement). + +use fpsmaxxing_contracts::{ + Decision, DecisionBounds, MetricSample, MetricSummary, Verdict, VerdictReason, +}; + +/// Applies the immutable decision rule to recorded samples. +/// +/// The returned [`Verdict`] carries the baseline and candidate aggregates the +/// decision used, so a journaled trial is self-describing and re-evaluation can +/// be checked against it. +#[must_use] +pub fn evaluate( + baseline: &[MetricSample], + candidate: &[MetricSample], + bounds: &DecisionBounds, +) -> Verdict { + let baseline_summary = summarize(baseline); + let candidate_summary = summarize(candidate); + let fps_improvement = candidate_summary.mean_fps - baseline_summary.mean_fps; + let min_samples = u64::from(bounds.min_samples.get()); + + let reason = + if baseline_summary.samples < min_samples || candidate_summary.samples < min_samples { + VerdictReason::InsufficientSamples + } else if candidate_summary.max_temperature_c > bounds.max_temperature_c { + VerdictReason::TemperatureExceeded + } else if candidate_summary.max_power_w > bounds.max_power_w { + VerdictReason::PowerExceeded + } else if candidate_summary.total_errors > bounds.max_errors { + VerdictReason::ErrorsExceeded + } else if fps_improvement >= bounds.min_fps_improvement { + VerdictReason::Promoted + } else { + VerdictReason::InsufficientImprovement + }; + + let decision = if matches!(reason, VerdictReason::Promoted) { + Decision::Promote + } else { + Decision::Reject + }; + + Verdict { + decision, + reason, + fps_improvement, + baseline: baseline_summary, + candidate: candidate_summary, + } +} + +/// Aggregates one measurement set deterministically. +/// +/// The FPS mean is the arithmetic mean (zero for an empty set), while the +/// temperature and power fields report the worst (highest) observed value and +/// errors are summed. Counting the divisor as an `f64` avoids a lossy +/// integer-to-float cast without changing the result for realistic sample +/// counts. +fn summarize(samples: &[MetricSample]) -> MetricSummary { + let mut sum_fps = 0.0_f64; + let mut divisor = 0.0_f64; + let mut max_temperature_c = 0.0_f64; + let mut max_power_w = 0.0_f64; + let mut total_errors = 0_u64; + for sample in samples { + sum_fps += sample.fps; + divisor += 1.0; + max_temperature_c = max_temperature_c.max(sample.temperature_c); + max_power_w = max_power_w.max(sample.power_w); + total_errors += sample.errors; + } + MetricSummary { + samples: samples.len() as u64, + mean_fps: if samples.is_empty() { + 0.0 + } else { + sum_fps / divisor + }, + max_temperature_c, + max_power_w, + total_errors, + } +} + +#[cfg(test)] +mod tests { + use std::num::NonZeroU32; + + use super::{Decision, DecisionBounds, MetricSample, VerdictReason, evaluate}; + + fn bounds() -> DecisionBounds { + DecisionBounds { + min_samples: NonZeroU32::new(3).expect("min samples is non-zero"), + min_fps_improvement: 5.0, + max_temperature_c: 80.0, + max_power_w: 200.0, + max_errors: 0, + } + } + + fn samples( + fps: f64, + temperature_c: f64, + power_w: f64, + errors: u64, + count: usize, + ) -> Vec { + (0..count) + .map(|_| MetricSample { + fps, + temperature_c, + power_w, + errors, + }) + .collect() + } + + #[test] + fn promotes_when_every_bound_is_met() { + let baseline = samples(100.0, 60.0, 150.0, 0, 5); + let candidate = samples(120.0, 70.0, 180.0, 0, 5); + let verdict = evaluate(&baseline, &candidate, &bounds()); + assert_eq!(verdict.decision, Decision::Promote); + assert_eq!(verdict.reason, VerdictReason::Promoted); + assert!((verdict.fps_improvement - 20.0).abs() < f64::EPSILON); + assert_eq!(verdict.baseline.samples, 5); + assert!((verdict.candidate.mean_fps - 120.0).abs() < f64::EPSILON); + } + + #[test] + fn rejects_when_a_set_is_below_minimum_samples() { + let baseline = samples(100.0, 60.0, 150.0, 0, 5); + let candidate = samples(120.0, 70.0, 180.0, 0, 2); + let verdict = evaluate(&baseline, &candidate, &bounds()); + assert_eq!(verdict.decision, Decision::Reject); + assert_eq!(verdict.reason, VerdictReason::InsufficientSamples); + } + + #[test] + fn temperature_ceiling_is_checked_before_improvement() { + let baseline = samples(100.0, 60.0, 150.0, 0, 5); + let candidate = samples(140.0, 80.1, 180.0, 0, 5); + let verdict = evaluate(&baseline, &candidate, &bounds()); + assert_eq!(verdict.decision, Decision::Reject); + assert_eq!(verdict.reason, VerdictReason::TemperatureExceeded); + } + + #[test] + fn power_ceiling_rejects_even_with_a_large_gain() { + let baseline = samples(100.0, 60.0, 150.0, 0, 5); + let candidate = samples(140.0, 70.0, 200.1, 0, 5); + let verdict = evaluate(&baseline, &candidate, &bounds()); + assert_eq!(verdict.decision, Decision::Reject); + assert_eq!(verdict.reason, VerdictReason::PowerExceeded); + } + + #[test] + fn errors_ceiling_rejects_a_regressing_candidate() { + let baseline = samples(100.0, 60.0, 150.0, 0, 5); + let candidate = samples(140.0, 70.0, 180.0, 1, 5); + let verdict = evaluate(&baseline, &candidate, &bounds()); + assert_eq!(verdict.decision, Decision::Reject); + assert_eq!(verdict.reason, VerdictReason::ErrorsExceeded); + } + + #[test] + fn rejects_when_improvement_is_below_threshold() { + let baseline = samples(100.0, 60.0, 150.0, 0, 5); + let candidate = samples(104.0, 70.0, 180.0, 0, 5); + let verdict = evaluate(&baseline, &candidate, &bounds()); + assert_eq!(verdict.decision, Decision::Reject); + assert_eq!(verdict.reason, VerdictReason::InsufficientImprovement); + assert!((verdict.fps_improvement - 4.0).abs() < f64::EPSILON); + } + + #[test] + fn improvement_exactly_at_threshold_promotes() { + let baseline = samples(100.0, 60.0, 150.0, 0, 3); + let candidate = samples(105.0, 70.0, 180.0, 0, 3); + let verdict = evaluate(&baseline, &candidate, &bounds()); + assert_eq!(verdict.decision, Decision::Promote); + assert_eq!(verdict.reason, VerdictReason::Promoted); + } + + #[test] + fn evaluation_is_deterministic_across_repeated_calls() { + let baseline = samples(100.0, 60.0, 150.0, 0, 4); + let candidate = samples(118.0, 72.0, 190.0, 0, 4); + let first = evaluate(&baseline, &candidate, &bounds()); + let second = evaluate(&baseline, &candidate, &bounds()); + assert_eq!(first, second); + } +} diff --git a/apps/experiment-runner/src/lib.rs b/apps/experiment-runner/src/lib.rs new file mode 100644 index 0000000..4fde3de --- /dev/null +++ b/apps/experiment-runner/src/lib.rs @@ -0,0 +1,22 @@ +//! Deterministic experiment engine. +//! +//! This crate provides the pieces that turn a typed [`ExperimentSpec`] into an +//! auditable, replayable trial: +//! +//! - a deterministic measurement model that stands in for a live telemetry +//! source on the Linux safe-alpha path, +//! - the immutable [`evaluate`] decision rule, and +//! - a trial runner that journals every sample and verdict so a trial can be +//! replayed and re-evaluated from the journal alone. +//! +//! [`ExperimentSpec`]: fpsmaxxing_contracts::ExperimentSpec + +mod evaluator; +mod model; +mod runner; + +pub use evaluator::evaluate; +pub use model::measure; +pub use runner::{ + LifecycleOutcome, ReplayOutcome, RunnerError, StoredTrial, TrialRecord, replay_trial, run_trial, +}; diff --git a/apps/experiment-runner/src/main.rs b/apps/experiment-runner/src/main.rs index 1296f77..05da6c8 100644 --- a/apps/experiment-runner/src/main.rs +++ b/apps/experiment-runner/src/main.rs @@ -1,5 +1,64 @@ -//! Deterministic performance experiment runner. +//! Demonstration entry point for the deterministic experiment engine. +//! +//! Runs one measured experiment against the mock provider through the broker, +//! journals it, then replays it from the durable journal alone and confirms the +//! re-evaluated verdict matches the one recorded at run time. The journal is an +//! in-memory `SQLite` database, so the demo leaves nothing on disk. -fn main() { - println!("fpsmaxxing-experiment-runner: scaffold ready; no benchmark configured"); +use std::num::{NonZeroU32, NonZeroU64}; + +use fpsmaxxing_contracts::{ChangeRequest, DecisionBounds, ExperimentSpec}; +use fpsmaxxing_control_plane::ControlPlane; +use fpsmaxxing_experiment_runner::{RunnerError, replay_trial, run_trial}; +use fpsmaxxing_mock_provider::MockProvider; +use serde_json::json; + +fn main() -> Result<(), RunnerError> { + let mut plane = ControlPlane::open(Box::new(MockProvider::new(10)), ":memory:")?; + let spec = demo_spec(); + + let trial = run_trial(&mut plane, &spec)?; + let verdict = &trial.record.verdict; + println!( + "trial {} -> {:?} ({:?}); fps_improvement = {:.1}", + trial.id, verdict.decision, verdict.reason, verdict.fps_improvement + ); + if let Some(lifecycle) = &trial.record.lifecycle { + println!( + " lifecycle: provider {}, verified = {}, rolled_back = {}", + lifecycle.provider_id, lifecycle.verified, lifecycle.rolled_back + ); + } + + let replay = replay_trial(&plane, trial.id)?; + println!( + "replay {} -> recomputed {:?}; consistent with journal = {}", + replay.trial_id, + replay.recomputed.decision, + replay.is_consistent() + ); + Ok(()) +} + +/// Builds a spec that raises the mock knob within the safety envelope. +fn demo_spec() -> ExperimentSpec { + ExperimentSpec { + hypothesis: "raising mock.value from 10 to 40 improves FPS within thermal and power limits" + .to_owned(), + target: ChangeRequest { + capability_id: "mock.value".to_owned(), + parameters: json!({ "value": 40 }), + lease_seconds: NonZeroU64::new(30).expect("lease is non-zero"), + }, + warmup_samples: 2, + baseline_samples: NonZeroU32::new(5).expect("baseline count is non-zero"), + candidate_samples: NonZeroU32::new(5).expect("candidate count is non-zero"), + bounds: DecisionBounds { + min_samples: NonZeroU32::new(3).expect("min samples is non-zero"), + min_fps_improvement: 5.0, + max_temperature_c: 80.0, + max_power_w: 200.0, + max_errors: 0, + }, + } } diff --git a/apps/experiment-runner/src/model.rs b/apps/experiment-runner/src/model.rs new file mode 100644 index 0000000..c13303a --- /dev/null +++ b/apps/experiment-runner/src/model.rs @@ -0,0 +1,114 @@ +//! Deterministic measurement model. +//! +//! On the Linux safe-alpha path there is no `PresentMon` or hardware telemetry to +//! sample, so [`measure`] stands in for a live source. It is a pure function of +//! the knob value under test: the same value always yields the same samples, +//! which is what lets a trial be replayed and re-evaluated from the journal +//! alone. +//! +//! The stand-in models three coupled effects of the mock knob (`0..=100`): +//! frames per second and both temperature and power draw rise with the value, +//! and faults appear only once the value climbs past a safe onset. A leading +//! run of `warmup` samples is generated at a cold-start FPS and then discarded, +//! so the counted samples reflect steady state. + +use fpsmaxxing_contracts::MetricSample; + +/// Frames per second reported while warming up, before steady state. +const COLD_FPS: f64 = 60.0; +/// Steady-state frames per second at the lowest knob value. +const BASE_FPS: f64 = 60.0; +/// Steady-state frames-per-second gain per knob unit. +const FPS_GAIN_PER_UNIT: f64 = 1.0; +/// Component temperature in degrees Celsius at the lowest knob value. +const BASE_TEMPERATURE_C: f64 = 50.0; +/// Additional degrees Celsius per knob unit. +const TEMPERATURE_C_PER_UNIT: f64 = 0.5; +/// Board power draw in watts at the lowest knob value. +const BASE_POWER_W: f64 = 100.0; +/// Additional watts per knob unit. +const POWER_W_PER_UNIT: f64 = 1.0; +/// Knob value at or below which no correctness faults are modeled. +const ERROR_ONSET: u64 = 90; + +/// Produces `counted` steady-state samples for a knob value. +/// +/// `warmup` leading samples are generated at a cold-start FPS and dropped, so +/// the returned vector always holds exactly `counted` steady-state samples. The +/// output depends only on `value`, `warmup`, and `counted`, never on wall-clock +/// time or external state. +#[must_use] +pub fn measure(value: u64, warmup: u32, counted: u32) -> Vec { + let setting = knob_to_setting(value); + let total = warmup.saturating_add(counted); + let mut samples: Vec = (0..total) + .map(|index| sample_at(setting, value, index < warmup)) + .collect(); + samples.split_off(warmup as usize) +} + +/// Builds one deterministic sample at a given setting. +fn sample_at(setting: f64, value: u64, warming: bool) -> MetricSample { + MetricSample { + fps: if warming { + COLD_FPS + } else { + BASE_FPS + FPS_GAIN_PER_UNIT * setting + }, + temperature_c: BASE_TEMPERATURE_C + TEMPERATURE_C_PER_UNIT * setting, + power_w: BASE_POWER_W + POWER_W_PER_UNIT * setting, + errors: value.saturating_sub(ERROR_ONSET), + } +} + +/// Converts a bounded knob value into a float setting without a lossy cast. +/// +/// Mock values are policy-bounded to `0..=100`, so the `u32` conversion never +/// saturates in practice; the fallback keeps the function total. +fn knob_to_setting(value: u64) -> f64 { + f64::from(u32::try_from(value).unwrap_or(u32::MAX)) +} + +#[cfg(test)] +mod tests { + use super::{BASE_FPS, COLD_FPS, measure}; + + #[test] + fn returns_exactly_the_counted_samples() { + let samples = measure(40, 3, 5); + assert_eq!(samples.len(), 5); + } + + #[test] + fn warmup_samples_are_discarded() { + // Every returned sample must be steady state; a cold-start FPS leaking + // through would prove the warmup prefix was not dropped. + let samples = measure(40, 4, 6); + assert!(samples.iter().all(|sample| sample.fps > COLD_FPS)); + assert!( + samples + .iter() + .all(|sample| (sample.fps - (BASE_FPS + 40.0)).abs() < f64::EPSILON) + ); + } + + #[test] + fn higher_values_raise_fps_temperature_and_power() { + let low = &measure(10, 0, 1)[0]; + let high = &measure(60, 0, 1)[0]; + assert!(high.fps > low.fps); + assert!(high.temperature_c > low.temperature_c); + assert!(high.power_w > low.power_w); + } + + #[test] + fn faults_appear_only_past_the_safe_onset() { + assert_eq!(measure(90, 0, 1)[0].errors, 0); + assert_eq!(measure(95, 0, 1)[0].errors, 5); + } + + #[test] + fn measurement_is_deterministic() { + assert_eq!(measure(37, 2, 4), measure(37, 2, 4)); + } +} diff --git a/apps/experiment-runner/src/runner.rs b/apps/experiment-runner/src/runner.rs new file mode 100644 index 0000000..039412b --- /dev/null +++ b/apps/experiment-runner/src/runner.rs @@ -0,0 +1,211 @@ +//! The trial runner and its journal-only replay. +//! +//! [`run_trial`] measures a baseline, measures the candidate, applies the +//! immutable evaluator, and - only when the verdict promotes - runs the +//! candidate through the broker lifecycle. Every trial is written to the +//! durable trial journal as a self-describing [`TrialRecord`], so +//! [`replay_trial`] can re-evaluate it from the journal alone and confirm the +//! recorded verdict without chat history or re-running the workload. +//! +//! # Keep-or-rollback in the safe alpha +//! +//! [`ControlPlane::run_lifecycle`] always restores the pre-state before it +//! returns (every mock capability is leased), so physical persistence cannot be +//! driven by the verdict on this path. The verdict instead gates whether the +//! candidate is applied at all: a [`Promote`](Decision::Promote) exercises the +//! full snapshot/preview/apply/verify/rollback lifecycle, while a +//! [`Reject`](Decision::Reject) never mutates the knob, leaving the baseline +//! untouched. The recorded [`LifecycleOutcome`] captures what happened. + +use fpsmaxxing_contracts::{Decision, ExperimentSpec, MetricSample, Verdict}; +use fpsmaxxing_control_plane::{ControlPlane, ControlPlaneError, LifecycleResult}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use thiserror::Error; + +use crate::{evaluate, model}; + +/// Fail-closed errors raised while running or replaying a trial. +#[derive(Debug, Error)] +pub enum RunnerError { + /// The broker or the durable journal rejected an operation. + #[error(transparent)] + ControlPlane(#[from] ControlPlaneError), + /// The experiment target did not carry an unsigned mock value. + #[error("experiment target is missing an unsigned mock value")] + InvalidTarget, + /// The provider snapshot did not carry an unsigned mock value. + #[error("provider snapshot is missing an unsigned mock value")] + InvalidBaseline, + /// A journaled trial record could not be decoded for replay. + #[error(transparent)] + Decode(#[from] serde_json::Error), +} + +/// A durable, `Deserialize`-able mirror of a broker [`LifecycleResult`]. +/// +/// [`LifecycleResult`] is serialize-only; this record round-trips so a promoted +/// trial's lifecycle outcome can be read back during replay and audit. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +pub struct LifecycleOutcome { + /// Provider that owned the change. + pub provider_id: String, + /// Human-readable preview produced before the write. + pub preview: String, + /// Whether the requested value was observed after apply. + pub verified: bool, + /// Whether the captured baseline was restored before returning. + pub rolled_back: bool, +} + +impl From<&LifecycleResult> for LifecycleOutcome { + fn from(result: &LifecycleResult) -> Self { + Self { + provider_id: result.provider_id.clone(), + preview: result.preview.clone(), + verified: result.verified, + rolled_back: result.rolled_back, + } + } +} + +/// The complete, replayable record of one trial. +/// +/// It holds everything the immutable evaluator needs, so re-evaluation reads +/// only this record: the spec (for the decision bounds), the recorded baseline +/// and candidate samples, and the verdict the evaluator produced. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +pub struct TrialRecord { + /// The experiment specification that was run. + pub spec: ExperimentSpec, + /// The knob value measured as the baseline. + pub baseline_value: u64, + /// The knob value measured as the candidate. + pub candidate_value: u64, + /// The recorded baseline measurement set. + pub baseline_samples: Vec, + /// The recorded candidate measurement set. + pub candidate_samples: Vec, + /// The verdict the immutable evaluator produced. + pub verdict: Verdict, + /// The broker lifecycle outcome, present only when the trial promoted. + pub lifecycle: Option, +} + +/// A [`TrialRecord`] paired with the journal identifier it was stored under. +#[derive(Clone, Debug, PartialEq)] +pub struct StoredTrial { + /// The durable trial-journal identifier. + pub id: i64, + /// The record that was journaled. + pub record: TrialRecord, +} + +/// The result of re-evaluating a journaled trial. +#[derive(Clone, Debug)] +pub struct ReplayOutcome { + /// The trial-journal identifier that was replayed. + pub trial_id: i64, + /// The verdict read back from the journal. + pub recorded: Verdict, + /// The verdict recomputed from the journaled samples and bounds. + pub recomputed: Verdict, +} + +impl ReplayOutcome { + /// Whether the recomputed verdict matches the one recorded at run time. + #[must_use] + pub fn is_consistent(&self) -> bool { + self.recorded == self.recomputed + } +} + +/// Runs one trial end to end and journals a replayable record. +/// +/// Measures the baseline from the provider's current state, measures the +/// candidate from the spec target, evaluates the two, and - only on a +/// [`Promote`](Decision::Promote) - runs the candidate through the broker +/// lifecycle. The trial is written to the durable trial journal before the +/// stored record is returned. +/// +/// # Errors +/// +/// Returns an error if the spec target or provider snapshot lacks an unsigned +/// mock value, or if the broker or durable journal rejects an operation. +pub fn run_trial( + plane: &mut ControlPlane, + spec: &ExperimentSpec, +) -> Result { + let baseline_value = baseline_value(plane)?; + let candidate_value = candidate_value(spec)?; + let baseline_samples = model::measure( + baseline_value, + spec.warmup_samples, + spec.baseline_samples.get(), + ); + let candidate_samples = model::measure( + candidate_value, + spec.warmup_samples, + spec.candidate_samples.get(), + ); + let verdict = evaluate(&baseline_samples, &candidate_samples, &spec.bounds); + let lifecycle = match verdict.decision { + Decision::Promote => Some(LifecycleOutcome::from(&plane.run_lifecycle(&spec.target)?)), + Decision::Reject => None, + }; + let record = TrialRecord { + spec: spec.clone(), + baseline_value, + candidate_value, + baseline_samples, + candidate_samples, + verdict, + lifecycle, + }; + let id = plane.record_trial(&record)?; + Ok(StoredTrial { id, record }) +} + +/// Re-evaluates a journaled trial from the journal alone. +/// +/// Reads the [`TrialRecord`], recomputes the verdict from its recorded samples +/// and bounds with the same immutable evaluator, and returns both the recorded +/// and recomputed verdicts for comparison. It consults no chat history and +/// re-runs no workload. +/// +/// # Errors +/// +/// Returns an error if the trial cannot be read from the durable journal or its +/// record cannot be decoded. +pub fn replay_trial(plane: &ControlPlane, id: i64) -> Result { + let record: TrialRecord = serde_json::from_value(plane.read_trial(id)?)?; + let recomputed = evaluate( + &record.baseline_samples, + &record.candidate_samples, + &record.spec.bounds, + ); + Ok(ReplayOutcome { + trial_id: id, + recorded: record.verdict, + recomputed, + }) +} + +/// Reads the baseline knob value from the provider's current state. +fn baseline_value(plane: &ControlPlane) -> Result { + plane + .snapshot()? + .state + .get("value") + .and_then(Value::as_u64) + .ok_or(RunnerError::InvalidBaseline) +} + +/// Reads the candidate knob value from the spec target parameters. +fn candidate_value(spec: &ExperimentSpec) -> Result { + spec.target + .parameters + .get("value") + .and_then(Value::as_u64) + .ok_or(RunnerError::InvalidTarget) +} diff --git a/apps/experiment-runner/tests/integration.rs b/apps/experiment-runner/tests/integration.rs new file mode 100644 index 0000000..ba4f977 --- /dev/null +++ b/apps/experiment-runner/tests/integration.rs @@ -0,0 +1,154 @@ +//! Integration tests for the deterministic experiment engine. +//! +//! These exercise the trial runner and journal-only replay end to end against +//! the mock provider through the broker: a promoted experiment that runs the +//! full lifecycle, a rejected experiment that is never applied and leaves the +//! baseline untouched, and a replay from the durable journal alone - reopened +//! as a fresh handle - that reproduces the recorded verdict exactly. The final +//! test states the MVP acceptance criterion directly. + +use std::num::{NonZeroU32, NonZeroU64}; + +use fpsmaxxing_contracts::{ + ChangeRequest, Decision, DecisionBounds, ExperimentSpec, VerdictReason, +}; +use fpsmaxxing_control_plane::ControlPlane; +use fpsmaxxing_experiment_runner::{evaluate, replay_trial, run_trial}; +use fpsmaxxing_mock_provider::MockProvider; +use serde_json::json; +use tempfile::NamedTempFile; + +/// Builds a spec that drives the mock knob to `candidate` under the given +/// improvement threshold and temperature ceiling. +fn spec_for(candidate: u64, min_fps_improvement: f64, max_temperature_c: f64) -> ExperimentSpec { + ExperimentSpec { + hypothesis: format!("raise mock.value to {candidate}"), + target: ChangeRequest { + capability_id: "mock.value".to_owned(), + parameters: json!({ "value": candidate }), + lease_seconds: NonZeroU64::new(30).expect("lease is non-zero"), + }, + warmup_samples: 2, + baseline_samples: NonZeroU32::new(5).expect("baseline count is non-zero"), + candidate_samples: NonZeroU32::new(5).expect("candidate count is non-zero"), + bounds: DecisionBounds { + min_samples: NonZeroU32::new(3).expect("min samples is non-zero"), + min_fps_improvement, + max_temperature_c, + max_power_w: 200.0, + max_errors: 0, + }, + } +} + +/// Reads the mock provider's current knob value through a broker snapshot. +fn current_value(plane: &ControlPlane) -> u64 { + plane + .snapshot() + .expect("snapshot") + .state + .get("value") + .and_then(serde_json::Value::as_u64) + .expect("mock value") +} + +#[test] +fn a_promoted_experiment_runs_the_full_lifecycle() { + let journal = NamedTempFile::new().expect("temp journal"); + let mut plane = + ControlPlane::open(Box::new(MockProvider::new(10)), journal.path()).expect("open"); + + let trial = run_trial(&mut plane, &spec_for(40, 5.0, 80.0)).expect("run trial"); + + assert_eq!(trial.record.verdict.decision, Decision::Promote); + assert_eq!(trial.record.verdict.reason, VerdictReason::Promoted); + + let lifecycle = trial + .record + .lifecycle + .expect("a promoted trial records a lifecycle"); + assert_eq!(lifecycle.provider_id, "mock"); + assert!( + lifecycle.verified, + "candidate value must verify after apply" + ); + assert!(lifecycle.rolled_back, "leased change must be rolled back"); + + // The leased lifecycle restores the pre-state, so the provider is left at + // its baseline value even after a promotion. + assert_eq!(current_value(&plane), 10); +} + +#[test] +fn a_rejected_experiment_is_never_applied_and_leaves_the_baseline() { + let journal = NamedTempFile::new().expect("temp journal"); + let mut plane = + ControlPlane::open(Box::new(MockProvider::new(10)), journal.path()).expect("open"); + + // Candidate 70 would raise FPS but drives modeled temperature to 85 C, + // above the 80 C ceiling, so the evaluator rejects it on safety. + let trial = run_trial(&mut plane, &spec_for(70, 5.0, 80.0)).expect("run trial"); + + assert_eq!(trial.record.verdict.decision, Decision::Reject); + assert_eq!( + trial.record.verdict.reason, + VerdictReason::TemperatureExceeded + ); + assert!( + trial.record.lifecycle.is_none(), + "a rejected candidate is never applied" + ); + + // Nothing was applied, so the provider still holds the baseline value. + assert_eq!(current_value(&plane), 10); +} + +#[test] +fn a_trial_replays_from_the_journal_alone_with_an_identical_verdict() { + let journal = NamedTempFile::new().expect("temp journal"); + + let trial_id = { + let mut plane = + ControlPlane::open(Box::new(MockProvider::new(10)), journal.path()).expect("open"); + run_trial(&mut plane, &spec_for(40, 5.0, 80.0)) + .expect("run trial") + .id + }; + + // Reopen the journal as a brand new handle - no in-memory trial state, no + // chat history - and with a provider at an unrelated value to prove replay + // reads only what was persisted. + let replayed = + ControlPlane::open(Box::new(MockProvider::new(0)), journal.path()).expect("reopen"); + let outcome = replay_trial(&replayed, trial_id).expect("replay trial"); + + assert!( + outcome.is_consistent(), + "recomputed verdict must match the journal" + ); + assert_eq!(outcome.recorded, outcome.recomputed); + assert_eq!(outcome.recomputed.decision, Decision::Promote); +} + +#[test] +fn mvp_one_measured_experiment_is_promoted_or_rejected_by_the_evaluator() { + let journal = NamedTempFile::new().expect("temp journal"); + let mut plane = + ControlPlane::open(Box::new(MockProvider::new(10)), journal.path()).expect("open"); + + let trial = run_trial(&mut plane, &spec_for(40, 5.0, 80.0)).expect("run trial"); + + // The acceptance criterion: a measured experiment reaches a promote/reject + // decision, and that decision is the immutable evaluator's - recomputing it + // from the recorded samples and bounds reproduces the journaled verdict. + assert!(matches!( + trial.record.verdict.decision, + Decision::Promote | Decision::Reject + )); + let recomputed = evaluate( + &trial.record.baseline_samples, + &trial.record.candidate_samples, + &trial.record.spec.bounds, + ); + assert_eq!(recomputed, trial.record.verdict); +} diff --git a/crates/contracts/src/lib.rs b/crates/contracts/src/lib.rs index eb3a9fb..c1265a5 100644 --- a/crates/contracts/src/lib.rs +++ b/crates/contracts/src/lib.rs @@ -87,6 +87,122 @@ pub struct StateSnapshot { pub state: Value, } +/// A single deterministic performance measurement captured during a trial. +/// +/// Values are recorded verbatim in the experiment journal so a trial can be +/// re-evaluated later without re-running the workload or consulting chat +/// history. +#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct MetricSample { + /// Frames per second observed for this sample; higher is better. + pub fps: f64, + /// Peak component temperature in degrees Celsius for this sample. + pub temperature_c: f64, + /// Board power draw in watts for this sample. + pub power_w: f64, + /// Correctness errors detected during this sample; zero is nominal. + pub errors: u64, +} + +/// Deterministic aggregate of one measurement set used by the evaluator. +#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct MetricSummary { + /// Number of samples aggregated. + pub samples: u64, + /// Mean frames per second across the samples. + pub mean_fps: f64, + /// Highest temperature observed across the samples. + pub max_temperature_c: f64, + /// Highest power draw observed across the samples. + pub max_power_w: f64, + /// Total correctness errors observed across the samples. + pub total_errors: u64, +} + +/// Fixed thresholds the immutable evaluator applies to a trial. +#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct DecisionBounds { + /// Minimum samples required in each of the baseline and candidate sets + /// before a promotion can be considered. + pub min_samples: NonZeroU32, + /// Minimum mean-FPS gain the candidate must show over the baseline. + pub min_fps_improvement: f64, + /// Inclusive ceiling for candidate temperature in degrees Celsius. + pub max_temperature_c: f64, + /// Inclusive ceiling for candidate power draw in watts. + pub max_power_w: f64, + /// Inclusive ceiling for candidate correctness errors. + pub max_errors: u64, +} + +/// A declarative, typed experiment the runner can execute, journal, and replay. +#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct ExperimentSpec { + /// Human-authored hypothesis the trial tests. + pub hypothesis: String, + /// Bounded, policy-checkable capability change under test. + pub target: ChangeRequest, + /// Leading measurements discarded from each phase before counting. + pub warmup_samples: u32, + /// Counted baseline measurements to record. + pub baseline_samples: NonZeroU32, + /// Counted candidate measurements to record. + pub candidate_samples: NonZeroU32, + /// Thresholds the evaluator uses to promote or reject. + pub bounds: DecisionBounds, +} + +/// The terminal keep-or-rollback decision for a trial. +#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "kebab-case")] +pub enum Decision { + /// The candidate met every bound and is kept. + Promote, + /// The candidate failed a bound and is rolled back. + Reject, +} + +/// The single machine-readable reason behind a [`Verdict`]. +#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "kebab-case")] +pub enum VerdictReason { + /// The candidate cleared every bound. + Promoted, + /// A measurement set held fewer than the required samples. + InsufficientSamples, + /// Candidate temperature exceeded the ceiling. + TemperatureExceeded, + /// Candidate power draw exceeded the ceiling. + PowerExceeded, + /// Candidate correctness errors exceeded the ceiling. + ErrorsExceeded, + /// The candidate did not beat the baseline by the required margin. + InsufficientImprovement, +} + +/// The immutable evaluator's deterministic decision for a trial. +/// +/// The verdict carries the aggregates the decision used so a journaled trial +/// is self-describing and re-evaluation can be checked against it. +#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct Verdict { + /// Whether the candidate is kept or rolled back. + pub decision: Decision, + /// The reason behind the decision. + pub reason: VerdictReason, + /// Candidate mean-FPS gain over the baseline. + pub fps_improvement: f64, + /// Baseline aggregate the decision used. + pub baseline: MetricSummary, + /// Candidate aggregate the decision used. + pub candidate: MetricSummary, +} + #[cfg(test)] mod tests { use std::collections::BTreeSet; @@ -94,11 +210,15 @@ mod tests { use serde_json::{Value, json}; use super::{ - CapabilityDescriptor, ChangeRequest, NonZeroU32, Persistence, ProviderManifest, RiskClass, + CapabilityDescriptor, ChangeRequest, Decision, DecisionBounds, ExperimentSpec, + MetricSummary, NonZeroU32, NonZeroU64, Persistence, ProviderManifest, RiskClass, Verdict, + VerdictReason, }; const CAPABILITY_SCHEMA: &str = include_str!("../../../schemas/capability.schema.json"); const SIDECAR_SCHEMA: &str = include_str!("../../../schemas/sidecar.schema.json"); + const EXPERIMENT_SCHEMA: &str = include_str!("../../../schemas/experiment.schema.json"); + const VERDICT_SCHEMA: &str = include_str!("../../../schemas/verdict.schema.json"); fn wire_string(value: impl serde::Serialize) -> String { serde_json::to_value(value) @@ -321,4 +441,164 @@ mod tests { serde_json::from_value(serialized).expect("manifest should deserialize"); assert_eq!(manifest, deserialized); } + + fn sample_spec() -> ExperimentSpec { + ExperimentSpec { + hypothesis: "Raising mock.value improves throughput".to_owned(), + target: ChangeRequest { + capability_id: "mock.value".to_owned(), + parameters: json!({ "value": 60 }), + lease_seconds: NonZeroU64::new(30).expect("lease is non-zero"), + }, + warmup_samples: 2, + baseline_samples: NonZeroU32::new(5).expect("count is non-zero"), + candidate_samples: NonZeroU32::new(5).expect("count is non-zero"), + bounds: DecisionBounds { + min_samples: NonZeroU32::new(3).expect("count is non-zero"), + min_fps_improvement: 5.0, + max_temperature_c: 85.0, + max_power_w: 200.0, + max_errors: 0, + }, + } + } + + fn sample_verdict() -> Verdict { + Verdict { + decision: Decision::Promote, + reason: VerdictReason::Promoted, + fps_improvement: 12.5, + baseline: MetricSummary { + samples: 5, + mean_fps: 100.0, + max_temperature_c: 60.0, + max_power_w: 120.0, + total_errors: 0, + }, + candidate: MetricSummary { + samples: 5, + mean_fps: 112.5, + max_temperature_c: 70.0, + max_power_w: 140.0, + total_errors: 0, + }, + } + } + + #[test] + fn experiment_and_verdict_schemas_match_checked_in() { + let cases = [ + ( + schemars::schema_for!(ExperimentSpec), + serde_json::from_str::(EXPERIMENT_SCHEMA) + .expect("experiment schema should parse"), + ), + ( + schemars::schema_for!(Verdict), + serde_json::from_str::(VERDICT_SCHEMA).expect("verdict schema should parse"), + ), + ]; + + for (generated, checked_in) in cases { + let generated = + serde_json::to_value(generated).expect("generated schema should serialize"); + let generated_properties: BTreeSet = generated["properties"] + .as_object() + .expect("generated schema should declare properties") + .keys() + .cloned() + .collect(); + let checked_in_properties: BTreeSet = checked_in["properties"] + .as_object() + .expect("checked-in schema should declare properties") + .keys() + .cloned() + .collect(); + assert_eq!(generated_properties, checked_in_properties); + assert_eq!( + string_set(&generated["required"]), + string_set(&checked_in["required"]) + ); + assert_eq!( + generated["additionalProperties"], + checked_in["additionalProperties"] + ); + } + } + + #[test] + fn verdict_enum_wire_strings_match_schema() { + assert_eq!(wire_string(Decision::Promote), "promote"); + assert_eq!(wire_string(Decision::Reject), "reject"); + assert_eq!(wire_string(VerdictReason::Promoted), "promoted"); + assert_eq!( + wire_string(VerdictReason::InsufficientSamples), + "insufficient-samples" + ); + assert_eq!( + wire_string(VerdictReason::TemperatureExceeded), + "temperature-exceeded" + ); + assert_eq!(wire_string(VerdictReason::PowerExceeded), "power-exceeded"); + assert_eq!( + wire_string(VerdictReason::ErrorsExceeded), + "errors-exceeded" + ); + assert_eq!( + wire_string(VerdictReason::InsufficientImprovement), + "insufficient-improvement" + ); + + let schema: Value = + serde_json::from_str(VERDICT_SCHEMA).expect("verdict schema should parse"); + assert_eq!( + string_set(&schema["properties"]["decision"]["enum"]), + [Decision::Promote, Decision::Reject] + .map(wire_string) + .into_iter() + .collect() + ); + assert_eq!( + string_set(&schema["properties"]["reason"]["enum"]), + [ + VerdictReason::Promoted, + VerdictReason::InsufficientSamples, + VerdictReason::TemperatureExceeded, + VerdictReason::PowerExceeded, + VerdictReason::ErrorsExceeded, + VerdictReason::InsufficientImprovement, + ] + .map(wire_string) + .into_iter() + .collect() + ); + } + + #[test] + fn experiment_and_verdict_wire_types_round_trip() { + let spec = sample_spec(); + let deserialized: ExperimentSpec = + serde_json::from_value(serde_json::to_value(&spec).expect("spec should serialize")) + .expect("spec should deserialize"); + assert_eq!(spec, deserialized); + + let verdict = sample_verdict(); + let deserialized: Verdict = serde_json::from_value( + serde_json::to_value(&verdict).expect("verdict should serialize"), + ) + .expect("verdict should deserialize"); + assert_eq!(verdict, deserialized); + } + + #[test] + fn experiment_types_reject_unknown_fields() { + let mut serialized = serde_json::to_value(sample_spec()).expect("spec should serialize"); + serialized["unexpected"] = json!(true); + assert!(serde_json::from_value::(serialized).is_err()); + + let mut serialized = + serde_json::to_value(sample_verdict()).expect("verdict should serialize"); + serialized["unexpected"] = json!(true); + assert!(serde_json::from_value::(serialized).is_err()); + } } diff --git a/crates/control-plane/src/lib.rs b/crates/control-plane/src/lib.rs index 95f15bf..42caa59 100644 --- a/crates/control-plane/src/lib.rs +++ b/crates/control-plane/src/lib.rs @@ -124,6 +124,11 @@ impl ControlPlane { stage TEXT NOT NULL, provider_id TEXT NOT NULL, payload TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS experiment_trials ( + id INTEGER PRIMARY KEY, + recorded_at TEXT NOT NULL, + payload TEXT NOT NULL );", )?; Ok(Self { @@ -183,6 +188,67 @@ impl ControlPlane { .map_err(ControlPlaneError::from) } + /// Reads the provider's current typed state snapshot. + /// + /// This read-only accessor lets an experiment runner establish a baseline + /// before measuring a candidate; it mutates no provider or journal state. + /// + /// # Errors + /// + /// Returns an error if the provider cannot produce a snapshot. + pub fn snapshot(&self) -> Result { + Ok(self.provider.snapshot()?) + } + + /// Appends a self-describing trial record to the durable trial journal and + /// returns its identifier. + /// + /// The payload is opaque to the control plane: a runner stores the spec, + /// recorded samples, and verdict together so the trial can be replayed and + /// re-evaluated from the journal alone, without chat history. + /// + /// # Errors + /// + /// Returns an error if the payload cannot be encoded or the durable journal + /// cannot be written. + pub fn record_trial(&self, payload: &impl Serialize) -> Result { + self.journal.execute( + "INSERT INTO experiment_trials (recorded_at, payload) + VALUES (strftime('%Y-%m-%dT%H:%M:%fZ', 'now'), ?1)", + params![serde_json::to_string(payload)?], + )?; + Ok(self.journal.last_insert_rowid()) + } + + /// Reads one trial record by identifier for replay and re-evaluation. + /// + /// # Errors + /// + /// Returns an error if no trial has the identifier, or the durable journal + /// cannot be read or decoded. + pub fn read_trial(&self, id: i64) -> Result { + let payload: String = self.journal.query_row( + "SELECT payload FROM experiment_trials WHERE id = ?1", + params![id], + |row| row.get(0), + )?; + Ok(serde_json::from_str(&payload)?) + } + + /// Lists recorded trial identifiers in insertion order. + /// + /// # Errors + /// + /// Returns an error if the durable journal cannot be queried. + pub fn trial_ids(&self) -> Result, ControlPlaneError> { + let mut statement = self + .journal + .prepare("SELECT id FROM experiment_trials ORDER BY id")?; + let rows = statement.query_map([], |row| row.get(0))?; + rows.collect::, _>>() + .map_err(ControlPlaneError::from) + } + fn validate(&self, request: &ChangeRequest) -> Result<(), ControlPlaneError> { let capability = self .manifest @@ -774,4 +840,31 @@ mod tests { .expect("foreign targets should be rejected"); assert!(matches!(error, ControlPlaneError::PolicyDenied(_))); } + + #[test] + fn trial_records_round_trip_through_the_journal() { + let plane = plane(false); + assert!( + plane.trial_ids().expect("ids should read").is_empty(), + "a fresh journal has no trials" + ); + let first = plane + .record_trial(&json!({ "hypothesis": "higher clocks help", "decision": "promote" })) + .expect("first trial should record"); + let second = plane + .record_trial(&json!({ "hypothesis": "wider power budget", "decision": "reject" })) + .expect("second trial should record"); + assert_eq!(plane.trial_ids().expect("ids should read"), [first, second]); + let read = plane.read_trial(first).expect("trial should read"); + assert_eq!(read["hypothesis"], "higher clocks help"); + assert_eq!(read["decision"], "promote"); + } + + #[test] + fn snapshot_reads_the_current_provider_state() { + let plane = plane(false); + let snapshot = plane.snapshot().expect("snapshot should read"); + assert_eq!(snapshot.state["value"], 7); + assert_eq!(snapshot.provider_id, "fake"); + } } diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 16cada2..fc6de9a 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -2,8 +2,8 @@ FPSMaxxing separates reasoning, policy, privilege, hardware integration, measurement, and recovery. -The current read-only alpha implements the gateway and an in-process control-plane seam (`crates/control-plane`) holding the capability registry, bounded policy, broker lifecycle, and durable SQLite experiment journal, wired to a single mock provider. -The independent watchdog restore path is implemented against that journal on the Linux-safe mock path (`apps/watchdog`); the privileged broker and experiment runner remain scaffolds. +The current read-only alpha implements the gateway, an in-process control-plane seam (`crates/control-plane`) holding the capability registry, bounded policy, broker lifecycle, and durable SQLite experiment journal, and a deterministic experiment runner (`apps/experiment-runner`) that gates measured trials through an immutable evaluator, all wired to a single mock provider. +The independent watchdog restore path is implemented against that journal on the Linux-safe mock path (`apps/watchdog`); the privileged broker remains a scaffold. ## Processes @@ -30,6 +30,8 @@ A steady-state poll reclaims only expired leases, while a crash-recovery pass (` The runner controls workload setup, warmup, repeated measurements, cooldown, correctness checks, and promotion decisions. Evaluator code is outside the LLM's writable surface. +In the safe alpha (`apps/experiment-runner`) the runner measures a baseline and a candidate against a deterministic model, then a pure immutable evaluator returns a promote or reject verdict from the recorded samples and fixed bounds alone - no clock, no LLM, no I/O. Every trial is journaled as a self-describing record so it can be replayed and re-evaluated from the journal without the original conversation. Because mock capabilities are leased and the broker lifecycle always rolls back, the verdict gates whether the candidate is applied at all rather than whether it persists; durable keep-or-rollback awaits the privileged broker. + ### Provider sidecars Each sidecar integrates exactly one service or vendor API. Sidecars advertise semantic capabilities and implement snapshot, preview, apply, verify, and rollback. They do not make cross-provider decisions. diff --git a/schemas/experiment.schema.json b/schemas/experiment.schema.json new file mode 100644 index 0000000..e5da4a0 --- /dev/null +++ b/schemas/experiment.schema.json @@ -0,0 +1,53 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/undeemed/fpsmaxxing/schemas/experiment.schema.json", + "title": "FPSMaxxing experiment specification", + "type": "object", + "additionalProperties": false, + "required": [ + "hypothesis", + "target", + "warmup_samples", + "baseline_samples", + "candidate_samples", + "bounds" + ], + "properties": { + "hypothesis": { "type": "string", "minLength": 1 }, + "target": { "$ref": "#/$defs/change_request" }, + "warmup_samples": { "type": "integer", "minimum": 0 }, + "baseline_samples": { "type": "integer", "minimum": 1 }, + "candidate_samples": { "type": "integer", "minimum": 1 }, + "bounds": { "$ref": "#/$defs/decision_bounds" } + }, + "$defs": { + "change_request": { + "type": "object", + "additionalProperties": false, + "required": ["capability_id", "parameters", "lease_seconds"], + "properties": { + "capability_id": { "type": "string", "minLength": 1 }, + "parameters": { "type": "object" }, + "lease_seconds": { "type": "integer", "minimum": 1 } + } + }, + "decision_bounds": { + "type": "object", + "additionalProperties": false, + "required": [ + "min_samples", + "min_fps_improvement", + "max_temperature_c", + "max_power_w", + "max_errors" + ], + "properties": { + "min_samples": { "type": "integer", "minimum": 1 }, + "min_fps_improvement": { "type": "number" }, + "max_temperature_c": { "type": "number" }, + "max_power_w": { "type": "number" }, + "max_errors": { "type": "integer", "minimum": 0 } + } + } + } +} diff --git a/schemas/verdict.schema.json b/schemas/verdict.schema.json new file mode 100644 index 0000000..96017d1 --- /dev/null +++ b/schemas/verdict.schema.json @@ -0,0 +1,44 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/undeemed/fpsmaxxing/schemas/verdict.schema.json", + "title": "FPSMaxxing experiment verdict", + "type": "object", + "additionalProperties": false, + "required": ["decision", "reason", "fps_improvement", "baseline", "candidate"], + "properties": { + "decision": { "enum": ["promote", "reject"] }, + "reason": { + "enum": [ + "promoted", + "insufficient-samples", + "temperature-exceeded", + "power-exceeded", + "errors-exceeded", + "insufficient-improvement" + ] + }, + "fps_improvement": { "type": "number" }, + "baseline": { "$ref": "#/$defs/metric_summary" }, + "candidate": { "$ref": "#/$defs/metric_summary" } + }, + "$defs": { + "metric_summary": { + "type": "object", + "additionalProperties": false, + "required": [ + "samples", + "mean_fps", + "max_temperature_c", + "max_power_w", + "total_errors" + ], + "properties": { + "samples": { "type": "integer", "minimum": 0 }, + "mean_fps": { "type": "number" }, + "max_temperature_c": { "type": "number" }, + "max_power_w": { "type": "number" }, + "total_errors": { "type": "integer", "minimum": 0 } + } + } + } +} From 6dc10d4b5abdcb8ba529dedb8f22d12066a04563 Mon Sep 17 00:00:00 2001 From: Jerry Xiao Date: Mon, 27 Jul 2026 06:57:03 +0000 Subject: [PATCH 02/16] no-mistakes(review): write-ahead trial records, bounded sample counts, nested schema parity --- apps/experiment-runner/src/lib.rs | 3 +- apps/experiment-runner/src/model.rs | 13 +- apps/experiment-runner/src/runner.rs | 206 ++++++++++++++++-- apps/experiment-runner/tests/integration.rs | 102 ++++++++- crates/contracts/src/lib.rs | 230 ++++++++++++++------ crates/control-plane/src/lib.rs | 63 ++++++ docs/ARCHITECTURE.md | 4 +- docs/adr/0002-alpha-experiment-journal.md | 13 +- schemas/experiment.schema.json | 6 +- 9 files changed, 533 insertions(+), 107 deletions(-) diff --git a/apps/experiment-runner/src/lib.rs b/apps/experiment-runner/src/lib.rs index 4fde3de..59121c0 100644 --- a/apps/experiment-runner/src/lib.rs +++ b/apps/experiment-runner/src/lib.rs @@ -18,5 +18,6 @@ mod runner; pub use evaluator::evaluate; pub use model::measure; pub use runner::{ - LifecycleOutcome, ReplayOutcome, RunnerError, StoredTrial, TrialRecord, replay_trial, run_trial, + LifecycleFailure, LifecycleOutcome, ReplayOutcome, RunnerError, StoredTrial, + TRIAL_RECORD_VERSION, TrialRecord, replay_trial, run_trial, }; diff --git a/apps/experiment-runner/src/model.rs b/apps/experiment-runner/src/model.rs index c13303a..63d0de8 100644 --- a/apps/experiment-runner/src/model.rs +++ b/apps/experiment-runner/src/model.rs @@ -40,11 +40,14 @@ const ERROR_ONSET: u64 = 90; #[must_use] pub fn measure(value: u64, warmup: u32, counted: u32) -> Vec { let setting = knob_to_setting(value); - let total = warmup.saturating_add(counted); - let mut samples: Vec = (0..total) - .map(|index| sample_at(setting, value, index < warmup)) - .collect(); - samples.split_off(warmup as usize) + let warmup = u64::from(warmup); + // Widening before the sum keeps the count exact for every `u32` pair, and + // dropping the warmup prefix as it is produced keeps the allocation at + // `counted` however long the prefix is. + (0..warmup + u64::from(counted)) + .map(|index| (index, sample_at(setting, value, index < warmup))) + .filter_map(|(index, sample)| (index >= warmup).then_some(sample)) + .collect() } /// Builds one deterministic sample at a given setting. diff --git a/apps/experiment-runner/src/runner.rs b/apps/experiment-runner/src/runner.rs index 039412b..c95aca2 100644 --- a/apps/experiment-runner/src/runner.rs +++ b/apps/experiment-runner/src/runner.rs @@ -7,6 +7,16 @@ //! [`replay_trial`] can re-evaluate it from the journal alone and confirm the //! recorded verdict without chat history or re-running the workload. //! +//! # Write-ahead trial records +//! +//! The record is journaled *before* the lifecycle runs, following the same +//! write-ahead principle the lifecycle journal uses (ADR 0002). A lifecycle +//! that fails after a promotion - a policy denial, a provider fault, or a +//! rollback that could not be verified - therefore still leaves a replayable +//! record of the measurements that authorized the apply; the failure is +//! amended onto that record as a [`LifecycleFailure`] and also returned to the +//! caller. +//! //! # Keep-or-rollback in the safe alpha //! //! [`ControlPlane::run_lifecycle`] always restores the pre-state before it @@ -17,7 +27,7 @@ //! [`Reject`](Decision::Reject) never mutates the knob, leaving the baseline //! untouched. The recorded [`LifecycleOutcome`] captures what happened. -use fpsmaxxing_contracts::{Decision, ExperimentSpec, MetricSample, Verdict}; +use fpsmaxxing_contracts::{Decision, ExperimentSpec, MAX_SAMPLES, MetricSample, Verdict}; use fpsmaxxing_control_plane::{ControlPlane, ControlPlaneError, LifecycleResult}; use serde::{Deserialize, Serialize}; use serde_json::Value; @@ -25,18 +35,30 @@ use thiserror::Error; use crate::{evaluate, model}; +/// Version of the durable [`TrialRecord`] format written to the journal. +/// +/// A reader refuses any record it was not built to decode, so adding a field +/// to the record is a version bump rather than a silent misread of history. +pub const TRIAL_RECORD_VERSION: u32 = 1; + /// Fail-closed errors raised while running or replaying a trial. #[derive(Debug, Error)] pub enum RunnerError { /// The broker or the durable journal rejected an operation. #[error(transparent)] ControlPlane(#[from] ControlPlaneError), + /// The experiment specification is outside the bounded alpha envelope. + #[error("experiment spec rejected: {0}")] + InvalidSpec(String), /// The experiment target did not carry an unsigned mock value. #[error("experiment target is missing an unsigned mock value")] InvalidTarget, /// The provider snapshot did not carry an unsigned mock value. #[error("provider snapshot is missing an unsigned mock value")] InvalidBaseline, + /// A journaled trial record was written by an unsupported record version. + #[error("journaled trial uses unsupported record version {0}")] + UnsupportedRecordVersion(u32), /// A journaled trial record could not be decoded for replay. #[error(transparent)] Decode(#[from] serde_json::Error), @@ -69,13 +91,42 @@ impl From<&LifecycleResult> for LifecycleOutcome { } } +/// The durable record of a lifecycle that failed after the trial was measured. +/// +/// It mirrors the `kind` and `error` fields of the lifecycle journal's terminal +/// `failed` record, so a promotion the broker refused is auditable from the +/// trial row alone. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +pub struct LifecycleFailure { + /// Stable machine-readable error kind reported by the broker. + pub kind: String, + /// Human-readable text of the error the broker returned. + pub error: String, +} + +impl From<&ControlPlaneError> for LifecycleFailure { + fn from(error: &ControlPlaneError) -> Self { + Self { + kind: error.kind().to_owned(), + error: error.to_string(), + } + } +} + /// The complete, replayable record of one trial. /// /// It holds everything the immutable evaluator needs, so re-evaluation reads /// only this record: the spec (for the decision bounds), the recorded baseline /// and candidate samples, and the verdict the evaluator produced. +/// +/// On a [`Promote`](Decision::Promote) exactly one of `lifecycle` and +/// `lifecycle_error` is normally set. Both being absent means the amend that +/// follows the lifecycle never reached the journal, and the lifecycle journal's +/// stage records for that experiment are the authoritative account. #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] pub struct TrialRecord { + /// Version of the record format; see [`TRIAL_RECORD_VERSION`]. + pub schema_version: u32, /// The experiment specification that was run. pub spec: ExperimentSpec, /// The knob value measured as the baseline. @@ -88,8 +139,12 @@ pub struct TrialRecord { pub candidate_samples: Vec, /// The verdict the immutable evaluator produced. pub verdict: Verdict, - /// The broker lifecycle outcome, present only when the trial promoted. + /// The broker lifecycle outcome, present only when a promoted trial's + /// lifecycle completed. pub lifecycle: Option, + /// Why a promoted trial's lifecycle did not complete, present only when the + /// broker returned an error. + pub lifecycle_error: Option, } /// A [`TrialRecord`] paired with the journal identifier it was stored under. @@ -122,20 +177,24 @@ impl ReplayOutcome { /// Runs one trial end to end and journals a replayable record. /// -/// Measures the baseline from the provider's current state, measures the -/// candidate from the spec target, evaluates the two, and - only on a -/// [`Promote`](Decision::Promote) - runs the candidate through the broker -/// lifecycle. The trial is written to the durable trial journal before the -/// stored record is returned. +/// Validates the spec, measures the baseline from the provider's current +/// state, measures the candidate from the spec target, evaluates the two, and +/// runs the candidate through the broker lifecycle only on a +/// [`Promote`](Decision::Promote). The record is written to the durable trial +/// journal before the lifecycle runs and amended with its outcome afterwards, +/// so the trial is replayable whether or not the lifecycle succeeds. /// /// # Errors /// -/// Returns an error if the spec target or provider snapshot lacks an unsigned -/// mock value, or if the broker or durable journal rejects an operation. +/// Returns an error if the spec is outside the bounded envelope, if the spec +/// target or provider snapshot lacks an unsigned mock value, or if the broker +/// or durable journal rejects an operation. A lifecycle error is returned only +/// after the trial record has been amended with it. pub fn run_trial( plane: &mut ControlPlane, spec: &ExperimentSpec, ) -> Result { + validate(spec)?; let baseline_value = baseline_value(plane)?; let candidate_value = candidate_value(spec)?; let baseline_samples = model::measure( @@ -149,20 +208,36 @@ pub fn run_trial( spec.candidate_samples.get(), ); let verdict = evaluate(&baseline_samples, &candidate_samples, &spec.bounds); - let lifecycle = match verdict.decision { - Decision::Promote => Some(LifecycleOutcome::from(&plane.run_lifecycle(&spec.target)?)), - Decision::Reject => None, - }; - let record = TrialRecord { + let mut record = TrialRecord { + schema_version: TRIAL_RECORD_VERSION, spec: spec.clone(), baseline_value, candidate_value, baseline_samples, candidate_samples, verdict, - lifecycle, + lifecycle: None, + lifecycle_error: None, }; let id = plane.record_trial(&record)?; + match record.verdict.decision { + Decision::Promote => match plane.run_lifecycle(&spec.target) { + Ok(result) => { + record.lifecycle = Some(LifecycleOutcome::from(&result)); + plane.amend_trial(id, &record)?; + } + Err(error) => { + record.lifecycle_error = Some(LifecycleFailure::from(&error)); + if let Err(journal_error) = plane.amend_trial(id, &record) { + eprintln!( + "fpsmaxxing-experiment-runner: could not amend trial {id} with its lifecycle failure: {journal_error}" + ); + } + return Err(error.into()); + } + }, + Decision::Reject => {} + } Ok(StoredTrial { id, record }) } @@ -175,10 +250,14 @@ pub fn run_trial( /// /// # Errors /// -/// Returns an error if the trial cannot be read from the durable journal or its -/// record cannot be decoded. +/// Returns an error if the trial cannot be read from the durable journal, its +/// record cannot be decoded, or the record was written by an unsupported +/// [`TRIAL_RECORD_VERSION`]. pub fn replay_trial(plane: &ControlPlane, id: i64) -> Result { let record: TrialRecord = serde_json::from_value(plane.read_trial(id)?)?; + if record.schema_version != TRIAL_RECORD_VERSION { + return Err(RunnerError::UnsupportedRecordVersion(record.schema_version)); + } let recomputed = evaluate( &record.baseline_samples, &record.candidate_samples, @@ -191,6 +270,38 @@ pub fn replay_trial(plane: &ControlPlane, id: i64) -> Result Result<(), RunnerError> { + let min_samples = spec.bounds.min_samples.get(); + for (label, count) in [ + ("warmup_samples", spec.warmup_samples), + ("baseline_samples", spec.baseline_samples.get()), + ("candidate_samples", spec.candidate_samples.get()), + ] { + if count > MAX_SAMPLES { + return Err(RunnerError::InvalidSpec(format!( + "{label} is {count}, above the {MAX_SAMPLES} ceiling" + ))); + } + } + for (label, count) in [ + ("baseline_samples", spec.baseline_samples.get()), + ("candidate_samples", spec.candidate_samples.get()), + ] { + if count < min_samples { + return Err(RunnerError::InvalidSpec(format!( + "{label} is {count}, below the {min_samples} the bounds require" + ))); + } + } + Ok(()) +} + /// Reads the baseline knob value from the provider's current state. fn baseline_value(plane: &ControlPlane) -> Result { plane @@ -209,3 +320,64 @@ fn candidate_value(spec: &ExperimentSpec) -> Result { .and_then(Value::as_u64) .ok_or(RunnerError::InvalidTarget) } + +#[cfg(test)] +mod tests { + use std::num::{NonZeroU32, NonZeroU64}; + + use fpsmaxxing_contracts::{ChangeRequest, DecisionBounds}; + use serde_json::json; + + use super::{ExperimentSpec, MAX_SAMPLES, RunnerError, validate}; + + fn spec(warmup: u32, baseline: u32, candidate: u32, min_samples: u32) -> ExperimentSpec { + ExperimentSpec { + hypothesis: "raise mock.value".to_owned(), + target: ChangeRequest { + capability_id: "mock.value".to_owned(), + parameters: json!({ "value": 40 }), + lease_seconds: NonZeroU64::new(30).expect("lease is non-zero"), + }, + warmup_samples: warmup, + baseline_samples: NonZeroU32::new(baseline).expect("baseline count is non-zero"), + candidate_samples: NonZeroU32::new(candidate).expect("candidate count is non-zero"), + bounds: DecisionBounds { + min_samples: NonZeroU32::new(min_samples).expect("min samples is non-zero"), + min_fps_improvement: 5.0, + max_temperature_c: 80.0, + max_power_w: 200.0, + max_errors: 0, + }, + } + } + + fn rejection(spec: &ExperimentSpec) -> String { + match validate(spec) { + Err(RunnerError::InvalidSpec(message)) => message, + other => panic!("spec should be rejected, got {other:?}"), + } + } + + #[test] + fn accepts_a_spec_inside_the_envelope() { + assert!(validate(&spec(2, 5, 5, 3)).is_ok()); + assert!(validate(&spec(MAX_SAMPLES, MAX_SAMPLES, MAX_SAMPLES, 3)).is_ok()); + } + + #[test] + fn rejects_sample_counts_above_the_ceiling() { + // Materializing this many samples would abort the process, so the spec + // is refused before any measurement work runs. + assert!(rejection(&spec(u32::MAX, 5, 5, 3)).contains("warmup_samples")); + assert!(rejection(&spec(2, MAX_SAMPLES + 1, 5, 3)).contains("baseline_samples")); + assert!(rejection(&spec(2, 5, u32::MAX, 3)).contains("candidate_samples")); + } + + #[test] + fn rejects_counts_below_the_minimum_the_bounds_require() { + // Such a spec can only ever reject with InsufficientSamples, so it is + // refused rather than measured first. + assert!(rejection(&spec(2, 2, 5, 3)).contains("baseline_samples")); + assert!(rejection(&spec(2, 5, 2, 3)).contains("candidate_samples")); + } +} diff --git a/apps/experiment-runner/tests/integration.rs b/apps/experiment-runner/tests/integration.rs index ba4f977..93188b4 100644 --- a/apps/experiment-runner/tests/integration.rs +++ b/apps/experiment-runner/tests/integration.rs @@ -3,17 +3,18 @@ //! These exercise the trial runner and journal-only replay end to end against //! the mock provider through the broker: a promoted experiment that runs the //! full lifecycle, a rejected experiment that is never applied and leaves the -//! baseline untouched, and a replay from the durable journal alone - reopened -//! as a fresh handle - that reproduces the recorded verdict exactly. The final -//! test states the MVP acceptance criterion directly. +//! baseline untouched, a promoted experiment whose lifecycle the broker refuses +//! but whose write-ahead record survives, and a replay from the durable journal +//! alone - reopened as a fresh handle - that reproduces the recorded verdict +//! exactly. The final test states the MVP acceptance criterion directly. use std::num::{NonZeroU32, NonZeroU64}; use fpsmaxxing_contracts::{ ChangeRequest, Decision, DecisionBounds, ExperimentSpec, VerdictReason, }; -use fpsmaxxing_control_plane::ControlPlane; -use fpsmaxxing_experiment_runner::{evaluate, replay_trial, run_trial}; +use fpsmaxxing_control_plane::{ControlPlane, ControlPlaneError}; +use fpsmaxxing_experiment_runner::{RunnerError, TrialRecord, evaluate, replay_trial, run_trial}; use fpsmaxxing_mock_provider::MockProvider; use serde_json::json; use tempfile::NamedTempFile; @@ -21,12 +22,23 @@ use tempfile::NamedTempFile; /// Builds a spec that drives the mock knob to `candidate` under the given /// improvement threshold and temperature ceiling. fn spec_for(candidate: u64, min_fps_improvement: f64, max_temperature_c: f64) -> ExperimentSpec { + spec_with_lease(candidate, min_fps_improvement, max_temperature_c, 30) +} + +/// Builds the same spec with an explicit lease, so a test can drive the broker +/// policy into denying the change the evaluator promoted. +fn spec_with_lease( + candidate: u64, + min_fps_improvement: f64, + max_temperature_c: f64, + lease_seconds: u64, +) -> ExperimentSpec { ExperimentSpec { hypothesis: format!("raise mock.value to {candidate}"), target: ChangeRequest { capability_id: "mock.value".to_owned(), parameters: json!({ "value": candidate }), - lease_seconds: NonZeroU64::new(30).expect("lease is non-zero"), + lease_seconds: NonZeroU64::new(lease_seconds).expect("lease is non-zero"), }, warmup_samples: 2, baseline_samples: NonZeroU32::new(5).expect("baseline count is non-zero"), @@ -103,6 +115,47 @@ fn a_rejected_experiment_is_never_applied_and_leaves_the_baseline() { assert_eq!(current_value(&plane), 10); } +#[test] +fn a_promoted_trial_survives_a_lifecycle_the_broker_refuses() { + let journal = NamedTempFile::new().expect("temp journal"); + let mut plane = + ControlPlane::open(Box::new(MockProvider::new(10)), journal.path()).expect("open"); + + // The evaluator promotes on the measurements, but the 400 second lease is + // outside the broker's policy envelope, so the lifecycle never runs. + let spec = spec_with_lease(40, 5.0, 80.0, 400); + let error = run_trial(&mut plane, &spec).expect_err("policy should deny the lease"); + assert!(matches!( + error, + RunnerError::ControlPlane(ControlPlaneError::PolicyDenied(_)) + )); + + // The measurements that authorized the promotion were journaled ahead of + // the lifecycle, so the trial is still discoverable and replayable. + let ids = plane.trial_ids().expect("trial ids"); + assert_eq!(ids.len(), 1, "the failed promotion is still journaled"); + let record: TrialRecord = + serde_json::from_value(plane.read_trial(ids[0]).expect("trial should read")) + .expect("trial should decode"); + assert_eq!(record.verdict.decision, Decision::Promote); + assert!( + record.lifecycle.is_none(), + "no lifecycle completed for a denied change" + ); + let failure = record + .lifecycle_error + .expect("the refused lifecycle is recorded on the trial"); + assert_eq!(failure.kind, "policy-denied"); + assert!(failure.error.contains("lease exceeds 300 seconds")); + + let outcome = replay_trial(&plane, ids[0]).expect("replay trial"); + assert!(outcome.is_consistent()); + assert_eq!(outcome.recomputed.decision, Decision::Promote); + + // Nothing reached the provider, so the baseline is untouched. + assert_eq!(current_value(&plane), 10); +} + #[test] fn a_trial_replays_from_the_journal_alone_with_an_identical_verdict() { let journal = NamedTempFile::new().expect("temp journal"); @@ -130,6 +183,43 @@ fn a_trial_replays_from_the_journal_alone_with_an_identical_verdict() { assert_eq!(outcome.recomputed.decision, Decision::Promote); } +#[test] +fn an_out_of_envelope_spec_is_refused_before_any_measurement() { + let journal = NamedTempFile::new().expect("temp journal"); + let mut plane = + ControlPlane::open(Box::new(MockProvider::new(10)), journal.path()).expect("open"); + + let mut spec = spec_for(40, 5.0, 80.0); + spec.warmup_samples = u32::MAX; + let error = run_trial(&mut plane, &spec).expect_err("unbounded counts should be refused"); + assert!(matches!(error, RunnerError::InvalidSpec(_))); + + assert!( + plane.trial_ids().expect("trial ids").is_empty(), + "a refused spec journals nothing" + ); + assert_eq!(current_value(&plane), 10); +} + +#[test] +fn a_trial_record_from_an_unsupported_version_fails_closed() { + let journal = NamedTempFile::new().expect("temp journal"); + let mut plane = + ControlPlane::open(Box::new(MockProvider::new(10)), journal.path()).expect("open"); + let trial = run_trial(&mut plane, &spec_for(40, 5.0, 80.0)).expect("run trial"); + + // Stand in for a record a future runner wrote: the reader must refuse it + // rather than decode it under this version's field meanings. + let mut payload = plane.read_trial(trial.id).expect("trial should read"); + payload["schema_version"] = json!(2); + plane + .amend_trial(trial.id, &payload) + .expect("trial should amend"); + + let error = replay_trial(&plane, trial.id).expect_err("a future record should be refused"); + assert!(matches!(error, RunnerError::UnsupportedRecordVersion(2))); +} + #[test] fn mvp_one_measured_experiment_is_promoted_or_rejected_by_the_evaluator() { let journal = NamedTempFile::new().expect("temp journal"); diff --git a/crates/contracts/src/lib.rs b/crates/contracts/src/lib.rs index c1265a5..85aa5c5 100644 --- a/crates/contracts/src/lib.rs +++ b/crates/contracts/src/lib.rs @@ -138,6 +138,14 @@ pub struct DecisionBounds { pub max_errors: u64, } +/// Inclusive ceiling on every sample count in an [`ExperimentSpec`]. +/// +/// Sample counts arrive over the wire from an LLM-authored spec and size the +/// measurement buffers a runner allocates, so they are bounded like every other +/// parameter the broker accepts. The ceiling is mirrored as `maximum` in +/// `schemas/experiment.schema.json`. +pub const MAX_SAMPLES: u32 = 100_000; + /// A declarative, typed experiment the runner can execute, journal, and replay. #[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] #[serde(deny_unknown_fields)] @@ -146,11 +154,15 @@ pub struct ExperimentSpec { pub hypothesis: String, /// Bounded, policy-checkable capability change under test. pub target: ChangeRequest, - /// Leading measurements discarded from each phase before counting. + /// Leading measurements discarded from each phase before counting; at most + /// [`MAX_SAMPLES`]. + #[schemars(range(max = MAX_SAMPLES))] pub warmup_samples: u32, - /// Counted baseline measurements to record. + /// Counted baseline measurements to record; at most [`MAX_SAMPLES`]. + #[schemars(range(max = MAX_SAMPLES))] pub baseline_samples: NonZeroU32, - /// Counted candidate measurements to record. + /// Counted candidate measurements to record; at most [`MAX_SAMPLES`]. + #[schemars(range(max = MAX_SAMPLES))] pub candidate_samples: NonZeroU32, /// Thresholds the evaluator uses to promote or reject. pub bounds: DecisionBounds, @@ -210,7 +222,7 @@ mod tests { use serde_json::{Value, json}; use super::{ - CapabilityDescriptor, ChangeRequest, Decision, DecisionBounds, ExperimentSpec, + CapabilityDescriptor, ChangeRequest, Decision, DecisionBounds, ExperimentSpec, MAX_SAMPLES, MetricSummary, NonZeroU32, NonZeroU64, Persistence, ProviderManifest, RiskClass, Verdict, VerdictReason, }; @@ -392,44 +404,142 @@ mod tests { assert!(serde_json::from_value::(serialized).is_err()); } + /// Asserts that two object schemas declare the same fields. + fn assert_object_parity(label: &str, generated: &Value, checked_in: &Value) { + let generated_properties: BTreeSet = generated["properties"] + .as_object() + .unwrap_or_else(|| panic!("generated {label} should declare properties")) + .keys() + .cloned() + .collect(); + let checked_in_properties: BTreeSet = checked_in["properties"] + .as_object() + .unwrap_or_else(|| panic!("checked-in {label} should declare properties")) + .keys() + .cloned() + .collect(); + assert_eq!(generated_properties, checked_in_properties, "{label}"); + assert_eq!( + string_set(&generated["required"]), + string_set(&checked_in["required"]), + "{label}" + ); + assert_eq!( + generated["additionalProperties"], checked_in["additionalProperties"], + "{label}" + ); + } + + /// Names the `$defs` entries of a generated schema that describe objects. + /// + /// Enum definitions are skipped: they carry no `properties` and the + /// checked-in schemas inline them, so the `*_enum_wire_strings_match_*` + /// tests cover their parity instead. + fn generated_object_definitions(generated: &Value) -> BTreeSet { + generated["$defs"] + .as_object() + .map(|definitions| { + definitions + .iter() + .filter(|(_, definition)| definition.get("properties").is_some()) + .map(|(name, _)| name.clone()) + .collect() + }) + .unwrap_or_default() + } + + /// One generated schema paired with the checked-in file it must mirror. + struct SchemaCase { + label: &'static str, + generated: Value, + checked_in: Value, + /// Every object definition the generated schema puts in `$defs`, mapped + /// to the checked-in `$defs` key that mirrors it. `None` marks a + /// definition that a separate checked-in schema file already covers. + definitions: &'static [(&'static str, Option<&'static str>)], + } + + fn schema_cases() -> Vec { + vec![ + SchemaCase { + label: "CapabilityDescriptor", + generated: serde_json::to_value(schemars::schema_for!(CapabilityDescriptor)) + .expect("generated schema should serialize"), + checked_in: serde_json::from_str(CAPABILITY_SCHEMA) + .expect("capability schema should parse"), + definitions: &[], + }, + SchemaCase { + label: "ProviderManifest", + generated: serde_json::to_value(schemars::schema_for!(ProviderManifest)) + .expect("generated schema should serialize"), + checked_in: serde_json::from_str(SIDECAR_SCHEMA) + .expect("sidecar schema should parse"), + // capability.schema.json is referenced across files and checked + // as its own case. + definitions: &[("CapabilityDescriptor", None)], + }, + SchemaCase { + label: "ExperimentSpec", + generated: serde_json::to_value(schemars::schema_for!(ExperimentSpec)) + .expect("generated schema should serialize"), + checked_in: serde_json::from_str(EXPERIMENT_SCHEMA) + .expect("experiment schema should parse"), + definitions: &[ + ("ChangeRequest", Some("change_request")), + ("DecisionBounds", Some("decision_bounds")), + ], + }, + SchemaCase { + label: "Verdict", + generated: serde_json::to_value(schemars::schema_for!(Verdict)) + .expect("generated schema should serialize"), + checked_in: serde_json::from_str(VERDICT_SCHEMA) + .expect("verdict schema should parse"), + definitions: &[("MetricSummary", Some("metric_summary"))], + }, + ] + } + #[test] fn generated_schemas_match_checked_in_schemas() { - let cases = [ - ( - schemars::schema_for!(CapabilityDescriptor), - serde_json::from_str::(CAPABILITY_SCHEMA) - .expect("capability schema should parse"), - ), - ( - schemars::schema_for!(ProviderManifest), - serde_json::from_str::(SIDECAR_SCHEMA).expect("sidecar schema should parse"), - ), - ]; - - for (generated, checked_in) in cases { - let generated = - serde_json::to_value(generated).expect("generated schema should serialize"); - let generated_properties: BTreeSet = generated["properties"] - .as_object() - .expect("generated schema should declare properties") - .keys() - .cloned() - .collect(); - let checked_in_properties: BTreeSet = checked_in["properties"] - .as_object() - .expect("checked-in schema should declare properties") - .keys() - .cloned() - .collect(); - assert_eq!(generated_properties, checked_in_properties); + for case in schema_cases() { + assert_object_parity(case.label, &case.generated, &case.checked_in); + + // Every nested object type must be mapped, so introducing one + // fails this test until the checked-in schema gains a matching + // definition. assert_eq!( - string_set(&generated["required"]), - string_set(&checked_in["required"]) + generated_object_definitions(&case.generated), + case.definitions + .iter() + .map(|(name, _)| (*name).to_owned()) + .collect::>(), + "{} nested object definitions", + case.label ); + let checked_in_definitions: BTreeSet = case.checked_in["$defs"] + .as_object() + .map(|definitions| definitions.keys().cloned().collect()) + .unwrap_or_default(); assert_eq!( - generated["additionalProperties"], - checked_in["additionalProperties"] + checked_in_definitions, + case.definitions + .iter() + .filter_map(|(_, target)| target.map(ToOwned::to_owned)) + .collect::>(), + "{} checked-in $defs", + case.label ); + + for (name, target) in case.definitions { + let Some(target) = target else { continue }; + assert_object_parity( + &format!("{}/$defs/{target}", case.label), + &case.generated["$defs"][name], + &case.checked_in["$defs"][target], + ); + } } } @@ -486,42 +596,20 @@ mod tests { } #[test] - fn experiment_and_verdict_schemas_match_checked_in() { - let cases = [ - ( - schemars::schema_for!(ExperimentSpec), - serde_json::from_str::(EXPERIMENT_SCHEMA) - .expect("experiment schema should parse"), - ), - ( - schemars::schema_for!(Verdict), - serde_json::from_str::(VERDICT_SCHEMA).expect("verdict schema should parse"), - ), - ]; - - for (generated, checked_in) in cases { - let generated = - serde_json::to_value(generated).expect("generated schema should serialize"); - let generated_properties: BTreeSet = generated["properties"] - .as_object() - .expect("generated schema should declare properties") - .keys() - .cloned() - .collect(); - let checked_in_properties: BTreeSet = checked_in["properties"] - .as_object() - .expect("checked-in schema should declare properties") - .keys() - .cloned() - .collect(); - assert_eq!(generated_properties, checked_in_properties); - assert_eq!( - string_set(&generated["required"]), - string_set(&checked_in["required"]) - ); + fn sample_counts_are_bounded_like_the_schema() { + let schema: Value = + serde_json::from_str(EXPERIMENT_SCHEMA).expect("experiment schema should parse"); + for field in ["warmup_samples", "baseline_samples", "candidate_samples"] { + assert_eq!(schema["properties"][field]["maximum"], json!(MAX_SAMPLES)); + } + + let generated = serde_json::to_value(schemars::schema_for!(ExperimentSpec)) + .expect("generated schema should serialize"); + for field in ["warmup_samples", "baseline_samples", "candidate_samples"] { assert_eq!( - generated["additionalProperties"], - checked_in["additionalProperties"] + generated["properties"][field]["maximum"], + json!(MAX_SAMPLES), + "{field}" ); } } diff --git a/crates/control-plane/src/lib.rs b/crates/control-plane/src/lib.rs index 42caa59..4248a77 100644 --- a/crates/control-plane/src/lib.rs +++ b/crates/control-plane/src/lib.rs @@ -220,6 +220,31 @@ impl ControlPlane { Ok(self.journal.last_insert_rowid()) } + /// Replaces the payload of an already-recorded trial. + /// + /// A runner journals its measured trial ahead of the broker lifecycle so + /// the measurements that authorized a promotion survive a lifecycle + /// failure, then amends the same row with the outcome it could not know + /// yet. The trial's identity, spec, samples, and verdict are fixed at + /// insertion; only the outcome fields are filled in. + /// + /// # Errors + /// + /// Returns an error if no trial has the identifier, the payload cannot be + /// encoded, or the durable journal cannot be written. + pub fn amend_trial(&self, id: i64, payload: &impl Serialize) -> Result<(), ControlPlaneError> { + let updated = self.journal.execute( + "UPDATE experiment_trials SET payload = ?2 WHERE id = ?1", + params![id, serde_json::to_string(payload)?], + )?; + if updated == 0 { + return Err(ControlPlaneError::Journal( + rusqlite::Error::QueryReturnedNoRows, + )); + } + Ok(()) + } + /// Reads one trial record by identifier for replay and re-evaluation. /// /// # Errors @@ -860,6 +885,44 @@ mod tests { assert_eq!(read["decision"], "promote"); } + #[test] + fn amending_a_trial_replaces_only_that_payload() { + let plane = plane(false); + let first = plane + .record_trial(&json!({ "decision": "promote", "lifecycle": null })) + .expect("first trial should record"); + let second = plane + .record_trial(&json!({ "decision": "reject" })) + .expect("second trial should record"); + plane + .amend_trial( + first, + &json!({ "decision": "promote", "lifecycle": { "verified": true } }), + ) + .expect("trial should amend"); + assert_eq!( + plane.read_trial(first).expect("trial should read")["lifecycle"]["verified"], + true + ); + assert_eq!( + plane.read_trial(second).expect("trial should read")["decision"], + "reject" + ); + assert_eq!(plane.trial_ids().expect("ids should read"), [first, second]); + } + + #[test] + fn amending_an_unknown_trial_fails_closed() { + let plane = plane(false); + let error = plane + .amend_trial(404, &json!({ "decision": "promote" })) + .expect_err("an unrecorded trial cannot be amended"); + assert!(matches!( + error, + ControlPlaneError::Journal(rusqlite::Error::QueryReturnedNoRows) + )); + } + #[test] fn snapshot_reads_the_current_provider_state() { let plane = plane(false); diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index fc6de9a..565d6b4 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -30,7 +30,9 @@ A steady-state poll reclaims only expired leases, while a crash-recovery pass (` The runner controls workload setup, warmup, repeated measurements, cooldown, correctness checks, and promotion decisions. Evaluator code is outside the LLM's writable surface. -In the safe alpha (`apps/experiment-runner`) the runner measures a baseline and a candidate against a deterministic model, then a pure immutable evaluator returns a promote or reject verdict from the recorded samples and fixed bounds alone - no clock, no LLM, no I/O. Every trial is journaled as a self-describing record so it can be replayed and re-evaluated from the journal without the original conversation. Because mock capabilities are leased and the broker lifecycle always rolls back, the verdict gates whether the candidate is applied at all rather than whether it persists; durable keep-or-rollback awaits the privileged broker. +In the safe alpha (`apps/experiment-runner`) the runner measures a baseline and a candidate against a deterministic model, then a pure immutable evaluator returns a promote or reject verdict from the recorded samples and fixed bounds alone - no clock, no LLM, no I/O. Every trial is journaled as a self-describing, versioned record so it can be replayed and re-evaluated from the journal without the original conversation. +The record is written ahead of the broker lifecycle and amended with its outcome, so a promotion the broker refuses still leaves the measurements that authorized it. +Sample counts are bounded by the spec schema and rechecked by the runner before any measurement runs. Because mock capabilities are leased and the broker lifecycle always rolls back, the verdict gates whether the candidate is applied at all rather than whether it persists; durable keep-or-rollback awaits the privileged broker. ### Provider sidecars diff --git a/docs/adr/0002-alpha-experiment-journal.md b/docs/adr/0002-alpha-experiment-journal.md index 827dd62..c246b9b 100644 --- a/docs/adr/0002-alpha-experiment-journal.md +++ b/docs/adr/0002-alpha-experiment-journal.md @@ -8,16 +8,23 @@ The alpha journal keeps a single `experiment_journal` table of stage records. Each lifecycle allocates its correlation ID atomically with the snapshot record, commits a write-ahead `apply-intent` record holding the full change request before the provider mutates state, and closes with exactly one terminal `completed` or `failed` record carrying a structured error kind and the failing stage. When a restore failure supersedes an earlier apply or verify failure, the `failed` record embeds the suppressed failure so the audit trail never loses the primary error. -Full two-phase intent/result journaling for every stage and a dedicated experiments table are deferred until the seam is promoted to the privileged broker. +Full two-phase intent/result journaling for every stage is deferred until the seam is promoted to the privileged broker. + +The same journal database also keeps an `experiment_trials` table of self-describing trial records, added when the deterministic experiment engine landed. +This supersedes the original deferral of a dedicated experiments table. +A trial record is a different kind of row from a lifecycle stage record: it holds the spec, the recorded baseline and candidate samples, and the immutable evaluator's verdict, so a trial re-evaluates from the journal alone without chat history or a re-run workload. +Storing that under the stage schema would have meant either overloading `payload` with a shape `doctor` cannot interpret or restructuring the stage table, so a second table was the smaller change. +Trial rows follow the same write-ahead principle as the lifecycle journal: the runner records the measured trial before invoking the lifecycle and amends that row with the outcome, so a promotion the broker refuses still leaves the measurements that authorized it. ## Rationale The write-ahead apply intent makes a crash between mutation and journaling distinguishable from an apply that never started, and the terminal record guarantees that a surviving process never leaves an experiment with only partial stage rows. -A per-stage two-phase protocol and a separate experiments table would duplicate that machinery for the mock-only alpha before the broker owns the transaction log. +A per-stage two-phase protocol would duplicate that machinery for the mock-only alpha before the broker owns the transaction log. +Trial records carry a `schema_version` so a future field addition is a version bump a reader can refuse rather than a silent misread of journaled history. ## Consequences -- `doctor` reports experiments with an `apply-intent` record but no terminal outcome as dangling. +- `doctor` reports experiments with an `apply-intent` record but no terminal outcome as dangling; it does not yet inspect `experiment_trials`. - Correlation IDs are allocated inside an immediate transaction with a busy timeout, so concurrent gateways sharing one journal file cannot mint duplicate IDs. - `LifecycleResult` stays in `crates/control-plane` as a deliberate alpha seam even though the gateway serializes it into MCP tool-result text; it moves to `crates/contracts` with a pinned JSON schema at broker promotion. - Broker promotion revisits journaling as part of the privileged transaction log design. diff --git a/schemas/experiment.schema.json b/schemas/experiment.schema.json index e5da4a0..4998190 100644 --- a/schemas/experiment.schema.json +++ b/schemas/experiment.schema.json @@ -15,9 +15,9 @@ "properties": { "hypothesis": { "type": "string", "minLength": 1 }, "target": { "$ref": "#/$defs/change_request" }, - "warmup_samples": { "type": "integer", "minimum": 0 }, - "baseline_samples": { "type": "integer", "minimum": 1 }, - "candidate_samples": { "type": "integer", "minimum": 1 }, + "warmup_samples": { "type": "integer", "minimum": 0, "maximum": 100000 }, + "baseline_samples": { "type": "integer", "minimum": 1, "maximum": 100000 }, + "candidate_samples": { "type": "integer", "minimum": 1, "maximum": 100000 }, "bounds": { "$ref": "#/$defs/decision_bounds" } }, "$defs": { From 05927884c2a6e005d0426b24bb0d6d8d063d7a4e Mon Sep 17 00:00:00 2001 From: Jerry Xiao Date: Mon, 27 Jul 2026 07:11:27 +0000 Subject: [PATCH 03/16] no-mistakes(review): saturate error totals, bound candidate value, tighten replay gate --- README.md | 3 +- apps/experiment-runner/src/evaluator.rs | 18 ++++- apps/experiment-runner/src/main.rs | 20 +++++- apps/experiment-runner/src/runner.rs | 73 +++++++++++++++++---- apps/experiment-runner/tests/integration.rs | 32 +++++++++ crates/contracts/src/lib.rs | 7 +- crates/control-plane/src/lib.rs | 15 +++-- schemas/experiment.schema.json | 6 +- 8 files changed, 147 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index b5dbb5b..2991b4e 100644 --- a/README.md +++ b/README.md @@ -88,7 +88,8 @@ The repository currently includes: - A working stdio MCP gateway that serves the mock path end to end - A CLI `doctor` command that reports gateway and journal status - An independent watchdog that restores prior state from the journal after a crash or lease expiry, on the Linux-safe mock path -- Scaffolds for the privileged broker and experiment runner +- A deterministic experiment runner that gates measured trials through an immutable evaluator and replays them from the journal alone +- Scaffolds for the privileged broker - OSS governance, security policy, issue templates, and CI - An organized [documentation index](docs/README.md) with architecture, plans, threat model, and provider guides diff --git a/apps/experiment-runner/src/evaluator.rs b/apps/experiment-runner/src/evaluator.rs index c8736b4..b451342 100644 --- a/apps/experiment-runner/src/evaluator.rs +++ b/apps/experiment-runner/src/evaluator.rs @@ -77,7 +77,9 @@ pub fn evaluate( /// temperature and power fields report the worst (highest) observed value and /// errors are summed. Counting the divisor as an `f64` avoids a lossy /// integer-to-float cast without changing the result for realistic sample -/// counts. +/// counts. Samples may come from a journaled record this build did not write, +/// so the error total saturates rather than overflowing: a saturated total is +/// far above any ceiling and still rejects. fn summarize(samples: &[MetricSample]) -> MetricSummary { let mut sum_fps = 0.0_f64; let mut divisor = 0.0_f64; @@ -89,7 +91,7 @@ fn summarize(samples: &[MetricSample]) -> MetricSummary { divisor += 1.0; max_temperature_c = max_temperature_c.max(sample.temperature_c); max_power_w = max_power_w.max(sample.power_w); - total_errors += sample.errors; + total_errors = total_errors.saturating_add(sample.errors); } MetricSummary { samples: samples.len() as u64, @@ -204,6 +206,18 @@ mod tests { assert_eq!(verdict.reason, VerdictReason::Promoted); } + #[test] + fn an_error_total_beyond_u64_saturates_and_rejects() { + // Journaled samples can come from a record this build did not write, so + // aggregation must not overflow on hostile counts. + let baseline = samples(100.0, 60.0, 150.0, 0, 3); + let candidate = samples(140.0, 70.0, 180.0, u64::MAX, 3); + let verdict = evaluate(&baseline, &candidate, &bounds()); + assert_eq!(verdict.candidate.total_errors, u64::MAX); + assert_eq!(verdict.decision, Decision::Reject); + assert_eq!(verdict.reason, VerdictReason::ErrorsExceeded); + } + #[test] fn evaluation_is_deterministic_across_repeated_calls() { let baseline = samples(100.0, 60.0, 150.0, 0, 4); diff --git a/apps/experiment-runner/src/main.rs b/apps/experiment-runner/src/main.rs index 05da6c8..013e752 100644 --- a/apps/experiment-runner/src/main.rs +++ b/apps/experiment-runner/src/main.rs @@ -4,8 +4,15 @@ //! journals it, then replays it from the durable journal alone and confirms the //! re-evaluated verdict matches the one recorded at run time. The journal is an //! in-memory `SQLite` database, so the demo leaves nothing on disk. +//! +//! A replay that diverges from the journal means the record was tampered with +//! or the immutable evaluator drifted, so the demo exits non-zero rather than +//! reporting the divergence as a successful run. -use std::num::{NonZeroU32, NonZeroU64}; +use std::{ + num::{NonZeroU32, NonZeroU64}, + process::ExitCode, +}; use fpsmaxxing_contracts::{ChangeRequest, DecisionBounds, ExperimentSpec}; use fpsmaxxing_control_plane::ControlPlane; @@ -13,7 +20,7 @@ use fpsmaxxing_experiment_runner::{RunnerError, replay_trial, run_trial}; use fpsmaxxing_mock_provider::MockProvider; use serde_json::json; -fn main() -> Result<(), RunnerError> { +fn main() -> Result { let mut plane = ControlPlane::open(Box::new(MockProvider::new(10)), ":memory:")?; let spec = demo_spec(); @@ -37,7 +44,14 @@ fn main() -> Result<(), RunnerError> { replay.recomputed.decision, replay.is_consistent() ); - Ok(()) + if !replay.is_consistent() { + eprintln!( + "fpsmaxxing-experiment-runner: replay of trial {} diverged from the journal: recorded {:?}, recomputed {:?}", + replay.trial_id, replay.recorded.reason, replay.recomputed.reason + ); + return Ok(ExitCode::FAILURE); + } + Ok(ExitCode::SUCCESS) } /// Builds a spec that raises the mock knob within the safety envelope. diff --git a/apps/experiment-runner/src/runner.rs b/apps/experiment-runner/src/runner.rs index c95aca2..2885ca6 100644 --- a/apps/experiment-runner/src/runner.rs +++ b/apps/experiment-runner/src/runner.rs @@ -26,9 +26,15 @@ //! full snapshot/preview/apply/verify/rollback lifecycle, while a //! [`Reject`](Decision::Reject) never mutates the knob, leaving the baseline //! untouched. The recorded [`LifecycleOutcome`] captures what happened. +//! +//! Measuring the candidate before it is applied is likewise specific to the +//! pure stand-in model, whose samples depend only on the knob value. Real +//! `PresentMon` or hardware telemetry cannot observe a candidate that was never +//! written, so swapping it in means moving the candidate measurement inside the +//! apply/lease window and running the evaluator gate after it. use fpsmaxxing_contracts::{Decision, ExperimentSpec, MAX_SAMPLES, MetricSample, Verdict}; -use fpsmaxxing_control_plane::{ControlPlane, ControlPlaneError, LifecycleResult}; +use fpsmaxxing_control_plane::{ControlPlane, ControlPlaneError, LifecycleResult, MAX_MOCK_VALUE}; use serde::{Deserialize, Serialize}; use serde_json::Value; use thiserror::Error; @@ -254,10 +260,19 @@ pub fn run_trial( /// record cannot be decoded, or the record was written by an unsupported /// [`TRIAL_RECORD_VERSION`]. pub fn replay_trial(plane: &ControlPlane, id: i64) -> Result { - let record: TrialRecord = serde_json::from_value(plane.read_trial(id)?)?; - if record.schema_version != TRIAL_RECORD_VERSION { - return Err(RunnerError::UnsupportedRecordVersion(record.schema_version)); + let payload = plane.read_trial(id)?; + // The version is read off the raw payload so a record whose fields this + // build cannot decode still reports the version that wrote it rather than a + // decode error that says nothing about why. + let version = payload + .get("schema_version") + .and_then(Value::as_u64) + .and_then(|version| u32::try_from(version).ok()) + .unwrap_or_default(); + if version != TRIAL_RECORD_VERSION { + return Err(RunnerError::UnsupportedRecordVersion(version)); } + let record: TrialRecord = serde_json::from_value(payload)?; let recomputed = evaluate( &record.baseline_samples, &record.candidate_samples, @@ -270,12 +285,15 @@ pub fn replay_trial(plane: &ControlPlane, id: i64) -> Result Result<(), RunnerError> { let min_samples = spec.bounds.min_samples.get(); for (label, count) in [ @@ -299,6 +317,12 @@ fn validate(spec: &ExperimentSpec) -> Result<(), RunnerError> { ))); } } + let value = candidate_value(spec)?; + if value > MAX_MOCK_VALUE { + return Err(RunnerError::InvalidSpec(format!( + "target value is {value}, above the {MAX_MOCK_VALUE} the policy allows" + ))); + } Ok(()) } @@ -328,14 +352,24 @@ mod tests { use fpsmaxxing_contracts::{ChangeRequest, DecisionBounds}; use serde_json::json; - use super::{ExperimentSpec, MAX_SAMPLES, RunnerError, validate}; + use super::{ExperimentSpec, MAX_MOCK_VALUE, MAX_SAMPLES, RunnerError, validate}; fn spec(warmup: u32, baseline: u32, candidate: u32, min_samples: u32) -> ExperimentSpec { + spec_for_value(warmup, baseline, candidate, min_samples, 40) + } + + fn spec_for_value( + warmup: u32, + baseline: u32, + candidate: u32, + min_samples: u32, + value: u64, + ) -> ExperimentSpec { ExperimentSpec { hypothesis: "raise mock.value".to_owned(), target: ChangeRequest { capability_id: "mock.value".to_owned(), - parameters: json!({ "value": 40 }), + parameters: json!({ "value": value }), lease_seconds: NonZeroU64::new(30).expect("lease is non-zero"), }, warmup_samples: warmup, @@ -380,4 +414,21 @@ mod tests { assert!(rejection(&spec(2, 2, 5, 3)).contains("baseline_samples")); assert!(rejection(&spec(2, 5, 2, 3)).contains("candidate_samples")); } + + #[test] + fn rejects_a_candidate_value_above_the_policy_bound() { + // The broker only sees the value inside the lifecycle, which a rejected + // trial never reaches, so the measurement path bounds it itself. + assert!(validate(&spec_for_value(2, 5, 5, 3, MAX_MOCK_VALUE)).is_ok()); + let message = rejection(&spec_for_value(2, 5, 5, 3, MAX_MOCK_VALUE + 1)); + assert!(message.contains("target value"), "{message}"); + assert!(rejection(&spec_for_value(2, 5, 5, 3, u64::MAX)).contains("target value")); + } + + #[test] + fn rejects_a_target_without_an_unsigned_value() { + let mut spec = spec(2, 5, 5, 3); + spec.target.parameters = json!({ "value": "high" }); + assert!(matches!(validate(&spec), Err(RunnerError::InvalidTarget))); + } } diff --git a/apps/experiment-runner/tests/integration.rs b/apps/experiment-runner/tests/integration.rs index 93188b4..832fd30 100644 --- a/apps/experiment-runner/tests/integration.rs +++ b/apps/experiment-runner/tests/integration.rs @@ -201,6 +201,26 @@ fn an_out_of_envelope_spec_is_refused_before_any_measurement() { assert_eq!(current_value(&plane), 10); } +#[test] +fn a_candidate_outside_the_policy_bound_is_refused_before_any_measurement() { + let journal = NamedTempFile::new().expect("temp journal"); + let mut plane = + ControlPlane::open(Box::new(MockProvider::new(10)), journal.path()).expect("open"); + + // The broker's policy bound is only checked inside the lifecycle, which a + // rejected trial never reaches; measuring such a candidate first would + // aggregate errors past `u64::MAX`. + let error = run_trial(&mut plane, &spec_for(u64::MAX, 5.0, 80.0)) + .expect_err("an out-of-policy candidate should be refused"); + assert!(matches!(error, RunnerError::InvalidSpec(_))); + + assert!( + plane.trial_ids().expect("trial ids").is_empty(), + "a refused spec journals nothing" + ); + assert_eq!(current_value(&plane), 10); +} + #[test] fn a_trial_record_from_an_unsupported_version_fails_closed() { let journal = NamedTempFile::new().expect("temp journal"); @@ -218,6 +238,18 @@ fn a_trial_record_from_an_unsupported_version_fails_closed() { let error = replay_trial(&plane, trial.id).expect_err("a future record should be refused"); assert!(matches!(error, RunnerError::UnsupportedRecordVersion(2))); + + // A future record that also reshaped a field must still report the version + // that wrote it, not an opaque decode error about the reshaped field. + payload["verdict"] = json!("promote"); + plane + .amend_trial(trial.id, &payload) + .expect("trial should amend"); + let error = replay_trial(&plane, trial.id).expect_err("a future record should be refused"); + assert!( + matches!(error, RunnerError::UnsupportedRecordVersion(2)), + "expected a version error, got {error:?}" + ); } #[test] diff --git a/crates/contracts/src/lib.rs b/crates/contracts/src/lib.rs index 85aa5c5..1ff194d 100644 --- a/crates/contracts/src/lib.rs +++ b/crates/contracts/src/lib.rs @@ -142,9 +142,10 @@ pub struct DecisionBounds { /// /// Sample counts arrive over the wire from an LLM-authored spec and size the /// measurement buffers a runner allocates, so they are bounded like every other -/// parameter the broker accepts. The ceiling is mirrored as `maximum` in -/// `schemas/experiment.schema.json`. -pub const MAX_SAMPLES: u32 = 100_000; +/// parameter the broker accepts. The ceiling also bounds the journaled trial +/// record, which carries every counted sample in a single row. The ceiling is +/// mirrored as `maximum` in `schemas/experiment.schema.json`. +pub const MAX_SAMPLES: u32 = 10_000; /// A declarative, typed experiment the runner can execute, journal, and replay. #[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] diff --git a/crates/control-plane/src/lib.rs b/crates/control-plane/src/lib.rs index 4248a77..5a66544 100644 --- a/crates/control-plane/src/lib.rs +++ b/crates/control-plane/src/lib.rs @@ -9,6 +9,13 @@ use serde::Serialize; use serde_json::{Value, json}; use thiserror::Error; +/// Inclusive ceiling the bounded alpha policy enforces on `mock.value`. +/// +/// The broker rejects any change above it. Callers that act on a value before +/// the lifecycle runs - a runner measuring a candidate, for instance - check it +/// against this constant so their envelope cannot drift from the policy's. +pub const MAX_MOCK_VALUE: u64 = 100; + /// Fail-closed errors from the broker seam. #[derive(Debug, Error)] pub enum ControlPlaneError { @@ -298,10 +305,10 @@ impl ControlPlane { .ok_or_else(|| { ControlPlaneError::PolicyDenied("mock.value requires an unsigned value".to_owned()) })?; - if value > 100 { - return Err(ControlPlaneError::PolicyDenied( - "mock.value is bounded to 0..=100".to_owned(), - )); + if value > MAX_MOCK_VALUE { + return Err(ControlPlaneError::PolicyDenied(format!( + "mock.value is bounded to 0..={MAX_MOCK_VALUE}" + ))); } Ok(()) } diff --git a/schemas/experiment.schema.json b/schemas/experiment.schema.json index 4998190..871d591 100644 --- a/schemas/experiment.schema.json +++ b/schemas/experiment.schema.json @@ -15,9 +15,9 @@ "properties": { "hypothesis": { "type": "string", "minLength": 1 }, "target": { "$ref": "#/$defs/change_request" }, - "warmup_samples": { "type": "integer", "minimum": 0, "maximum": 100000 }, - "baseline_samples": { "type": "integer", "minimum": 1, "maximum": 100000 }, - "candidate_samples": { "type": "integer", "minimum": 1, "maximum": 100000 }, + "warmup_samples": { "type": "integer", "minimum": 0, "maximum": 10000 }, + "baseline_samples": { "type": "integer", "minimum": 1, "maximum": 10000 }, + "candidate_samples": { "type": "integer", "minimum": 1, "maximum": 10000 }, "bounds": { "$ref": "#/$defs/decision_bounds" } }, "$defs": { From 12e66e6c22446baaca6612a881b0cd0d0c067f6c Mon Sep 17 00:00:00 2001 From: Jerry Xiao Date: Mon, 27 Jul 2026 07:32:01 +0000 Subject: [PATCH 04/16] no-mistakes(review): policy-owned decision bounds, append-only trials, metric-sample schema --- apps/experiment-runner/src/evaluator.rs | 59 ++++-- apps/experiment-runner/src/lib.rs | 1 - apps/experiment-runner/src/model.rs | 6 +- apps/experiment-runner/src/runner.rs | 212 +++++++++++++++----- apps/experiment-runner/tests/integration.rs | 60 ++++-- crates/contracts/src/lib.rs | 13 +- crates/control-plane/src/lib.rs | 82 ++++---- docs/ARCHITECTURE.md | 4 +- docs/adr/0002-alpha-experiment-journal.md | 4 +- schemas/metric-sample.schema.json | 14 ++ 10 files changed, 325 insertions(+), 130 deletions(-) create mode 100644 schemas/metric-sample.schema.json diff --git a/apps/experiment-runner/src/evaluator.rs b/apps/experiment-runner/src/evaluator.rs index b451342..1ef50ca 100644 --- a/apps/experiment-runner/src/evaluator.rs +++ b/apps/experiment-runner/src/evaluator.rs @@ -73,18 +73,29 @@ pub fn evaluate( /// Aggregates one measurement set deterministically. /// -/// The FPS mean is the arithmetic mean (zero for an empty set), while the -/// temperature and power fields report the worst (highest) observed value and -/// errors are summed. Counting the divisor as an `f64` avoids a lossy -/// integer-to-float cast without changing the result for realistic sample -/// counts. Samples may come from a journaled record this build did not write, -/// so the error total saturates rather than overflowing: a saturated total is -/// far above any ceiling and still rejects. +/// The FPS mean is the arithmetic mean, while the temperature and power fields +/// report the worst (highest) observed value and errors are summed. An empty +/// set reports zero throughout, having observed nothing. Counting the divisor +/// as an `f64` avoids a lossy integer-to-float cast without changing the result +/// for realistic sample counts. Samples may come from a journaled record this +/// build did not write, so the aggregates state only what the samples hold: the +/// error total saturates rather than overflowing, and the maxima fold from +/// negative infinity so an all-negative set reports its true worst observation +/// instead of a value the journal never recorded. fn summarize(samples: &[MetricSample]) -> MetricSummary { + if samples.is_empty() { + return MetricSummary { + samples: 0, + mean_fps: 0.0, + max_temperature_c: 0.0, + max_power_w: 0.0, + total_errors: 0, + }; + } let mut sum_fps = 0.0_f64; let mut divisor = 0.0_f64; - let mut max_temperature_c = 0.0_f64; - let mut max_power_w = 0.0_f64; + let mut max_temperature_c = f64::NEG_INFINITY; + let mut max_power_w = f64::NEG_INFINITY; let mut total_errors = 0_u64; for sample in samples { sum_fps += sample.fps; @@ -95,11 +106,7 @@ fn summarize(samples: &[MetricSample]) -> MetricSummary { } MetricSummary { samples: samples.len() as u64, - mean_fps: if samples.is_empty() { - 0.0 - } else { - sum_fps / divisor - }, + mean_fps: sum_fps / divisor, max_temperature_c, max_power_w, total_errors, @@ -218,6 +225,30 @@ mod tests { assert_eq!(verdict.reason, VerdictReason::ErrorsExceeded); } + #[test] + fn maxima_report_the_worst_observation_even_when_all_are_negative() { + // The summary is journaled verbatim in the verdict, so it must not + // claim a temperature or power draw the samples never held. + let baseline = samples(100.0, -40.0, -10.0, 0, 3); + let candidate = samples(140.0, -30.0, -5.0, 0, 3); + let verdict = evaluate(&baseline, &candidate, &bounds()); + assert!((verdict.baseline.max_temperature_c - -40.0).abs() < f64::EPSILON); + assert!((verdict.baseline.max_power_w - -10.0).abs() < f64::EPSILON); + assert!((verdict.candidate.max_temperature_c - -30.0).abs() < f64::EPSILON); + assert!((verdict.candidate.max_power_w - -5.0).abs() < f64::EPSILON); + } + + #[test] + fn an_empty_set_reports_zero_throughout() { + let verdict = evaluate(&[], &[], &bounds()); + assert_eq!(verdict.baseline.samples, 0); + assert!((verdict.baseline.mean_fps - 0.0).abs() < f64::EPSILON); + assert!((verdict.baseline.max_temperature_c - 0.0).abs() < f64::EPSILON); + assert!((verdict.baseline.max_power_w - 0.0).abs() < f64::EPSILON); + assert_eq!(verdict.decision, Decision::Reject); + assert_eq!(verdict.reason, VerdictReason::InsufficientSamples); + } + #[test] fn evaluation_is_deterministic_across_repeated_calls() { let baseline = samples(100.0, 60.0, 150.0, 0, 4); diff --git a/apps/experiment-runner/src/lib.rs b/apps/experiment-runner/src/lib.rs index 59121c0..cda510d 100644 --- a/apps/experiment-runner/src/lib.rs +++ b/apps/experiment-runner/src/lib.rs @@ -16,7 +16,6 @@ mod model; mod runner; pub use evaluator::evaluate; -pub use model::measure; pub use runner::{ LifecycleFailure, LifecycleOutcome, ReplayOutcome, RunnerError, StoredTrial, TRIAL_RECORD_VERSION, TrialRecord, replay_trial, run_trial, diff --git a/apps/experiment-runner/src/model.rs b/apps/experiment-runner/src/model.rs index 63d0de8..046bf55 100644 --- a/apps/experiment-runner/src/model.rs +++ b/apps/experiment-runner/src/model.rs @@ -37,8 +37,12 @@ const ERROR_ONSET: u64 = 90; /// the returned vector always holds exactly `counted` steady-state samples. The /// output depends only on `value`, `warmup`, and `counted`, never on wall-clock /// time or external state. +/// +/// This is crate-internal because the allocation it makes is only as bounded as +/// its arguments: the runner intersects every count with the spec ceiling before +/// it calls in. #[must_use] -pub fn measure(value: u64, warmup: u32, counted: u32) -> Vec { +pub(crate) fn measure(value: u64, warmup: u32, counted: u32) -> Vec { let setting = knob_to_setting(value); let warmup = u64::from(warmup); // Widening before the sum keeps the count exact for every `u32` pair, and diff --git a/apps/experiment-runner/src/runner.rs b/apps/experiment-runner/src/runner.rs index 2885ca6..96c33ba 100644 --- a/apps/experiment-runner/src/runner.rs +++ b/apps/experiment-runner/src/runner.rs @@ -7,15 +7,16 @@ //! [`replay_trial`] can re-evaluate it from the journal alone and confirm the //! recorded verdict without chat history or re-running the workload. //! -//! # Write-ahead trial records +//! # Append-only trial records //! -//! The record is journaled *before* the lifecycle runs, following the same -//! write-ahead principle the lifecycle journal uses (ADR 0002). A lifecycle -//! that fails after a promotion - a policy denial, a provider fault, or a -//! rollback that could not be verified - therefore still leaves a replayable -//! record of the measurements that authorized the apply; the failure is -//! amended onto that record as a [`LifecycleFailure`] and also returned to the -//! caller. +//! A trial is journaled exactly once, after the lifecycle it authorized has +//! finished, and the trial journal exposes no write that could rewrite that row +//! afterwards. A lifecycle that fails after a promotion - a policy denial, a +//! provider fault, or a rollback that could not be verified - is therefore +//! still recorded, as a [`LifecycleFailure`] carried by the same record as the +//! measurements that authorized the apply, and is also returned to the caller. +//! Crash safety for the window between the mutation and that record stays with +//! the lifecycle journal's write-ahead `apply-intent` stage (ADR 0002). //! //! # Keep-or-rollback in the safe alpha //! @@ -33,8 +34,13 @@ //! written, so swapping it in means moving the candidate measurement inside the //! apply/lease window and running the evaluator gate after it. -use fpsmaxxing_contracts::{Decision, ExperimentSpec, MAX_SAMPLES, MetricSample, Verdict}; -use fpsmaxxing_control_plane::{ControlPlane, ControlPlaneError, LifecycleResult, MAX_MOCK_VALUE}; +use fpsmaxxing_contracts::{ + Decision, DecisionBounds, ExperimentSpec, MAX_SAMPLES, MetricSample, Verdict, +}; +use fpsmaxxing_control_plane::{ + ControlPlane, ControlPlaneError, LifecycleResult, MAX_DECISION_ERRORS, MAX_DECISION_POWER_W, + MAX_DECISION_TEMPERATURE_C, MAX_MOCK_VALUE, +}; use serde::{Deserialize, Serialize}; use serde_json::Value; use thiserror::Error; @@ -126,9 +132,9 @@ impl From<&ControlPlaneError> for LifecycleFailure { /// and candidate samples, and the verdict the evaluator produced. /// /// On a [`Promote`](Decision::Promote) exactly one of `lifecycle` and -/// `lifecycle_error` is normally set. Both being absent means the amend that -/// follows the lifecycle never reached the journal, and the lifecycle journal's -/// stage records for that experiment are the authoritative account. +/// `lifecycle_error` is set, because the record is written once the lifecycle +/// has finished. On a [`Reject`](Decision::Reject) both are absent: no +/// lifecycle ran. #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] pub struct TrialRecord { /// Version of the record format; see [`TRIAL_RECORD_VERSION`]. @@ -186,23 +192,25 @@ impl ReplayOutcome { /// Validates the spec, measures the baseline from the provider's current /// state, measures the candidate from the spec target, evaluates the two, and /// runs the candidate through the broker lifecycle only on a -/// [`Promote`](Decision::Promote). The record is written to the durable trial -/// journal before the lifecycle runs and amended with its outcome afterwards, -/// so the trial is replayable whether or not the lifecycle succeeds. +/// [`Promote`](Decision::Promote). The record is appended to the durable trial +/// journal once, carrying whichever of the lifecycle outcome or the lifecycle +/// error the promotion produced, so the trial is replayable whether or not the +/// lifecycle succeeded. /// /// # Errors /// /// Returns an error if the spec is outside the bounded envelope, if the spec /// target or provider snapshot lacks an unsigned mock value, or if the broker /// or durable journal rejects an operation. A lifecycle error is returned only -/// after the trial record has been amended with it. +/// after the trial record carrying it has been journaled; if that write also +/// fails, the lifecycle error still takes precedence and the journal failure is +/// traced to stderr. pub fn run_trial( plane: &mut ControlPlane, spec: &ExperimentSpec, ) -> Result { - validate(spec)?; + let candidate_value = validate(spec)?; let baseline_value = baseline_value(plane)?; - let candidate_value = candidate_value(spec)?; let baseline_samples = model::measure( baseline_value, spec.warmup_samples, @@ -214,7 +222,11 @@ pub fn run_trial( spec.candidate_samples.get(), ); let verdict = evaluate(&baseline_samples, &candidate_samples, &spec.bounds); - let mut record = TrialRecord { + let outcome = match verdict.decision { + Decision::Promote => Some(plane.run_lifecycle(&spec.target)), + Decision::Reject => None, + }; + let record = TrialRecord { schema_version: TRIAL_RECORD_VERSION, spec: spec.clone(), baseline_value, @@ -222,29 +234,28 @@ pub fn run_trial( baseline_samples, candidate_samples, verdict, - lifecycle: None, - lifecycle_error: None, + lifecycle: outcome + .as_ref() + .and_then(|outcome| outcome.as_ref().ok()) + .map(LifecycleOutcome::from), + lifecycle_error: outcome + .as_ref() + .and_then(|outcome| outcome.as_ref().err()) + .map(LifecycleFailure::from), }; - let id = plane.record_trial(&record)?; - match record.verdict.decision { - Decision::Promote => match plane.run_lifecycle(&spec.target) { - Ok(result) => { - record.lifecycle = Some(LifecycleOutcome::from(&result)); - plane.amend_trial(id, &record)?; - } - Err(error) => { - record.lifecycle_error = Some(LifecycleFailure::from(&error)); - if let Err(journal_error) = plane.amend_trial(id, &record) { - eprintln!( - "fpsmaxxing-experiment-runner: could not amend trial {id} with its lifecycle failure: {journal_error}" - ); - } - return Err(error.into()); - } - }, - Decision::Reject => {} + let journaled = plane.record_trial(&record); + if let Some(Err(error)) = outcome { + if let Err(journal_error) = journaled { + eprintln!( + "fpsmaxxing-experiment-runner: could not journal the trial whose lifecycle failed: {journal_error}" + ); + } + return Err(error.into()); } - Ok(StoredTrial { id, record }) + Ok(StoredTrial { + id: journaled?, + record, + }) } /// Re-evaluates a journaled trial from the journal alone. @@ -285,21 +296,26 @@ pub fn replay_trial(plane: &ControlPlane, id: i64) -> Result Result<(), RunnerError> { +/// Everything the measurement phase and the decision gate consume arrives over +/// the wire, so all of it is bounded before any measurement work runs: sample +/// counts size the measurement buffers and are checked against [`MAX_SAMPLES`], +/// the decision bounds are intersected with the policy envelope by +/// [`validate_bounds`], and the candidate knob value drives the modeled metrics +/// and is checked against [`MAX_MOCK_VALUE`], the same ceiling the broker policy +/// enforces later in the lifecycle. A spec that asks for fewer counted samples +/// than its own bounds require can never promote, so it is refused up front +/// rather than measured and then rejected. The validated value is returned so +/// the measurement uses exactly what was bounded here. +fn validate(spec: &ExperimentSpec) -> Result { let min_samples = spec.bounds.min_samples.get(); for (label, count) in [ ("warmup_samples", spec.warmup_samples), ("baseline_samples", spec.baseline_samples.get()), ("candidate_samples", spec.candidate_samples.get()), + ("min_samples", min_samples), ] { if count > MAX_SAMPLES { return Err(RunnerError::InvalidSpec(format!( @@ -317,12 +333,53 @@ fn validate(spec: &ExperimentSpec) -> Result<(), RunnerError> { ))); } } + validate_bounds(&spec.bounds)?; let value = candidate_value(spec)?; if value > MAX_MOCK_VALUE { return Err(RunnerError::InvalidSpec(format!( "target value is {value}, above the {MAX_MOCK_VALUE} the policy allows" ))); } + Ok(value) +} + +/// Rejects decision bounds that are looser than the policy envelope. +/// +/// The evaluator is immutable, but its thresholds arrive in the spec, so a spec +/// could otherwise disarm the gate it is supposed to pass by declaring a +/// ceiling nothing can exceed. Each threshold is intersected with the +/// policy-owned envelope: a spec may tighten a bound but never loosen it past +/// [`MAX_DECISION_TEMPERATURE_C`], [`MAX_DECISION_POWER_W`], or +/// [`MAX_DECISION_ERRORS`], and a required improvement must be a finite, +/// non-negative gain. Non-finite thresholds are refused outright, because a +/// `NaN` compares false against every ceiling and would silently pass the gate. +fn validate_bounds(bounds: &DecisionBounds) -> Result<(), RunnerError> { + for (label, bound, ceiling) in [ + ( + "max_temperature_c", + bounds.max_temperature_c, + MAX_DECISION_TEMPERATURE_C, + ), + ("max_power_w", bounds.max_power_w, MAX_DECISION_POWER_W), + ] { + if !bound.is_finite() || bound <= 0.0 || bound > ceiling { + return Err(RunnerError::InvalidSpec(format!( + "{label} is {bound}, outside the 0 exclusive to {ceiling} inclusive the policy allows" + ))); + } + } + let improvement = bounds.min_fps_improvement; + if !improvement.is_finite() || improvement < 0.0 { + return Err(RunnerError::InvalidSpec(format!( + "min_fps_improvement is {improvement}, but the policy requires a finite gain of at least 0" + ))); + } + if bounds.max_errors > MAX_DECISION_ERRORS { + return Err(RunnerError::InvalidSpec(format!( + "max_errors is {}, above the {MAX_DECISION_ERRORS} the policy allows", + bounds.max_errors + ))); + } Ok(()) } @@ -352,7 +409,10 @@ mod tests { use fpsmaxxing_contracts::{ChangeRequest, DecisionBounds}; use serde_json::json; - use super::{ExperimentSpec, MAX_MOCK_VALUE, MAX_SAMPLES, RunnerError, validate}; + use super::{ + ExperimentSpec, MAX_DECISION_ERRORS, MAX_DECISION_POWER_W, MAX_DECISION_TEMPERATURE_C, + MAX_MOCK_VALUE, MAX_SAMPLES, RunnerError, validate, + }; fn spec(warmup: u32, baseline: u32, candidate: u32, min_samples: u32) -> ExperimentSpec { spec_for_value(warmup, baseline, candidate, min_samples, 40) @@ -394,10 +454,58 @@ mod tests { #[test] fn accepts_a_spec_inside_the_envelope() { - assert!(validate(&spec(2, 5, 5, 3)).is_ok()); + assert_eq!( + validate(&spec(2, 5, 5, 3)).expect("spec is in envelope"), + 40 + ); assert!(validate(&spec(MAX_SAMPLES, MAX_SAMPLES, MAX_SAMPLES, 3)).is_ok()); } + #[test] + fn rejects_bounds_looser_than_the_policy_envelope() { + // The gate's own thresholds arrive in the spec, so a spec that declares + // ceilings nothing can exceed would promote unconditionally. + let mut loosened = spec(2, 5, 5, 3); + loosened.bounds.max_temperature_c = MAX_DECISION_TEMPERATURE_C + 0.1; + assert!(rejection(&loosened).contains("max_temperature_c")); + loosened.bounds.max_temperature_c = f64::INFINITY; + assert!(rejection(&loosened).contains("max_temperature_c")); + loosened.bounds.max_temperature_c = f64::NAN; + assert!(rejection(&loosened).contains("max_temperature_c")); + loosened.bounds.max_temperature_c = 0.0; + assert!(rejection(&loosened).contains("max_temperature_c")); + + let mut loosened = spec(2, 5, 5, 3); + loosened.bounds.max_power_w = MAX_DECISION_POWER_W + 0.1; + assert!(rejection(&loosened).contains("max_power_w")); + + let mut loosened = spec(2, 5, 5, 3); + loosened.bounds.min_fps_improvement = -1.0; + assert!(rejection(&loosened).contains("min_fps_improvement")); + loosened.bounds.min_fps_improvement = f64::NAN; + assert!(rejection(&loosened).contains("min_fps_improvement")); + + let mut loosened = spec(2, 5, 5, 3); + loosened.bounds.max_errors = MAX_DECISION_ERRORS + 1; + assert!(rejection(&loosened).contains("max_errors")); + + let mut loosened = spec(2, 5, 5, 3); + loosened.bounds.min_samples = + NonZeroU32::new(MAX_SAMPLES + 1).expect("min samples is non-zero"); + assert!(rejection(&loosened).contains("min_samples")); + } + + #[test] + fn accepts_bounds_exactly_at_the_policy_envelope() { + // The envelope is inclusive, and a spec is free to tighten within it. + let mut tightest = spec(2, 5, 5, 3); + tightest.bounds.max_temperature_c = MAX_DECISION_TEMPERATURE_C; + tightest.bounds.max_power_w = MAX_DECISION_POWER_W; + tightest.bounds.max_errors = MAX_DECISION_ERRORS; + tightest.bounds.min_fps_improvement = 0.0; + assert!(validate(&tightest).is_ok()); + } + #[test] fn rejects_sample_counts_above_the_ceiling() { // Materializing this many samples would abort the process, so the spec diff --git a/apps/experiment-runner/tests/integration.rs b/apps/experiment-runner/tests/integration.rs index 832fd30..5ff8869 100644 --- a/apps/experiment-runner/tests/integration.rs +++ b/apps/experiment-runner/tests/integration.rs @@ -4,9 +4,9 @@ //! the mock provider through the broker: a promoted experiment that runs the //! full lifecycle, a rejected experiment that is never applied and leaves the //! baseline untouched, a promoted experiment whose lifecycle the broker refuses -//! but whose write-ahead record survives, and a replay from the durable journal -//! alone - reopened as a fresh handle - that reproduces the recorded verdict -//! exactly. The final test states the MVP acceptance criterion directly. +//! but which is still journaled, and a replay from the durable journal alone - +//! reopened as a fresh handle - that reproduces the recorded verdict exactly. +//! The final test states the MVP acceptance criterion directly. use std::num::{NonZeroU32, NonZeroU64}; @@ -78,6 +78,7 @@ fn a_promoted_experiment_runs_the_full_lifecycle() { let lifecycle = trial .record .lifecycle + .clone() .expect("a promoted trial records a lifecycle"); assert_eq!(lifecycle.provider_id, "mock"); assert!( @@ -85,6 +86,23 @@ fn a_promoted_experiment_runs_the_full_lifecycle() { "candidate value must verify after apply" ); assert!(lifecycle.rolled_back, "leased change must be rolled back"); + assert!( + trial.record.lifecycle_error.is_none(), + "a completed lifecycle records no failure" + ); + + // The trial journal is append-only: the outcome is carried by the single + // row the trial inserted, which is exactly what replay reads back. + assert_eq!(plane.trial_ids().expect("trial ids"), [trial.id]); + let journaled: TrialRecord = + serde_json::from_value(plane.read_trial(trial.id).expect("trial should read")) + .expect("trial should decode"); + assert_eq!(journaled, trial.record); + assert!( + replay_trial(&plane, trial.id) + .expect("replay trial") + .is_consistent() + ); // The leased lifecycle restores the pre-state, so the provider is left at // its baseline value even after a promotion. @@ -130,10 +148,14 @@ fn a_promoted_trial_survives_a_lifecycle_the_broker_refuses() { RunnerError::ControlPlane(ControlPlaneError::PolicyDenied(_)) )); - // The measurements that authorized the promotion were journaled ahead of - // the lifecycle, so the trial is still discoverable and replayable. + // The measurements that authorized the promotion are journaled with the + // refusal, in one append, so the trial is still discoverable and replayable. let ids = plane.trial_ids().expect("trial ids"); - assert_eq!(ids.len(), 1, "the failed promotion is still journaled"); + assert_eq!( + ids.len(), + 1, + "the failed promotion is journaled exactly once" + ); let record: TrialRecord = serde_json::from_value(plane.read_trial(ids[0]).expect("trial should read")) .expect("trial should decode"); @@ -228,28 +250,34 @@ fn a_trial_record_from_an_unsupported_version_fails_closed() { ControlPlane::open(Box::new(MockProvider::new(10)), journal.path()).expect("open"); let trial = run_trial(&mut plane, &spec_for(40, 5.0, 80.0)).expect("run trial"); - // Stand in for a record a future runner wrote: the reader must refuse it + // Stand in for a record a future runner appended: the reader must refuse it // rather than decode it under this version's field meanings. let mut payload = plane.read_trial(trial.id).expect("trial should read"); payload["schema_version"] = json!(2); - plane - .amend_trial(trial.id, &payload) - .expect("trial should amend"); - - let error = replay_trial(&plane, trial.id).expect_err("a future record should be refused"); + let future = plane + .record_trial(&payload) + .expect("a future record should append"); + let error = replay_trial(&plane, future).expect_err("a future record should be refused"); assert!(matches!(error, RunnerError::UnsupportedRecordVersion(2))); // A future record that also reshaped a field must still report the version // that wrote it, not an opaque decode error about the reshaped field. payload["verdict"] = json!("promote"); - plane - .amend_trial(trial.id, &payload) - .expect("trial should amend"); - let error = replay_trial(&plane, trial.id).expect_err("a future record should be refused"); + let reshaped = plane + .record_trial(&payload) + .expect("a future record should append"); + let error = replay_trial(&plane, reshaped).expect_err("a future record should be refused"); assert!( matches!(error, RunnerError::UnsupportedRecordVersion(2)), "expected a version error, got {error:?}" ); + + // Appending those rows left the original trial exactly as it was recorded. + assert!( + replay_trial(&plane, trial.id) + .expect("replay trial") + .is_consistent() + ); } #[test] diff --git a/crates/contracts/src/lib.rs b/crates/contracts/src/lib.rs index 1ff194d..c94225f 100644 --- a/crates/contracts/src/lib.rs +++ b/crates/contracts/src/lib.rs @@ -224,14 +224,15 @@ mod tests { use super::{ CapabilityDescriptor, ChangeRequest, Decision, DecisionBounds, ExperimentSpec, MAX_SAMPLES, - MetricSummary, NonZeroU32, NonZeroU64, Persistence, ProviderManifest, RiskClass, Verdict, - VerdictReason, + MetricSample, MetricSummary, NonZeroU32, NonZeroU64, Persistence, ProviderManifest, + RiskClass, Verdict, VerdictReason, }; const CAPABILITY_SCHEMA: &str = include_str!("../../../schemas/capability.schema.json"); const SIDECAR_SCHEMA: &str = include_str!("../../../schemas/sidecar.schema.json"); const EXPERIMENT_SCHEMA: &str = include_str!("../../../schemas/experiment.schema.json"); const VERDICT_SCHEMA: &str = include_str!("../../../schemas/verdict.schema.json"); + const METRIC_SAMPLE_SCHEMA: &str = include_str!("../../../schemas/metric-sample.schema.json"); fn wire_string(value: impl serde::Serialize) -> String { serde_json::to_value(value) @@ -499,6 +500,14 @@ mod tests { .expect("verdict schema should parse"), definitions: &[("MetricSummary", Some("metric_summary"))], }, + SchemaCase { + label: "MetricSample", + generated: serde_json::to_value(schemars::schema_for!(MetricSample)) + .expect("generated schema should serialize"), + checked_in: serde_json::from_str(METRIC_SAMPLE_SCHEMA) + .expect("metric sample schema should parse"), + definitions: &[], + }, ] } diff --git a/crates/control-plane/src/lib.rs b/crates/control-plane/src/lib.rs index 5a66544..8aba281 100644 --- a/crates/control-plane/src/lib.rs +++ b/crates/control-plane/src/lib.rs @@ -11,11 +11,36 @@ use thiserror::Error; /// Inclusive ceiling the bounded alpha policy enforces on `mock.value`. /// -/// The broker rejects any change above it. Callers that act on a value before -/// the lifecycle runs - a runner measuring a candidate, for instance - check it -/// against this constant so their envelope cannot drift from the policy's. +/// [`ControlPlane::run_lifecycle`] rejects any change above it. It is exported +/// so the experiment runner, which acts on a candidate value before the +/// lifecycle runs, can refuse the same values this policy would. pub const MAX_MOCK_VALUE: u64 = 100; +/// Inclusive ceiling the bounded alpha policy enforces on a trial's temperature +/// bound, in degrees Celsius. +/// +/// The thresholds an immutable evaluator applies arrive in an LLM-authored +/// experiment spec, so a spec could otherwise disarm its own safety gate by +/// declaring an unreachable ceiling. Policy owns the hard envelope: a spec may +/// tighten these bounds but never loosen them. This value sits below the +/// throttle point of the consumer hardware the alpha targets. +pub const MAX_DECISION_TEMPERATURE_C: f64 = 90.0; + +/// Inclusive ceiling the bounded alpha policy enforces on a trial's power +/// bound, in watts. +/// +/// See [`MAX_DECISION_TEMPERATURE_C`] for why the envelope is policy-owned. +pub const MAX_DECISION_POWER_W: f64 = 250.0; + +/// Inclusive ceiling the bounded alpha policy enforces on a trial's error +/// budget. +/// +/// A correctness fault is never an acceptable cost of a performance gain on +/// this path, so the alpha promotes nothing that reported one; a spec may +/// restate this ceiling but not raise it. See [`MAX_DECISION_TEMPERATURE_C`] +/// for why the envelope is policy-owned. +pub const MAX_DECISION_ERRORS: u64 = 0; + /// Fail-closed errors from the broker seam. #[derive(Debug, Error)] pub enum ControlPlaneError { @@ -214,6 +239,10 @@ impl ControlPlane { /// recorded samples, and verdict together so the trial can be replayed and /// re-evaluated from the journal alone, without chat history. /// + /// This is the only write the trial journal exposes, so the table is + /// append-only like the lifecycle journal: a recorded trial has no API that + /// can rewrite its verdict. + /// /// # Errors /// /// Returns an error if the payload cannot be encoded or the durable journal @@ -227,31 +256,6 @@ impl ControlPlane { Ok(self.journal.last_insert_rowid()) } - /// Replaces the payload of an already-recorded trial. - /// - /// A runner journals its measured trial ahead of the broker lifecycle so - /// the measurements that authorized a promotion survive a lifecycle - /// failure, then amends the same row with the outcome it could not know - /// yet. The trial's identity, spec, samples, and verdict are fixed at - /// insertion; only the outcome fields are filled in. - /// - /// # Errors - /// - /// Returns an error if no trial has the identifier, the payload cannot be - /// encoded, or the durable journal cannot be written. - pub fn amend_trial(&self, id: i64, payload: &impl Serialize) -> Result<(), ControlPlaneError> { - let updated = self.journal.execute( - "UPDATE experiment_trials SET payload = ?2 WHERE id = ?1", - params![id, serde_json::to_string(payload)?], - )?; - if updated == 0 { - return Err(ControlPlaneError::Journal( - rusqlite::Error::QueryReturnedNoRows, - )); - } - Ok(()) - } - /// Reads one trial record by identifier for replay and re-evaluation. /// /// # Errors @@ -893,23 +897,19 @@ mod tests { } #[test] - fn amending_a_trial_replaces_only_that_payload() { + fn recording_a_trial_never_disturbs_an_earlier_one() { + // The trial journal exposes no update, so a recorded verdict stays as + // it was written and later trials only ever append. let plane = plane(false); let first = plane - .record_trial(&json!({ "decision": "promote", "lifecycle": null })) + .record_trial(&json!({ "decision": "promote", "lifecycle": { "verified": true } })) .expect("first trial should record"); let second = plane .record_trial(&json!({ "decision": "reject" })) .expect("second trial should record"); - plane - .amend_trial( - first, - &json!({ "decision": "promote", "lifecycle": { "verified": true } }), - ) - .expect("trial should amend"); assert_eq!( - plane.read_trial(first).expect("trial should read")["lifecycle"]["verified"], - true + plane.read_trial(first).expect("trial should read"), + json!({ "decision": "promote", "lifecycle": { "verified": true } }) ); assert_eq!( plane.read_trial(second).expect("trial should read")["decision"], @@ -919,11 +919,11 @@ mod tests { } #[test] - fn amending_an_unknown_trial_fails_closed() { + fn reading_an_unknown_trial_fails_closed() { let plane = plane(false); let error = plane - .amend_trial(404, &json!({ "decision": "promote" })) - .expect_err("an unrecorded trial cannot be amended"); + .read_trial(404) + .expect_err("an unrecorded trial cannot be read"); assert!(matches!( error, ControlPlaneError::Journal(rusqlite::Error::QueryReturnedNoRows) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 565d6b4..42a6dd4 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -31,8 +31,8 @@ A steady-state poll reclaims only expired leases, while a crash-recovery pass (` The runner controls workload setup, warmup, repeated measurements, cooldown, correctness checks, and promotion decisions. Evaluator code is outside the LLM's writable surface. In the safe alpha (`apps/experiment-runner`) the runner measures a baseline and a candidate against a deterministic model, then a pure immutable evaluator returns a promote or reject verdict from the recorded samples and fixed bounds alone - no clock, no LLM, no I/O. Every trial is journaled as a self-describing, versioned record so it can be replayed and re-evaluated from the journal without the original conversation. -The record is written ahead of the broker lifecycle and amended with its outcome, so a promotion the broker refuses still leaves the measurements that authorized it. -Sample counts are bounded by the spec schema and rechecked by the runner before any measurement runs. Because mock capabilities are leased and the broker lifecycle always rolls back, the verdict gates whether the candidate is applied at all rather than whether it persists; durable keep-or-rollback awaits the privileged broker. +The record is appended once, after the broker lifecycle it authorized has finished and carrying either that lifecycle's outcome or the error the broker returned, so a promotion the broker refuses still leaves the measurements that authorized it and no API can rewrite a recorded verdict. +Sample counts are bounded by the spec schema and rechecked by the runner before any measurement runs, and the spec's decision bounds are intersected with a policy-owned envelope so a spec can tighten its own safety gate but never loosen it. Because mock capabilities are leased and the broker lifecycle always rolls back, the verdict gates whether the candidate is applied at all rather than whether it persists; durable keep-or-rollback awaits the privileged broker. ### Provider sidecars diff --git a/docs/adr/0002-alpha-experiment-journal.md b/docs/adr/0002-alpha-experiment-journal.md index c246b9b..28d0dac 100644 --- a/docs/adr/0002-alpha-experiment-journal.md +++ b/docs/adr/0002-alpha-experiment-journal.md @@ -14,7 +14,9 @@ The same journal database also keeps an `experiment_trials` table of self-descri This supersedes the original deferral of a dedicated experiments table. A trial record is a different kind of row from a lifecycle stage record: it holds the spec, the recorded baseline and candidate samples, and the immutable evaluator's verdict, so a trial re-evaluates from the journal alone without chat history or a re-run workload. Storing that under the stage schema would have meant either overloading `payload` with a shape `doctor` cannot interpret or restructuring the stage table, so a second table was the smaller change. -Trial rows follow the same write-ahead principle as the lifecycle journal: the runner records the measured trial before invoking the lifecycle and amends that row with the outcome, so a promotion the broker refuses still leaves the measurements that authorized it. +Trial rows are append-only: the journal exposes an insert and no update, and the runner writes each trial exactly once, after the lifecycle it authorized has finished, carrying either that lifecycle's outcome or the error the broker returned. +A promotion the broker refuses therefore still leaves the measurements that authorized it, and a recorded verdict has no API that can rewrite it. +Crash safety for the window between a mutation and that record stays with the lifecycle journal's write-ahead `apply-intent` record rather than being duplicated in the trial table. ## Rationale diff --git a/schemas/metric-sample.schema.json b/schemas/metric-sample.schema.json new file mode 100644 index 0000000..a78231f --- /dev/null +++ b/schemas/metric-sample.schema.json @@ -0,0 +1,14 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/undeemed/fpsmaxxing/schemas/metric-sample.schema.json", + "title": "FPSMaxxing metric sample", + "type": "object", + "additionalProperties": false, + "required": ["fps", "temperature_c", "power_w", "errors"], + "properties": { + "fps": { "type": "number" }, + "temperature_c": { "type": "number" }, + "power_w": { "type": "number" }, + "errors": { "type": "integer", "minimum": 0 } + } +} From eedd8dbe47909cfc8014905ce28d12a04097c60c Mon Sep 17 00:00:00 2001 From: Jerry Xiao Date: Mon, 27 Jul 2026 08:04:07 +0000 Subject: [PATCH 05/16] no-mistakes(review): fail closed on unknown capabilities, fields, trials, and replayed bounds --- apps/experiment-runner/src/main.rs | 18 ++- apps/experiment-runner/src/runner.rs | 142 +++++++++++++++----- apps/experiment-runner/tests/integration.rs | 103 ++++++++++++++ crates/contracts/src/lib.rs | 85 +++++++++++- crates/control-plane/src/lib.rs | 60 ++++----- docs/ARCHITECTURE.md | 2 +- docs/adr/0002-alpha-experiment-journal.md | 2 + schemas/experiment.schema.json | 18 ++- 8 files changed, 347 insertions(+), 83 deletions(-) diff --git a/apps/experiment-runner/src/main.rs b/apps/experiment-runner/src/main.rs index 013e752..8610a61 100644 --- a/apps/experiment-runner/src/main.rs +++ b/apps/experiment-runner/src/main.rs @@ -6,8 +6,10 @@ //! in-memory `SQLite` database, so the demo leaves nothing on disk. //! //! A replay that diverges from the journal means the record was tampered with -//! or the immutable evaluator drifted, so the demo exits non-zero rather than -//! reporting the divergence as a successful run. +//! or the immutable evaluator drifted, and a replay whose journaled bounds sit +//! outside the policy envelope means the verdict was decided under thresholds +//! policy never allowed. The demo exits non-zero on either rather than +//! reporting it as a successful run. use std::{ num::{NonZeroU32, NonZeroU64}, @@ -39,10 +41,11 @@ fn main() -> Result { let replay = replay_trial(&plane, trial.id)?; println!( - "replay {} -> recomputed {:?}; consistent with journal = {}", + "replay {} -> recomputed {:?}; consistent with journal = {}, policy legal = {}", replay.trial_id, replay.recomputed.decision, - replay.is_consistent() + replay.is_consistent(), + replay.policy_legal ); if !replay.is_consistent() { eprintln!( @@ -51,6 +54,13 @@ fn main() -> Result { ); return Ok(ExitCode::FAILURE); } + if !replay.policy_legal { + eprintln!( + "fpsmaxxing-experiment-runner: trial {} was decided under bounds outside the policy envelope", + replay.trial_id + ); + return Ok(ExitCode::FAILURE); + } Ok(ExitCode::SUCCESS) } diff --git a/apps/experiment-runner/src/runner.rs b/apps/experiment-runner/src/runner.rs index 96c33ba..7470321 100644 --- a/apps/experiment-runner/src/runner.rs +++ b/apps/experiment-runner/src/runner.rs @@ -35,12 +35,10 @@ //! apply/lease window and running the evaluator gate after it. use fpsmaxxing_contracts::{ - Decision, DecisionBounds, ExperimentSpec, MAX_SAMPLES, MetricSample, Verdict, -}; -use fpsmaxxing_control_plane::{ - ControlPlane, ControlPlaneError, LifecycleResult, MAX_DECISION_ERRORS, MAX_DECISION_POWER_W, - MAX_DECISION_TEMPERATURE_C, MAX_MOCK_VALUE, + Decision, DecisionBounds, ExperimentSpec, MAX_DECISION_ERRORS, MAX_DECISION_POWER_W, + MAX_DECISION_TEMPERATURE_C, MAX_SAMPLES, MetricSample, ProviderManifest, Verdict, }; +use fpsmaxxing_control_plane::{ControlPlane, ControlPlaneError, LifecycleResult, MAX_MOCK_VALUE}; use serde::{Deserialize, Serialize}; use serde_json::Value; use thiserror::Error; @@ -81,6 +79,7 @@ pub enum RunnerError { /// [`LifecycleResult`] is serialize-only; this record round-trips so a promoted /// trial's lifecycle outcome can be read back during replay and audit. #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] pub struct LifecycleOutcome { /// Provider that owned the change. pub provider_id: String, @@ -109,6 +108,7 @@ impl From<&LifecycleResult> for LifecycleOutcome { /// `failed` record, so a promotion the broker refused is auditable from the /// trial row alone. #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] pub struct LifecycleFailure { /// Stable machine-readable error kind reported by the broker. pub kind: String, @@ -136,6 +136,7 @@ impl From<&ControlPlaneError> for LifecycleFailure { /// has finished. On a [`Reject`](Decision::Reject) both are absent: no /// lifecycle ran. #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] pub struct TrialRecord { /// Version of the record format; see [`TRIAL_RECORD_VERSION`]. pub schema_version: u32, @@ -169,7 +170,13 @@ pub struct StoredTrial { } /// The result of re-evaluating a journaled trial. +/// +/// A replay is a tamper-detection read, so the outcome must be inspected: +/// [`is_consistent`](Self::is_consistent) reports whether the recomputation +/// agrees with the journal, and `policy_legal` reports whether the bounds the +/// journaled verdict was decided under are still inside the policy envelope. #[derive(Clone, Debug)] +#[must_use] pub struct ReplayOutcome { /// The trial-journal identifier that was replayed. pub trial_id: i64, @@ -177,6 +184,8 @@ pub struct ReplayOutcome { pub recorded: Verdict, /// The verdict recomputed from the journaled samples and bounds. pub recomputed: Verdict, + /// Whether the journaled decision bounds are inside the policy envelope. + pub policy_legal: bool, } impl ReplayOutcome { @@ -199,7 +208,8 @@ impl ReplayOutcome { /// /// # Errors /// -/// Returns an error if the spec is outside the bounded envelope, if the spec +/// Returns an error if the spec is outside the bounded envelope, if its target +/// names a capability the accepted provider does not advertise, if the spec /// target or provider snapshot lacks an unsigned mock value, or if the broker /// or durable journal rejects an operation. A lifecycle error is returned only /// after the trial record carrying it has been journaled; if that write also @@ -209,7 +219,7 @@ pub fn run_trial( plane: &mut ControlPlane, spec: &ExperimentSpec, ) -> Result { - let candidate_value = validate(spec)?; + let candidate_value = validate(plane.capabilities(), spec)?; let baseline_value = baseline_value(plane)?; let baseline_samples = model::measure( baseline_value, @@ -265,6 +275,11 @@ pub fn run_trial( /// and recomputed verdicts for comparison. It consults no chat history and /// re-runs no workload. /// +/// The journaled bounds are re-checked against the policy envelope as well, so +/// a row whose thresholds were widened after the fact is reported as +/// `policy_legal = false` even when re-evaluating under those same widened +/// thresholds reproduces the recorded verdict. +/// /// # Errors /// /// Returns an error if the trial cannot be read from the durable journal, its @@ -293,6 +308,7 @@ pub fn replay_trial(plane: &ControlPlane, id: i64) -> Result Result Result { +/// the wire, so all of it is bounded before any measurement work runs: the +/// target must name a capability the accepted provider actually advertises, +/// sample counts size the measurement buffers and are checked against +/// [`MAX_SAMPLES`], the decision bounds are intersected with the policy envelope +/// by [`validate_bounds`], and the candidate knob value drives the modeled +/// metrics and is checked against [`MAX_MOCK_VALUE`], the same ceiling the +/// broker policy enforces later in the lifecycle. A spec that asks for fewer +/// counted samples than its own bounds require can never promote, so it is +/// refused up front rather than measured and then rejected. The validated value +/// is returned so the measurement uses exactly what was bounded here. +/// +/// The capability check is what keeps unknown hardware failing closed. The +/// broker refuses an unadvertised capability too, but only inside the +/// lifecycle, which a rejected trial never reaches - and by then a measurement +/// the model never described would already have been journaled as an +/// authoritative trial. +fn validate(manifest: &ProviderManifest, spec: &ExperimentSpec) -> Result { + if !manifest + .capabilities + .iter() + .any(|capability| capability.id == spec.target.capability_id) + { + return Err(ControlPlaneError::UnknownCapability(spec.target.capability_id.clone()).into()); + } + validate_bounds(&spec.bounds)?; let min_samples = spec.bounds.min_samples.get(); for (label, count) in [ ("warmup_samples", spec.warmup_samples), ("baseline_samples", spec.baseline_samples.get()), ("candidate_samples", spec.candidate_samples.get()), - ("min_samples", min_samples), ] { if count > MAX_SAMPLES { return Err(RunnerError::InvalidSpec(format!( @@ -333,7 +363,6 @@ fn validate(spec: &ExperimentSpec) -> Result { ))); } } - validate_bounds(&spec.bounds)?; let value = candidate_value(spec)?; if value > MAX_MOCK_VALUE { return Err(RunnerError::InvalidSpec(format!( @@ -349,11 +378,22 @@ fn validate(spec: &ExperimentSpec) -> Result { /// could otherwise disarm the gate it is supposed to pass by declaring a /// ceiling nothing can exceed. Each threshold is intersected with the /// policy-owned envelope: a spec may tighten a bound but never loosen it past -/// [`MAX_DECISION_TEMPERATURE_C`], [`MAX_DECISION_POWER_W`], or -/// [`MAX_DECISION_ERRORS`], and a required improvement must be a finite, -/// non-negative gain. Non-finite thresholds are refused outright, because a -/// `NaN` compares false against every ceiling and would silently pass the gate. +/// [`MAX_DECISION_TEMPERATURE_C`], [`MAX_DECISION_POWER_W`], +/// [`MAX_DECISION_ERRORS`], or [`MAX_SAMPLES`], and a required improvement must +/// be a finite, non-negative gain. Non-finite thresholds are refused outright, +/// because a `NaN` compares false against every ceiling and would silently pass +/// the gate. +/// +/// This covers every field of [`DecisionBounds`], so [`replay_trial`] can re-run +/// it over a journaled spec to decide whether a recorded verdict was reached +/// under thresholds the policy ever allowed. fn validate_bounds(bounds: &DecisionBounds) -> Result<(), RunnerError> { + let min_samples = bounds.min_samples.get(); + if min_samples > MAX_SAMPLES { + return Err(RunnerError::InvalidSpec(format!( + "min_samples is {min_samples}, above the {MAX_SAMPLES} ceiling" + ))); + } for (label, bound, ceiling) in [ ( "max_temperature_c", @@ -406,14 +446,33 @@ fn candidate_value(spec: &ExperimentSpec) -> Result { mod tests { use std::num::{NonZeroU32, NonZeroU64}; - use fpsmaxxing_contracts::{ChangeRequest, DecisionBounds}; + use fpsmaxxing_contracts::{ + CapabilityDescriptor, ChangeRequest, DecisionBounds, Persistence, RiskClass, + }; + use fpsmaxxing_control_plane::ControlPlaneError; use serde_json::json; use super::{ ExperimentSpec, MAX_DECISION_ERRORS, MAX_DECISION_POWER_W, MAX_DECISION_TEMPERATURE_C, - MAX_MOCK_VALUE, MAX_SAMPLES, RunnerError, validate, + MAX_MOCK_VALUE, MAX_SAMPLES, ProviderManifest, RunnerError, validate, }; + /// A manifest advertising only the knob the measurement model describes. + fn manifest() -> ProviderManifest { + ProviderManifest { + id: "mock".to_owned(), + protocol_version: NonZeroU32::MIN, + targets: vec![std::env::consts::OS.to_owned()], + capabilities: vec![CapabilityDescriptor { + id: "mock.value".to_owned(), + description: "Sets an in-memory value".to_owned(), + risk: RiskClass::Reversible, + persistence: Persistence::Leased, + input_schema: json!({ "type": "object" }), + }], + } + } + fn spec(warmup: u32, baseline: u32, candidate: u32, min_samples: u32) -> ExperimentSpec { spec_for_value(warmup, baseline, candidate, min_samples, 40) } @@ -446,7 +505,7 @@ mod tests { } fn rejection(spec: &ExperimentSpec) -> String { - match validate(spec) { + match validate(&manifest(), spec) { Err(RunnerError::InvalidSpec(message)) => message, other => panic!("spec should be rejected, got {other:?}"), } @@ -455,10 +514,10 @@ mod tests { #[test] fn accepts_a_spec_inside_the_envelope() { assert_eq!( - validate(&spec(2, 5, 5, 3)).expect("spec is in envelope"), + validate(&manifest(), &spec(2, 5, 5, 3)).expect("spec is in envelope"), 40 ); - assert!(validate(&spec(MAX_SAMPLES, MAX_SAMPLES, MAX_SAMPLES, 3)).is_ok()); + assert!(validate(&manifest(), &spec(MAX_SAMPLES, MAX_SAMPLES, MAX_SAMPLES, 3)).is_ok()); } #[test] @@ -503,7 +562,7 @@ mod tests { tightest.bounds.max_power_w = MAX_DECISION_POWER_W; tightest.bounds.max_errors = MAX_DECISION_ERRORS; tightest.bounds.min_fps_improvement = 0.0; - assert!(validate(&tightest).is_ok()); + assert!(validate(&manifest(), &tightest).is_ok()); } #[test] @@ -527,16 +586,37 @@ mod tests { fn rejects_a_candidate_value_above_the_policy_bound() { // The broker only sees the value inside the lifecycle, which a rejected // trial never reaches, so the measurement path bounds it itself. - assert!(validate(&spec_for_value(2, 5, 5, 3, MAX_MOCK_VALUE)).is_ok()); + assert!(validate(&manifest(), &spec_for_value(2, 5, 5, 3, MAX_MOCK_VALUE)).is_ok()); let message = rejection(&spec_for_value(2, 5, 5, 3, MAX_MOCK_VALUE + 1)); assert!(message.contains("target value"), "{message}"); assert!(rejection(&spec_for_value(2, 5, 5, 3, u64::MAX)).contains("target value")); } + #[test] + fn rejects_a_capability_the_provider_does_not_advertise() { + // The measurement model only describes the mock knob, so a target the + // registry never accepted must fail closed before anything is measured + // or journaled - not later, inside the lifecycle. + let mut foreign = spec(2, 5, 5, 3); + foreign.target.capability_id = "gpu.core-clock-offset".to_owned(); + let error = validate(&manifest(), &foreign).expect_err("unknown hardware fails closed"); + assert!( + matches!( + &error, + RunnerError::ControlPlane(ControlPlaneError::UnknownCapability(id)) + if id == "gpu.core-clock-offset" + ), + "{error:?}" + ); + } + #[test] fn rejects_a_target_without_an_unsigned_value() { let mut spec = spec(2, 5, 5, 3); spec.target.parameters = json!({ "value": "high" }); - assert!(matches!(validate(&spec), Err(RunnerError::InvalidTarget))); + assert!(matches!( + validate(&manifest(), &spec), + Err(RunnerError::InvalidTarget) + )); } } diff --git a/apps/experiment-runner/tests/integration.rs b/apps/experiment-runner/tests/integration.rs index 5ff8869..ed2c026 100644 --- a/apps/experiment-runner/tests/integration.rs +++ b/apps/experiment-runner/tests/integration.rs @@ -280,6 +280,109 @@ fn a_trial_record_from_an_unsupported_version_fails_closed() { ); } +#[test] +fn a_trial_record_carrying_unknown_fields_fails_closed() { + let journal = NamedTempFile::new().expect("temp journal"); + let mut plane = + ControlPlane::open(Box::new(MockProvider::new(10)), journal.path()).expect("open"); + let trial = run_trial(&mut plane, &spec_for(40, 5.0, 80.0)).expect("run trial"); + + // A field this build does not know about, under a version it does, means a + // divergent writer or a rewritten row. Dropping it silently would let the + // replay call the record consistent, so decoding refuses it instead. + let mut payload = plane.read_trial(trial.id).expect("trial should read"); + payload["unexpected"] = json!(true); + let tampered = plane + .record_trial(&payload) + .expect("the extended record should append"); + let error = replay_trial(&plane, tampered).expect_err("an unknown field should be refused"); + assert!(matches!(error, RunnerError::Decode(_)), "{error:?}"); + + let mut payload = plane.read_trial(trial.id).expect("trial should read"); + payload["verdict"]["unexpected"] = json!(true); + let tampered = plane + .record_trial(&payload) + .expect("the extended record should append"); + let error = replay_trial(&plane, tampered).expect_err("an unknown field should be refused"); + assert!(matches!(error, RunnerError::Decode(_)), "{error:?}"); +} + +#[test] +fn a_replay_reports_bounds_outside_the_policy_envelope() { + let journal = NamedTempFile::new().expect("temp journal"); + let mut plane = + ControlPlane::open(Box::new(MockProvider::new(10)), journal.path()).expect("open"); + let trial = run_trial(&mut plane, &spec_for(40, 5.0, 80.0)).expect("run trial"); + + // Stand in for a row whose thresholds were widened after the fact. Both the + // recorded and the recomputed verdict promote under those widened bounds, + // so comparing verdicts alone cannot catch it - the replay re-checks the + // journaled bounds against the policy envelope instead. + let mut payload = plane.read_trial(trial.id).expect("trial should read"); + payload["spec"]["bounds"]["max_temperature_c"] = json!(200.0); + let widened = plane + .record_trial(&payload) + .expect("a widened record should append"); + + let outcome = replay_trial(&plane, widened).expect("replay trial"); + assert!( + outcome.is_consistent(), + "the widened bounds still reproduce the recorded verdict" + ); + assert!( + !outcome.policy_legal, + "a temperature ceiling of 200 C is outside the policy envelope" + ); + + // The trial as it was actually run stays legal. + let outcome = replay_trial(&plane, trial.id).expect("replay trial"); + assert!(outcome.is_consistent() && outcome.policy_legal); +} + +#[test] +fn replaying_a_trial_that_was_never_recorded_fails_closed() { + let journal = NamedTempFile::new().expect("temp journal"); + let plane = ControlPlane::open(Box::new(MockProvider::new(10)), journal.path()).expect("open"); + + // An absent row is a consistency signal about recorded history, so it is + // reported apart from a journal that could not be read at all. + let error = replay_trial(&plane, 404).expect_err("an unrecorded trial cannot be replayed"); + assert!( + matches!( + error, + RunnerError::ControlPlane(ControlPlaneError::UnknownTrial(404)) + ), + "{error:?}" + ); +} + +#[test] +fn a_trial_targeting_an_unadvertised_capability_is_refused() { + let journal = NamedTempFile::new().expect("temp journal"); + let mut plane = + ControlPlane::open(Box::new(MockProvider::new(10)), journal.path()).expect("open"); + + // The measurement model only describes the mock knob. Without this gate the + // trial would be measured, evaluated, and journaled as authoritative before + // the broker refused the same capability inside the lifecycle. + let mut spec = spec_for(40, 5.0, 80.0); + spec.target.capability_id = "gpu.core-clock-offset".to_owned(); + let error = run_trial(&mut plane, &spec).expect_err("unknown hardware should fail closed"); + assert!( + matches!( + error, + RunnerError::ControlPlane(ControlPlaneError::UnknownCapability(_)) + ), + "{error:?}" + ); + + assert!( + plane.trial_ids().expect("trial ids").is_empty(), + "a refused spec journals nothing" + ); + assert_eq!(current_value(&plane), 10); +} + #[test] fn mvp_one_measured_experiment_is_promoted_or_rejected_by_the_evaluator() { let journal = NamedTempFile::new().expect("temp journal"); diff --git a/crates/contracts/src/lib.rs b/crates/contracts/src/lib.rs index c94225f..f6cfd03 100644 --- a/crates/contracts/src/lib.rs +++ b/crates/contracts/src/lib.rs @@ -121,20 +121,59 @@ pub struct MetricSummary { pub total_errors: u64, } +/// Inclusive ceiling the bounded alpha policy enforces on a trial's temperature +/// bound, in degrees Celsius. +/// +/// The thresholds an immutable evaluator applies arrive in an LLM-authored +/// experiment spec, so a spec could otherwise disarm its own safety gate by +/// declaring an unreachable ceiling. Policy owns the hard envelope: a spec may +/// tighten these bounds but never loosen them. This value sits below the +/// throttle point of the consumer hardware the alpha targets. It is mirrored as +/// `maximum` on `max_temperature_c` in `schemas/experiment.schema.json`. +pub const MAX_DECISION_TEMPERATURE_C: f64 = 90.0; + +/// Inclusive ceiling the bounded alpha policy enforces on a trial's power +/// bound, in watts. +/// +/// See [`MAX_DECISION_TEMPERATURE_C`] for why the envelope is policy-owned. +pub const MAX_DECISION_POWER_W: f64 = 250.0; + +/// Inclusive ceiling the bounded alpha policy enforces on a trial's error +/// budget. +/// +/// A correctness fault is never an acceptable cost of a performance gain on +/// this path, so the alpha promotes nothing that reported one; a spec may +/// restate this ceiling but not raise it. See [`MAX_DECISION_TEMPERATURE_C`] +/// for why the envelope is policy-owned. +pub const MAX_DECISION_ERRORS: u64 = 0; + /// Fixed thresholds the immutable evaluator applies to a trial. +/// +/// Every threshold is bounded by the policy envelope a spec may tighten but +/// never loosen; the same bounds are mirrored in +/// `schemas/experiment.schema.json` and re-checked at run time by the runner. #[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] #[serde(deny_unknown_fields)] pub struct DecisionBounds { /// Minimum samples required in each of the baseline and candidate sets - /// before a promotion can be considered. + /// before a promotion can be considered; at most [`MAX_SAMPLES`]. + #[schemars(range(max = MAX_SAMPLES))] pub min_samples: NonZeroU32, - /// Minimum mean-FPS gain the candidate must show over the baseline. + /// Minimum mean-FPS gain the candidate must show over the baseline; a + /// non-negative gain. + #[schemars(range(min = 0.0))] pub min_fps_improvement: f64, - /// Inclusive ceiling for candidate temperature in degrees Celsius. + /// Inclusive ceiling for candidate temperature in degrees Celsius; above 0 + /// and at most [`MAX_DECISION_TEMPERATURE_C`]. + #[schemars(range(max = MAX_DECISION_TEMPERATURE_C), extend("exclusiveMinimum" = 0.0))] pub max_temperature_c: f64, - /// Inclusive ceiling for candidate power draw in watts. + /// Inclusive ceiling for candidate power draw in watts; above 0 and at most + /// [`MAX_DECISION_POWER_W`]. + #[schemars(range(max = MAX_DECISION_POWER_W), extend("exclusiveMinimum" = 0.0))] pub max_power_w: f64, - /// Inclusive ceiling for candidate correctness errors. + /// Inclusive ceiling for candidate correctness errors; at most + /// [`MAX_DECISION_ERRORS`]. + #[schemars(range(max = MAX_DECISION_ERRORS))] pub max_errors: u64, } @@ -223,7 +262,8 @@ mod tests { use serde_json::{Value, json}; use super::{ - CapabilityDescriptor, ChangeRequest, Decision, DecisionBounds, ExperimentSpec, MAX_SAMPLES, + CapabilityDescriptor, ChangeRequest, Decision, DecisionBounds, ExperimentSpec, + MAX_DECISION_ERRORS, MAX_DECISION_POWER_W, MAX_DECISION_TEMPERATURE_C, MAX_SAMPLES, MetricSample, MetricSummary, NonZeroU32, NonZeroU64, Persistence, ProviderManifest, RiskClass, Verdict, VerdictReason, }; @@ -624,6 +664,39 @@ mod tests { } } + #[test] + fn decision_bounds_are_bounded_like_the_schema() { + // The evaluator's thresholds arrive in the spec, so the policy envelope + // is declared once in this crate and mirrored by both schemas. + let checked_in: Value = + serde_json::from_str(EXPERIMENT_SCHEMA).expect("experiment schema should parse"); + let generated = serde_json::to_value(schemars::schema_for!(ExperimentSpec)) + .expect("generated schema should serialize"); + let envelope = [ + ("min_samples", "maximum", json!(MAX_SAMPLES)), + ("min_fps_improvement", "minimum", json!(0.0)), + ("max_temperature_c", "exclusiveMinimum", json!(0.0)), + ( + "max_temperature_c", + "maximum", + json!(MAX_DECISION_TEMPERATURE_C), + ), + ("max_power_w", "exclusiveMinimum", json!(0.0)), + ("max_power_w", "maximum", json!(MAX_DECISION_POWER_W)), + ("max_errors", "maximum", json!(MAX_DECISION_ERRORS)), + ]; + for (field, keyword, expected) in envelope { + assert_eq!( + checked_in["$defs"]["decision_bounds"]["properties"][field][keyword], expected, + "checked-in decision_bounds.{field}.{keyword}" + ); + assert_eq!( + generated["$defs"]["DecisionBounds"]["properties"][field][keyword], expected, + "generated DecisionBounds.{field}.{keyword}" + ); + } + } + #[test] fn verdict_enum_wire_strings_match_schema() { assert_eq!(wire_string(Decision::Promote), "promote"); diff --git a/crates/control-plane/src/lib.rs b/crates/control-plane/src/lib.rs index 8aba281..e944a94 100644 --- a/crates/control-plane/src/lib.rs +++ b/crates/control-plane/src/lib.rs @@ -4,7 +4,7 @@ use std::{num::NonZeroU64, path::Path, time::Duration}; use fpsmaxxing_contracts::{ChangeRequest, ProviderManifest, RiskClass, StateSnapshot}; use fpsmaxxing_provider_sdk::{Provider, ProviderError}; -use rusqlite::{Connection, TransactionBehavior, params}; +use rusqlite::{Connection, OptionalExtension, TransactionBehavior, params}; use serde::Serialize; use serde_json::{Value, json}; use thiserror::Error; @@ -16,31 +16,6 @@ use thiserror::Error; /// lifecycle runs, can refuse the same values this policy would. pub const MAX_MOCK_VALUE: u64 = 100; -/// Inclusive ceiling the bounded alpha policy enforces on a trial's temperature -/// bound, in degrees Celsius. -/// -/// The thresholds an immutable evaluator applies arrive in an LLM-authored -/// experiment spec, so a spec could otherwise disarm its own safety gate by -/// declaring an unreachable ceiling. Policy owns the hard envelope: a spec may -/// tighten these bounds but never loosen them. This value sits below the -/// throttle point of the consumer hardware the alpha targets. -pub const MAX_DECISION_TEMPERATURE_C: f64 = 90.0; - -/// Inclusive ceiling the bounded alpha policy enforces on a trial's power -/// bound, in watts. -/// -/// See [`MAX_DECISION_TEMPERATURE_C`] for why the envelope is policy-owned. -pub const MAX_DECISION_POWER_W: f64 = 250.0; - -/// Inclusive ceiling the bounded alpha policy enforces on a trial's error -/// budget. -/// -/// A correctness fault is never an acceptable cost of a performance gain on -/// this path, so the alpha promotes nothing that reported one; a spec may -/// restate this ceiling but not raise it. See [`MAX_DECISION_TEMPERATURE_C`] -/// for why the envelope is policy-owned. -pub const MAX_DECISION_ERRORS: u64 = 0; - /// Fail-closed errors from the broker seam. #[derive(Debug, Error)] pub enum ControlPlaneError { @@ -59,6 +34,13 @@ pub enum ControlPlaneError { /// The post-rollback probe did not match the captured snapshot. #[error("rollback verification failed")] RollbackVerificationFailed, + /// No trial has been recorded under the requested identifier. + /// + /// This is distinct from [`Journal`](Self::Journal): the trial journal is + /// append-only, so a missing identifier is a consistency signal about + /// recorded history rather than a transient storage fault. + #[error("unknown trial: {0}")] + UnknownTrial(i64), /// The durable journal could not be read or written. #[error(transparent)] Journal(#[from] rusqlite::Error), @@ -77,6 +59,7 @@ impl ControlPlaneError { Self::Provider(_) => "provider", Self::VerificationFailed => "verification-failed", Self::RollbackVerificationFailed => "rollback-verification-failed", + Self::UnknownTrial(_) => "unknown-trial", Self::Journal(_) => "journal", Self::Serialization(_) => "serialization", } @@ -260,14 +243,19 @@ impl ControlPlane { /// /// # Errors /// - /// Returns an error if no trial has the identifier, or the durable journal + /// Returns [`UnknownTrial`](ControlPlaneError::UnknownTrial) if no trial has + /// the identifier, keeping an absent record distinct from a journal that /// cannot be read or decoded. pub fn read_trial(&self, id: i64) -> Result { - let payload: String = self.journal.query_row( - "SELECT payload FROM experiment_trials WHERE id = ?1", - params![id], - |row| row.get(0), - )?; + let payload: String = self + .journal + .query_row( + "SELECT payload FROM experiment_trials WHERE id = ?1", + params![id], + |row| row.get(0), + ) + .optional()? + .ok_or(ControlPlaneError::UnknownTrial(id))?; Ok(serde_json::from_str(&payload)?) } @@ -920,14 +908,14 @@ mod tests { #[test] fn reading_an_unknown_trial_fails_closed() { + // An identifier that was never recorded is a consistency signal, so it + // is reported apart from a journal that could not be read at all. let plane = plane(false); let error = plane .read_trial(404) .expect_err("an unrecorded trial cannot be read"); - assert!(matches!( - error, - ControlPlaneError::Journal(rusqlite::Error::QueryReturnedNoRows) - )); + assert!(matches!(error, ControlPlaneError::UnknownTrial(404))); + assert_eq!(error.kind(), "unknown-trial"); } #[test] diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 42a6dd4..1c60952 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -32,7 +32,7 @@ The runner controls workload setup, warmup, repeated measurements, cooldown, cor In the safe alpha (`apps/experiment-runner`) the runner measures a baseline and a candidate against a deterministic model, then a pure immutable evaluator returns a promote or reject verdict from the recorded samples and fixed bounds alone - no clock, no LLM, no I/O. Every trial is journaled as a self-describing, versioned record so it can be replayed and re-evaluated from the journal without the original conversation. The record is appended once, after the broker lifecycle it authorized has finished and carrying either that lifecycle's outcome or the error the broker returned, so a promotion the broker refuses still leaves the measurements that authorized it and no API can rewrite a recorded verdict. -Sample counts are bounded by the spec schema and rechecked by the runner before any measurement runs, and the spec's decision bounds are intersected with a policy-owned envelope so a spec can tighten its own safety gate but never loosen it. Because mock capabilities are leased and the broker lifecycle always rolls back, the verdict gates whether the candidate is applied at all rather than whether it persists; durable keep-or-rollback awaits the privileged broker. +Sample counts are bounded by the spec schema and rechecked by the runner before any measurement runs, the target must name a capability the accepted provider advertises so unknown hardware fails closed before anything is measured or journaled, and the spec's decision bounds are intersected with a policy-owned envelope - declared once in `crates/contracts`, mirrored in `schemas/experiment.schema.json`, and re-checked on replay - so a spec can tighten its own safety gate but never loosen it. Because mock capabilities are leased and the broker lifecycle always rolls back, the verdict gates whether the candidate is applied at all rather than whether it persists; durable keep-or-rollback awaits the privileged broker. ### Provider sidecars diff --git a/docs/adr/0002-alpha-experiment-journal.md b/docs/adr/0002-alpha-experiment-journal.md index 28d0dac..652ec10 100644 --- a/docs/adr/0002-alpha-experiment-journal.md +++ b/docs/adr/0002-alpha-experiment-journal.md @@ -23,6 +23,8 @@ Crash safety for the window between a mutation and that record stays with the li The write-ahead apply intent makes a crash between mutation and journaling distinguishable from an apply that never started, and the terminal record guarantees that a surviving process never leaves an experiment with only partial stage rows. A per-stage two-phase protocol would duplicate that machinery for the mock-only alpha before the broker owns the transaction log. Trial records carry a `schema_version` so a future field addition is a version bump a reader can refuse rather than a silent misread of journaled history. +The version gate only catches a writer that bumps it, so the record types also reject unknown fields: a row carrying a field this build does not know, under a version it does, means a divergent writer or a rewritten row and fails the replay rather than decoding with that field dropped. +Replay re-checks the journaled decision bounds against the policy envelope as well, because a row whose thresholds were widened after the fact re-evaluates to the same verdict under those same widened thresholds and so is invisible to a verdict comparison alone. ## Consequences diff --git a/schemas/experiment.schema.json b/schemas/experiment.schema.json index 871d591..9d994d9 100644 --- a/schemas/experiment.schema.json +++ b/schemas/experiment.schema.json @@ -42,11 +42,19 @@ "max_errors" ], "properties": { - "min_samples": { "type": "integer", "minimum": 1 }, - "min_fps_improvement": { "type": "number" }, - "max_temperature_c": { "type": "number" }, - "max_power_w": { "type": "number" }, - "max_errors": { "type": "integer", "minimum": 0 } + "min_samples": { "type": "integer", "minimum": 1, "maximum": 10000 }, + "min_fps_improvement": { "type": "number", "minimum": 0.0 }, + "max_temperature_c": { + "type": "number", + "exclusiveMinimum": 0.0, + "maximum": 90.0 + }, + "max_power_w": { + "type": "number", + "exclusiveMinimum": 0.0, + "maximum": 250.0 + }, + "max_errors": { "type": "integer", "minimum": 0, "maximum": 0 } } } } From 321acba5ed44b1096f7cc93e61e657831c8d1746 Mon Sep 17 00:00:00 2001 From: Jerry Xiao Date: Mon, 27 Jul 2026 08:19:16 +0000 Subject: [PATCH 06/16] no-mistakes(review): recheck full policy gate on replay, bound baseline and modeled capability --- apps/experiment-runner/src/model.rs | 7 + apps/experiment-runner/src/runner.rs | 135 +++++++++++++++----- apps/experiment-runner/tests/integration.rs | 86 ++++++++++++- docs/ARCHITECTURE.md | 4 +- docs/adr/0002-alpha-experiment-journal.md | 3 +- 5 files changed, 202 insertions(+), 33 deletions(-) diff --git a/apps/experiment-runner/src/model.rs b/apps/experiment-runner/src/model.rs index 046bf55..f00c969 100644 --- a/apps/experiment-runner/src/model.rs +++ b/apps/experiment-runner/src/model.rs @@ -14,6 +14,13 @@ use fpsmaxxing_contracts::MetricSample; +/// The one capability [`measure`] describes. +/// +/// Every coefficient below is a property of the bounded mock knob, so a trial +/// against any other capability would be measured by a model that says nothing +/// about it. The runner refuses such a target rather than measuring it. +pub(crate) const MODELED_CAPABILITY_ID: &str = "mock.value"; + /// Frames per second reported while warming up, before steady state. const COLD_FPS: f64 = 60.0; /// Steady-state frames per second at the lowest knob value. diff --git a/apps/experiment-runner/src/runner.rs b/apps/experiment-runner/src/runner.rs index 7470321..820037a 100644 --- a/apps/experiment-runner/src/runner.rs +++ b/apps/experiment-runner/src/runner.rs @@ -34,6 +34,8 @@ //! written, so swapping it in means moving the candidate measurement inside the //! apply/lease window and running the evaluator gate after it. +use std::num::NonZeroU32; + use fpsmaxxing_contracts::{ Decision, DecisionBounds, ExperimentSpec, MAX_DECISION_ERRORS, MAX_DECISION_POWER_W, MAX_DECISION_TEMPERATURE_C, MAX_SAMPLES, MetricSample, ProviderManifest, Verdict, @@ -66,6 +68,9 @@ pub enum RunnerError { /// The provider snapshot did not carry an unsigned mock value. #[error("provider snapshot is missing an unsigned mock value")] InvalidBaseline, + /// The provider is sitting at a value outside the policy envelope. + #[error("provider snapshot value is {0}, above the {MAX_MOCK_VALUE} the policy allows")] + BaselineOutOfPolicy(u64), /// A journaled trial record was written by an unsupported record version. #[error("journaled trial uses unsupported record version {0}")] UnsupportedRecordVersion(u32), @@ -173,8 +178,8 @@ pub struct StoredTrial { /// /// A replay is a tamper-detection read, so the outcome must be inspected: /// [`is_consistent`](Self::is_consistent) reports whether the recomputation -/// agrees with the journal, and `policy_legal` reports whether the bounds the -/// journaled verdict was decided under are still inside the policy envelope. +/// agrees with the journal, and `policy_legal` reports whether the record is one +/// the run-time gates would still accept. #[derive(Clone, Debug)] #[must_use] pub struct ReplayOutcome { @@ -184,7 +189,8 @@ pub struct ReplayOutcome { pub recorded: Verdict, /// The verdict recomputed from the journaled samples and bounds. pub recomputed: Verdict, - /// Whether the journaled decision bounds are inside the policy envelope. + /// Whether the journaled record passes every run-time gate and agrees with + /// the spec it carries; see [`replay_trial`]. pub policy_legal: bool, } @@ -209,12 +215,13 @@ impl ReplayOutcome { /// # Errors /// /// Returns an error if the spec is outside the bounded envelope, if its target -/// names a capability the accepted provider does not advertise, if the spec -/// target or provider snapshot lacks an unsigned mock value, or if the broker -/// or durable journal rejects an operation. A lifecycle error is returned only -/// after the trial record carrying it has been journaled; if that write also -/// fails, the lifecycle error still takes precedence and the journal failure is -/// traced to stderr. +/// names a capability the measurement model does not describe or the accepted +/// provider does not advertise, if the spec target lacks an unsigned mock value, +/// if the provider snapshot lacks one or reports one outside the policy +/// envelope, or if the broker or durable journal rejects an operation. A +/// lifecycle error is returned only after the trial record carrying it has been +/// journaled; if that write also fails, the lifecycle error still takes +/// precedence and the journal failure is traced to stderr. pub fn run_trial( plane: &mut ControlPlane, spec: &ExperimentSpec, @@ -275,10 +282,10 @@ pub fn run_trial( /// and recomputed verdicts for comparison. It consults no chat history and /// re-runs no workload. /// -/// The journaled bounds are re-checked against the policy envelope as well, so -/// a row whose thresholds were widened after the fact is reported as -/// `policy_legal = false` even when re-evaluating under those same widened -/// thresholds reproduces the recorded verdict. +/// The journaled record is re-checked against the run-time gates as well, so a +/// row that was rewritten after the fact is reported as `policy_legal = false` +/// even when re-evaluating it reproduces the recorded verdict; see +/// [`is_within_policy`]. /// /// # Errors /// @@ -304,40 +311,74 @@ pub fn replay_trial(plane: &ControlPlane, id: i64) -> Result bool { + let Ok(candidate_value) = validate(manifest, &record.spec) else { + return false; + }; + candidate_value == record.candidate_value + && record.baseline_value <= MAX_MOCK_VALUE + && holds_declared_count(&record.baseline_samples, record.spec.baseline_samples) + && holds_declared_count(&record.candidate_samples, record.spec.candidate_samples) +} + +/// Whether a recorded measurement set holds exactly the count its spec declared. +fn holds_declared_count(samples: &[MetricSample], declared: NonZeroU32) -> bool { + u32::try_from(samples.len()).is_ok_and(|counted| counted == declared.get()) +} + /// Rejects a spec whose parameters are unbounded or self-contradictory and /// returns the validated candidate knob value. /// /// Everything the measurement phase and the decision gate consume arrives over /// the wire, so all of it is bounded before any measurement work runs: the -/// target must name a capability the accepted provider actually advertises, -/// sample counts size the measurement buffers and are checked against -/// [`MAX_SAMPLES`], the decision bounds are intersected with the policy envelope -/// by [`validate_bounds`], and the candidate knob value drives the modeled -/// metrics and is checked against [`MAX_MOCK_VALUE`], the same ceiling the -/// broker policy enforces later in the lifecycle. A spec that asks for fewer +/// target must name the single capability the measurement model describes +/// ([`MODELED_CAPABILITY_ID`](model::MODELED_CAPABILITY_ID)) and the accepted +/// provider must advertise it, sample counts size the measurement buffers and +/// are checked against [`MAX_SAMPLES`], the decision bounds are intersected with +/// the policy envelope by [`validate_bounds`], and the candidate knob value +/// drives the modeled metrics and is checked against [`MAX_MOCK_VALUE`], the +/// same ceiling the broker policy enforces later in the lifecycle and the same +/// one [`baseline_value`] holds the provider to. A spec that asks for fewer /// counted samples than its own bounds require can never promote, so it is /// refused up front rather than measured and then rejected. The validated value /// is returned so the measurement uses exactly what was bounded here. /// -/// The capability check is what keeps unknown hardware failing closed. The -/// broker refuses an unadvertised capability too, but only inside the -/// lifecycle, which a rejected trial never reaches - and by then a measurement -/// the model never described would already have been journaled as an -/// authoritative trial. +/// The capability check is what keeps unknown hardware failing closed, and it +/// is the model rather than the registry that decides what is known: the +/// measurement path is hard-wired to the mock knob, so advertising a second +/// capability must not make it measurable. The broker refuses an unadvertised +/// capability too, but only inside the lifecycle, which a rejected trial never +/// reaches - and by then a measurement the model never described would already +/// have been journaled as an authoritative trial. fn validate(manifest: &ProviderManifest, spec: &ExperimentSpec) -> Result { - if !manifest + let modeled = spec.target.capability_id == model::MODELED_CAPABILITY_ID; + let advertised = manifest .capabilities .iter() - .any(|capability| capability.id == spec.target.capability_id) - { + .any(|capability| capability.id == spec.target.capability_id); + if !modeled || !advertised { return Err(ControlPlaneError::UnknownCapability(spec.target.capability_id.clone()).into()); } validate_bounds(&spec.bounds)?; @@ -424,13 +465,23 @@ fn validate_bounds(bounds: &DecisionBounds) -> Result<(), RunnerError> { } /// Reads the baseline knob value from the provider's current state. +/// +/// The baseline arrives from the provider rather than the spec, but it drives +/// the same measurement model, so it is held to the same [`MAX_MOCK_VALUE`] +/// ceiling as the candidate. A provider sitting outside the policy envelope +/// fails closed instead of contributing a modeled baseline the envelope would +/// never have permitted to the journaled trial. fn baseline_value(plane: &ControlPlane) -> Result { - plane + let value = plane .snapshot()? .state .get("value") .and_then(Value::as_u64) - .ok_or(RunnerError::InvalidBaseline) + .ok_or(RunnerError::InvalidBaseline)?; + if value > MAX_MOCK_VALUE { + return Err(RunnerError::BaselineOutOfPolicy(value)); + } + Ok(value) } /// Reads the candidate knob value from the spec target parameters. @@ -610,6 +661,30 @@ mod tests { ); } + #[test] + fn rejects_an_advertised_capability_the_model_does_not_describe() { + // Advertising a knob does not make it measurable: the model encodes the + // mock knob's coefficients only, so a second capability on the same + // provider must not be measured with it and journaled as authoritative. + let mut manifest = manifest(); + let mut other = manifest.capabilities[0].clone(); + other.id = "mock.other".to_owned(); + manifest.capabilities.push(other); + + let mut foreign = spec(2, 5, 5, 3); + foreign.target.capability_id = "mock.other".to_owned(); + let error = validate(&manifest, &foreign).expect_err("an unmodeled knob fails closed"); + assert!( + matches!( + &error, + RunnerError::ControlPlane(ControlPlaneError::UnknownCapability(id)) + if id == "mock.other" + ), + "{error:?}" + ); + assert!(validate(&manifest, &spec(2, 5, 5, 3)).is_ok()); + } + #[test] fn rejects_a_target_without_an_unsigned_value() { let mut spec = spec(2, 5, 5, 3); diff --git a/apps/experiment-runner/tests/integration.rs b/apps/experiment-runner/tests/integration.rs index ed2c026..27a1745 100644 --- a/apps/experiment-runner/tests/integration.rs +++ b/apps/experiment-runner/tests/integration.rs @@ -13,7 +13,7 @@ use std::num::{NonZeroU32, NonZeroU64}; use fpsmaxxing_contracts::{ ChangeRequest, Decision, DecisionBounds, ExperimentSpec, VerdictReason, }; -use fpsmaxxing_control_plane::{ControlPlane, ControlPlaneError}; +use fpsmaxxing_control_plane::{ControlPlane, ControlPlaneError, MAX_MOCK_VALUE}; use fpsmaxxing_experiment_runner::{RunnerError, TrialRecord, evaluate, replay_trial, run_trial}; use fpsmaxxing_mock_provider::MockProvider; use serde_json::json; @@ -339,6 +339,90 @@ fn a_replay_reports_bounds_outside_the_policy_envelope() { assert!(outcome.is_consistent() && outcome.policy_legal); } +#[test] +fn a_replay_reports_a_record_the_run_time_gate_would_refuse() { + let journal = NamedTempFile::new().expect("temp journal"); + let mut plane = + ControlPlane::open(Box::new(MockProvider::new(10)), journal.path()).expect("open"); + let trial = run_trial(&mut plane, &spec_for(40, 5.0, 80.0)).expect("run trial"); + let recorded = plane.read_trial(trial.id).expect("trial should read"); + + // Each of these rows re-evaluates to exactly the verdict it carries, so + // comparing verdicts cannot catch any of them. Replay re-runs the whole + // run-time gate over the journaled spec instead, and cross-checks the + // record against the spec it carries. + let refused = [ + ("a capability the model never described", { + let mut payload = recorded.clone(); + payload["spec"]["target"]["capability_id"] = json!("gpu.core-clock-offset"); + payload + }), + ("a candidate value above the policy ceiling", { + let mut payload = recorded.clone(); + payload["spec"]["target"]["parameters"]["value"] = json!(MAX_MOCK_VALUE + 1); + payload["candidate_value"] = json!(MAX_MOCK_VALUE + 1); + payload + }), + ("a candidate value contradicting its own spec", { + let mut payload = recorded.clone(); + payload["candidate_value"] = json!(41); + payload + }), + ("a baseline outside the policy envelope", { + let mut payload = recorded.clone(); + payload["baseline_value"] = json!(MAX_MOCK_VALUE + 1); + payload + }), + ("fewer samples than the spec declared", { + let mut payload = recorded.clone(); + payload["spec"]["candidate_samples"] = json!(4); + payload + }), + ]; + + for (rewrite, payload) in refused { + let tampered = plane + .record_trial(&payload) + .expect("a rewritten record should append"); + let outcome = replay_trial(&plane, tampered).expect("replay trial"); + assert!( + outcome.is_consistent(), + "{rewrite} still reproduces the recorded verdict" + ); + assert!(!outcome.policy_legal, "{rewrite} must replay as illegal"); + } + + // The trial as it was actually run stays legal. + let outcome = replay_trial(&plane, trial.id).expect("replay trial"); + assert!(outcome.is_consistent() && outcome.policy_legal); +} + +#[test] +fn a_baseline_outside_the_policy_bound_is_refused_before_any_measurement() { + let journal = NamedTempFile::new().expect("temp journal"); + let mut plane = ControlPlane::open( + Box::new(MockProvider::new(MAX_MOCK_VALUE + 1)), + journal.path(), + ) + .expect("open"); + + // The baseline drives the same model as the candidate, so a provider parked + // outside the envelope fails closed rather than being measured into a + // journaled trial the policy would never have permitted. + let error = run_trial(&mut plane, &spec_for(40, 5.0, 80.0)) + .expect_err("an out-of-policy baseline should be refused"); + assert!( + matches!(error, RunnerError::BaselineOutOfPolicy(value) if value == MAX_MOCK_VALUE + 1), + "{error:?}" + ); + + assert!( + plane.trial_ids().expect("trial ids").is_empty(), + "a refused baseline journals nothing" + ); + assert_eq!(current_value(&plane), MAX_MOCK_VALUE + 1); +} + #[test] fn replaying_a_trial_that_was_never_recorded_fails_closed() { let journal = NamedTempFile::new().expect("temp journal"); diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 1c60952..c7097bf 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -32,7 +32,9 @@ The runner controls workload setup, warmup, repeated measurements, cooldown, cor In the safe alpha (`apps/experiment-runner`) the runner measures a baseline and a candidate against a deterministic model, then a pure immutable evaluator returns a promote or reject verdict from the recorded samples and fixed bounds alone - no clock, no LLM, no I/O. Every trial is journaled as a self-describing, versioned record so it can be replayed and re-evaluated from the journal without the original conversation. The record is appended once, after the broker lifecycle it authorized has finished and carrying either that lifecycle's outcome or the error the broker returned, so a promotion the broker refuses still leaves the measurements that authorized it and no API can rewrite a recorded verdict. -Sample counts are bounded by the spec schema and rechecked by the runner before any measurement runs, the target must name a capability the accepted provider advertises so unknown hardware fails closed before anything is measured or journaled, and the spec's decision bounds are intersected with a policy-owned envelope - declared once in `crates/contracts`, mirrored in `schemas/experiment.schema.json`, and re-checked on replay - so a spec can tighten its own safety gate but never loosen it. Because mock capabilities are leased and the broker lifecycle always rolls back, the verdict gates whether the candidate is applied at all rather than whether it persists; durable keep-or-rollback awaits the privileged broker. +Sample counts are bounded by the spec schema and rechecked by the runner before any measurement runs, the target must name the one capability the measurement model describes and the accepted provider advertises so unknown hardware fails closed before anything is measured or journaled, the candidate and the provider's own baseline are both held to the knob ceiling the broker policy enforces later, and the spec's decision bounds are intersected with a policy-owned envelope - declared once in `crates/contracts`, mirrored in `schemas/experiment.schema.json`, and re-checked on replay - so a spec can tighten its own safety gate but never loosen it. +Replay applies that whole gate again to the journaled record rather than the bounds alone, so a row rewritten after the fact is reported as outside policy even when re-evaluating it reproduces the recorded verdict. +Because mock capabilities are leased and the broker lifecycle always rolls back, the verdict gates whether the candidate is applied at all rather than whether it persists; durable keep-or-rollback awaits the privileged broker. ### Provider sidecars diff --git a/docs/adr/0002-alpha-experiment-journal.md b/docs/adr/0002-alpha-experiment-journal.md index 652ec10..a1e157e 100644 --- a/docs/adr/0002-alpha-experiment-journal.md +++ b/docs/adr/0002-alpha-experiment-journal.md @@ -24,7 +24,8 @@ The write-ahead apply intent makes a crash between mutation and journaling disti A per-stage two-phase protocol would duplicate that machinery for the mock-only alpha before the broker owns the transaction log. Trial records carry a `schema_version` so a future field addition is a version bump a reader can refuse rather than a silent misread of journaled history. The version gate only catches a writer that bumps it, so the record types also reject unknown fields: a row carrying a field this build does not know, under a version it does, means a divergent writer or a rewritten row and fails the replay rather than decoding with that field dropped. -Replay re-checks the journaled decision bounds against the policy envelope as well, because a row whose thresholds were widened after the fact re-evaluates to the same verdict under those same widened thresholds and so is invisible to a verdict comparison alone. +Replay re-runs the full run-time gate over the journaled spec as well - capability, sample counts, decision bounds, and candidate value - and cross-checks the record's redundant fields against that spec, because a coherently rewritten row re-evaluates to the verdict it carries and so is invisible to a verdict comparison alone. +A row whose thresholds were widened, whose target was pointed at a capability the measurement model never described, or whose samples contradict the counts its spec declared is reported as outside policy even when the recomputed verdict matches. ## Consequences From 72f4a5a5344e19d3daa2b9e30564fa42a824e3c9 Mon Sep 17 00:00:00 2001 From: Jerry Xiao Date: Mon, 27 Jul 2026 08:33:28 +0000 Subject: [PATCH 07/16] no-mistakes(review): decouple replay policy gate from live manifest, report rejection reason --- Cargo.lock | 1 + apps/experiment-runner/Cargo.toml | 1 + apps/experiment-runner/src/main.rs | 14 +- apps/experiment-runner/src/runner.rs | 160 ++++++++++++----- apps/experiment-runner/tests/integration.rs | 183 +++++++++++++++++--- docs/ARCHITECTURE.md | 2 +- docs/adr/0002-alpha-experiment-journal.md | 5 +- 7 files changed, 283 insertions(+), 83 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 031e646..52a6491 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -116,6 +116,7 @@ dependencies = [ "fpsmaxxing-contracts", "fpsmaxxing-control-plane", "fpsmaxxing-mock-provider", + "fpsmaxxing-provider-sdk", "serde", "serde_json", "tempfile", diff --git a/apps/experiment-runner/Cargo.toml b/apps/experiment-runner/Cargo.toml index bf6500a..01c85b6 100644 --- a/apps/experiment-runner/Cargo.toml +++ b/apps/experiment-runner/Cargo.toml @@ -16,6 +16,7 @@ serde_json.workspace = true thiserror.workspace = true [dev-dependencies] +fpsmaxxing-provider-sdk.workspace = true tempfile = "3" [lints] diff --git a/apps/experiment-runner/src/main.rs b/apps/experiment-runner/src/main.rs index 8610a61..3dd44b6 100644 --- a/apps/experiment-runner/src/main.rs +++ b/apps/experiment-runner/src/main.rs @@ -6,10 +6,11 @@ //! in-memory `SQLite` database, so the demo leaves nothing on disk. //! //! A replay that diverges from the journal means the record was tampered with -//! or the immutable evaluator drifted, and a replay whose journaled bounds sit -//! outside the policy envelope means the verdict was decided under thresholds -//! policy never allowed. The demo exits non-zero on either rather than -//! reporting it as a successful run. +//! or the immutable evaluator drifted, and a replay the policy gate refuses - +//! its capability, sample counts, decision bounds, candidate value, baseline +//! ceiling, or the agreement between the record and the spec it carries - means +//! the journaled row is not one this runner would have written. The demo exits +//! non-zero on either rather than reporting it as a successful run. use std::{ num::{NonZeroU32, NonZeroU64}, @@ -56,8 +57,9 @@ fn main() -> Result { } if !replay.policy_legal { eprintln!( - "fpsmaxxing-experiment-runner: trial {} was decided under bounds outside the policy envelope", - replay.trial_id + "fpsmaxxing-experiment-runner: trial {} does not pass the policy gate replay re-applies: {}", + replay.trial_id, + replay.policy_reason.as_deref().unwrap_or("no reason given") ); return Ok(ExitCode::FAILURE); } diff --git a/apps/experiment-runner/src/runner.rs b/apps/experiment-runner/src/runner.rs index 820037a..396a394 100644 --- a/apps/experiment-runner/src/runner.rs +++ b/apps/experiment-runner/src/runner.rs @@ -71,6 +71,9 @@ pub enum RunnerError { /// The provider is sitting at a value outside the policy envelope. #[error("provider snapshot value is {0}, above the {MAX_MOCK_VALUE} the policy allows")] BaselineOutOfPolicy(u64), + /// A journaled trial record disagrees with the spec it carries. + #[error("journaled trial contradicts its own spec: {0}")] + InconsistentRecord(String), /// A journaled trial record was written by an unsupported record version. #[error("journaled trial uses unsupported record version {0}")] UnsupportedRecordVersion(u32), @@ -179,7 +182,7 @@ pub struct StoredTrial { /// A replay is a tamper-detection read, so the outcome must be inspected: /// [`is_consistent`](Self::is_consistent) reports whether the recomputation /// agrees with the journal, and `policy_legal` reports whether the record is one -/// the run-time gates would still accept. +/// the policy gate would still accept. #[derive(Clone, Debug)] #[must_use] pub struct ReplayOutcome { @@ -189,9 +192,14 @@ pub struct ReplayOutcome { pub recorded: Verdict, /// The verdict recomputed from the journaled samples and bounds. pub recomputed: Verdict, - /// Whether the journaled record passes every run-time gate and agrees with - /// the spec it carries; see [`replay_trial`]. + /// Whether the journaled record passes the policy gate and agrees with the + /// spec it carries; see [`replay_trial`]. pub policy_legal: bool, + /// Which gate the record tripped, absent when `policy_legal` holds. + /// + /// The gate produces a precise message naming the offending field, so a + /// flagged trial is diagnosable without re-deriving the cause by hand. + pub policy_reason: Option, } impl ReplayOutcome { @@ -282,10 +290,10 @@ pub fn run_trial( /// and recomputed verdicts for comparison. It consults no chat history and /// re-runs no workload. /// -/// The journaled record is re-checked against the run-time gates as well, so a -/// row that was rewritten after the fact is reported as `policy_legal = false` -/// even when re-evaluating it reproduces the recorded verdict; see -/// [`is_within_policy`]. +/// The journaled record is re-checked against the policy gate as well, so a row +/// that was rewritten after the fact is reported as `policy_legal = false`, with +/// the gate it tripped in `policy_reason`, even when re-evaluating it reproduces +/// the recorded verdict; see [`check_policy`]. /// /// # Errors /// @@ -311,36 +319,78 @@ pub fn replay_trial(plane: &ControlPlane, id: i64) -> Result bool { - let Ok(candidate_value) = validate(manifest, &record.spec) else { - return false; - }; - candidate_value == record.candidate_value - && record.baseline_value <= MAX_MOCK_VALUE - && holds_declared_count(&record.baseline_samples, record.spec.baseline_samples) - && holds_declared_count(&record.candidate_samples, record.spec.candidate_samples) +/// written by this runner. +/// +/// Detection stops at that structural layer. The recorded samples are not +/// re-derived, so a rewrite of the measurements together with the verdict they +/// imply passes both this check and the verdict comparison; catching that needs +/// each row anchored outside itself - a signed or hash-chained journal - which +/// the alpha deliberately does not do. +/// +/// # Errors +/// +/// Returns the gate the record tripped, naming the offending field. +fn check_policy(record: &TrialRecord) -> Result<(), RunnerError> { + let candidate_value = validate_spec(&record.spec)?; + if candidate_value != record.candidate_value { + return Err(RunnerError::InconsistentRecord(format!( + "candidate_value is {}, but its spec asks for {candidate_value}", + record.candidate_value + ))); + } + if record.baseline_value > MAX_MOCK_VALUE { + return Err(RunnerError::BaselineOutOfPolicy(record.baseline_value)); + } + for (label, samples, declared) in [ + ( + "baseline_samples", + &record.baseline_samples, + record.spec.baseline_samples, + ), + ( + "candidate_samples", + &record.candidate_samples, + record.spec.candidate_samples, + ), + ] { + if !holds_declared_count(samples, declared) { + return Err(RunnerError::InconsistentRecord(format!( + "{label} holds {} samples, but its spec declares {}", + samples.len(), + declared.get() + ))); + } + } + Ok(()) } /// Whether a recorded measurement set holds exactly the count its spec declared. @@ -348,37 +398,53 @@ fn holds_declared_count(samples: &[MetricSample], declared: NonZeroU32) -> bool u32::try_from(samples.len()).is_ok_and(|counted| counted == declared.get()) } +/// Rejects a spec the attached provider cannot serve, then applies +/// [`validate_spec`] and returns the validated candidate knob value. +/// +/// The provider check belongs to the run-time path alone: a trial is about to +/// drive a lifecycle against whatever provider is attached now, so a target that +/// provider does not advertise must fail closed here. The broker refuses it too, +/// but only inside the lifecycle, which a rejected trial never reaches - and by +/// then a measurement would already have been journaled as an authoritative +/// trial. Replay has no lifecycle to run and so applies [`validate_spec`] on its +/// own; see [`check_policy`]. +fn validate(manifest: &ProviderManifest, spec: &ExperimentSpec) -> Result { + if !manifest + .capabilities + .iter() + .any(|capability| capability.id == spec.target.capability_id) + { + return Err(ControlPlaneError::UnknownCapability(spec.target.capability_id.clone()).into()); + } + validate_spec(spec) +} + /// Rejects a spec whose parameters are unbounded or self-contradictory and /// returns the validated candidate knob value. /// /// Everything the measurement phase and the decision gate consume arrives over /// the wire, so all of it is bounded before any measurement work runs: the /// target must name the single capability the measurement model describes -/// ([`MODELED_CAPABILITY_ID`](model::MODELED_CAPABILITY_ID)) and the accepted -/// provider must advertise it, sample counts size the measurement buffers and -/// are checked against [`MAX_SAMPLES`], the decision bounds are intersected with -/// the policy envelope by [`validate_bounds`], and the candidate knob value -/// drives the modeled metrics and is checked against [`MAX_MOCK_VALUE`], the -/// same ceiling the broker policy enforces later in the lifecycle and the same -/// one [`baseline_value`] holds the provider to. A spec that asks for fewer -/// counted samples than its own bounds require can never promote, so it is -/// refused up front rather than measured and then rejected. The validated value -/// is returned so the measurement uses exactly what was bounded here. +/// ([`MODELED_CAPABILITY_ID`](model::MODELED_CAPABILITY_ID)), sample counts size +/// the measurement buffers and are checked against [`MAX_SAMPLES`], the decision +/// bounds are intersected with the policy envelope by [`validate_bounds`], and +/// the candidate knob value drives the modeled metrics and is checked against +/// [`MAX_MOCK_VALUE`], the same ceiling the broker policy enforces later in the +/// lifecycle and the same one [`baseline_value`] holds the provider to. A spec +/// that asks for fewer counted samples than its own bounds require can never +/// promote, so it is refused up front rather than measured and then rejected. +/// The validated value is returned so the measurement uses exactly what was +/// bounded here. /// -/// The capability check is what keeps unknown hardware failing closed, and it -/// is the model rather than the registry that decides what is known: the +/// The capability check is what keeps unknown hardware failing closed, and it is +/// the model rather than the registry that decides what is known: the /// measurement path is hard-wired to the mock knob, so advertising a second -/// capability must not make it measurable. The broker refuses an unadvertised -/// capability too, but only inside the lifecycle, which a rejected trial never -/// reaches - and by then a measurement the model never described would already -/// have been journaled as an authoritative trial. -fn validate(manifest: &ProviderManifest, spec: &ExperimentSpec) -> Result { - let modeled = spec.target.capability_id == model::MODELED_CAPABILITY_ID; - let advertised = manifest - .capabilities - .iter() - .any(|capability| capability.id == spec.target.capability_id); - if !modeled || !advertised { +/// capability must not make it measurable. +/// +/// Nothing here reads process state, so [`replay_trial`] can re-run the whole of +/// it over a journaled spec. +fn validate_spec(spec: &ExperimentSpec) -> Result { + if spec.target.capability_id != model::MODELED_CAPABILITY_ID { return Err(ControlPlaneError::UnknownCapability(spec.target.capability_id.clone()).into()); } validate_bounds(&spec.bounds)?; diff --git a/apps/experiment-runner/tests/integration.rs b/apps/experiment-runner/tests/integration.rs index 27a1745..a7cc855 100644 --- a/apps/experiment-runner/tests/integration.rs +++ b/apps/experiment-runner/tests/integration.rs @@ -11,11 +11,13 @@ use std::num::{NonZeroU32, NonZeroU64}; use fpsmaxxing_contracts::{ - ChangeRequest, Decision, DecisionBounds, ExperimentSpec, VerdictReason, + CapabilityDescriptor, ChangeRequest, Decision, DecisionBounds, ExperimentSpec, Persistence, + ProviderManifest, RiskClass, StateSnapshot, VerdictReason, }; use fpsmaxxing_control_plane::{ControlPlane, ControlPlaneError, MAX_MOCK_VALUE}; use fpsmaxxing_experiment_runner::{RunnerError, TrialRecord, evaluate, replay_trial, run_trial}; use fpsmaxxing_mock_provider::MockProvider; +use fpsmaxxing_provider_sdk::{Provider, ProviderError}; use serde_json::json; use tempfile::NamedTempFile; @@ -53,6 +55,54 @@ fn spec_with_lease( } } +/// A provider that advertises a knob no journaled trial ever targeted, standing +/// in for auditing an archived journal on a machine whose hardware has changed. +struct ForeignProvider; + +impl Provider for ForeignProvider { + fn manifest(&self) -> ProviderManifest { + ProviderManifest { + id: "foreign".to_owned(), + protocol_version: NonZeroU32::MIN, + targets: vec![std::env::consts::OS.to_owned()], + capabilities: vec![CapabilityDescriptor { + id: "foreign.knob".to_owned(), + description: "A knob the measurement model never described".to_owned(), + risk: RiskClass::Reversible, + persistence: Persistence::Leased, + input_schema: json!({ "type": "object" }), + }], + } + } + + fn snapshot(&self) -> Result { + Ok(StateSnapshot { + provider_id: "foreign".to_owned(), + state: json!({ "knob": 0 }), + }) + } + + fn preview(&self, _request: &ChangeRequest) -> Result { + Err(ProviderError::UnsupportedCapability( + "foreign.knob".to_owned(), + )) + } + + fn apply(&mut self, _request: &ChangeRequest) -> Result<(), ProviderError> { + Err(ProviderError::UnsupportedCapability( + "foreign.knob".to_owned(), + )) + } + + fn verify(&self, _request: &ChangeRequest) -> Result { + Ok(false) + } + + fn rollback(&mut self, _snapshot: &StateSnapshot) -> Result<(), ProviderError> { + Ok(()) + } +} + /// Reads the mock provider's current knob value through a broker snapshot. fn current_value(plane: &ControlPlane) -> u64 { plane @@ -334,9 +384,16 @@ fn a_replay_reports_bounds_outside_the_policy_envelope() { "a temperature ceiling of 200 C is outside the policy envelope" ); + // The auditor is told which gate tripped, not merely that one did. + let reason = outcome + .policy_reason + .expect("an illegal row carries a reason"); + assert!(reason.contains("max_temperature_c"), "{reason}"); + // The trial as it was actually run stays legal. let outcome = replay_trial(&plane, trial.id).expect("replay trial"); assert!(outcome.is_consistent() && outcome.policy_legal); + assert!(outcome.policy_reason.is_none()); } #[test] @@ -352,35 +409,55 @@ fn a_replay_reports_a_record_the_run_time_gate_would_refuse() { // run-time gate over the journaled spec instead, and cross-checks the // record against the spec it carries. let refused = [ - ("a capability the model never described", { - let mut payload = recorded.clone(); - payload["spec"]["target"]["capability_id"] = json!("gpu.core-clock-offset"); - payload - }), - ("a candidate value above the policy ceiling", { - let mut payload = recorded.clone(); - payload["spec"]["target"]["parameters"]["value"] = json!(MAX_MOCK_VALUE + 1); - payload["candidate_value"] = json!(MAX_MOCK_VALUE + 1); - payload - }), - ("a candidate value contradicting its own spec", { - let mut payload = recorded.clone(); - payload["candidate_value"] = json!(41); - payload - }), - ("a baseline outside the policy envelope", { - let mut payload = recorded.clone(); - payload["baseline_value"] = json!(MAX_MOCK_VALUE + 1); - payload - }), - ("fewer samples than the spec declared", { - let mut payload = recorded.clone(); - payload["spec"]["candidate_samples"] = json!(4); - payload - }), + ( + "a capability the model never described", + "unknown capability", + { + let mut payload = recorded.clone(); + payload["spec"]["target"]["capability_id"] = json!("gpu.core-clock-offset"); + payload + }, + ), + ( + "a candidate value above the policy ceiling", + "target value", + { + let mut payload = recorded.clone(); + payload["spec"]["target"]["parameters"]["value"] = json!(MAX_MOCK_VALUE + 1); + payload["candidate_value"] = json!(MAX_MOCK_VALUE + 1); + payload + }, + ), + ( + "a candidate value contradicting its own spec", + "candidate_value", + { + let mut payload = recorded.clone(); + payload["candidate_value"] = json!(41); + payload + }, + ), + ( + "a baseline outside the policy envelope", + "snapshot value", + { + let mut payload = recorded.clone(); + payload["baseline_value"] = json!(MAX_MOCK_VALUE + 1); + payload + }, + ), + ( + "fewer samples than the spec declared", + "candidate_samples", + { + let mut payload = recorded.clone(); + payload["spec"]["candidate_samples"] = json!(4); + payload + }, + ), ]; - for (rewrite, payload) in refused { + for (rewrite, expected_reason, payload) in refused { let tampered = plane .record_trial(&payload) .expect("a rewritten record should append"); @@ -390,11 +467,61 @@ fn a_replay_reports_a_record_the_run_time_gate_would_refuse() { "{rewrite} still reproduces the recorded verdict" ); assert!(!outcome.policy_legal, "{rewrite} must replay as illegal"); + + // The reported reason names the gate that tripped, so an auditor is not + // left to re-derive which field was rewritten. + let reason = outcome + .policy_reason + .unwrap_or_else(|| panic!("{rewrite} must carry a reason")); + assert!( + reason.contains(expected_reason), + "{rewrite}: expected {expected_reason:?} in {reason:?}" + ); } // The trial as it was actually run stays legal. let outcome = replay_trial(&plane, trial.id).expect("replay trial"); assert!(outcome.is_consistent() && outcome.policy_legal); + assert!(outcome.policy_reason.is_none()); +} + +#[test] +fn a_replay_does_not_depend_on_the_attached_provider() { + let journal = NamedTempFile::new().expect("temp journal"); + + let trial_id = { + let mut plane = + ControlPlane::open(Box::new(MockProvider::new(10)), journal.path()).expect("open"); + run_trial(&mut plane, &spec_for(40, 5.0, 80.0)) + .expect("run trial") + .id + }; + + // The trial journal is a durable file that outlives the process that wrote + // it. Holding a historical row to what the provider attached now advertises + // would report every archived trial as tampered with, so replay checks the + // journaled capability against the one the measurement model describes. + let archived = ControlPlane::open(Box::new(ForeignProvider), journal.path()).expect("reopen"); + let outcome = replay_trial(&archived, trial_id).expect("replay trial"); + + assert!(outcome.is_consistent()); + assert!( + outcome.policy_legal, + "an untampered row must stay legal under any provider: {:?}", + outcome.policy_reason + ); + + // The run-time gate still refuses a trial that provider cannot serve. + let mut archived = archived; + let error = + run_trial(&mut archived, &spec_for(40, 5.0, 80.0)).expect_err("unknown hardware fails"); + assert!( + matches!( + error, + RunnerError::ControlPlane(ControlPlaneError::UnknownCapability(_)) + ), + "{error:?}" + ); } #[test] diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index c7097bf..0ee2fbb 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -33,7 +33,7 @@ The runner controls workload setup, warmup, repeated measurements, cooldown, cor In the safe alpha (`apps/experiment-runner`) the runner measures a baseline and a candidate against a deterministic model, then a pure immutable evaluator returns a promote or reject verdict from the recorded samples and fixed bounds alone - no clock, no LLM, no I/O. Every trial is journaled as a self-describing, versioned record so it can be replayed and re-evaluated from the journal without the original conversation. The record is appended once, after the broker lifecycle it authorized has finished and carrying either that lifecycle's outcome or the error the broker returned, so a promotion the broker refuses still leaves the measurements that authorized it and no API can rewrite a recorded verdict. Sample counts are bounded by the spec schema and rechecked by the runner before any measurement runs, the target must name the one capability the measurement model describes and the accepted provider advertises so unknown hardware fails closed before anything is measured or journaled, the candidate and the provider's own baseline are both held to the knob ceiling the broker policy enforces later, and the spec's decision bounds are intersected with a policy-owned envelope - declared once in `crates/contracts`, mirrored in `schemas/experiment.schema.json`, and re-checked on replay - so a spec can tighten its own safety gate but never loosen it. -Replay applies that whole gate again to the journaled record rather than the bounds alone, so a row rewritten after the fact is reported as outside policy even when re-evaluating it reproduces the recorded verdict. +Replay applies that whole gate again to the journaled record rather than the bounds alone, so a row whose spec, capability, declared sample counts, candidate value, or baseline was rewritten after the fact is reported as outside policy even when re-evaluating it reproduces the recorded verdict; the recorded samples themselves are not re-derived, so detecting a rewrite of the measurements together with the verdict they imply needs a signed or hash-chained journal, which the alpha does not have. Because mock capabilities are leased and the broker lifecycle always rolls back, the verdict gates whether the candidate is applied at all rather than whether it persists; durable keep-or-rollback awaits the privileged broker. ### Provider sidecars diff --git a/docs/adr/0002-alpha-experiment-journal.md b/docs/adr/0002-alpha-experiment-journal.md index a1e157e..71d02b6 100644 --- a/docs/adr/0002-alpha-experiment-journal.md +++ b/docs/adr/0002-alpha-experiment-journal.md @@ -24,8 +24,11 @@ The write-ahead apply intent makes a crash between mutation and journaling disti A per-stage two-phase protocol would duplicate that machinery for the mock-only alpha before the broker owns the transaction log. Trial records carry a `schema_version` so a future field addition is a version bump a reader can refuse rather than a silent misread of journaled history. The version gate only catches a writer that bumps it, so the record types also reject unknown fields: a row carrying a field this build does not know, under a version it does, means a divergent writer or a rewritten row and fails the replay rather than decoding with that field dropped. -Replay re-runs the full run-time gate over the journaled spec as well - capability, sample counts, decision bounds, and candidate value - and cross-checks the record's redundant fields against that spec, because a coherently rewritten row re-evaluates to the verdict it carries and so is invisible to a verdict comparison alone. +Replay re-runs the policy gate over the journaled spec as well - capability, sample counts, decision bounds, and candidate value - and cross-checks the record's redundant fields against that spec, because a rewritten row re-evaluates to the verdict it carries and so is invisible to a verdict comparison alone. A row whose thresholds were widened, whose target was pointed at a capability the measurement model never described, or whose samples contradict the counts its spec declared is reported as outside policy even when the recomputed verdict matches. +That detection is structural only: replay checks the spec, the capability, the declared sample counts, the candidate value, and the baseline ceiling, but does not re-derive the recorded samples, so a rewrite of the measurements together with the verdict they imply passes both checks. +Detecting a coherent rewrite of that kind requires anchoring each row outside itself - a signed or hash-chained journal - which is deliberately out of scope for the alpha. +Replay reads the journaled capability against the constant the measurement model describes rather than the attached provider's manifest, so an archived journal audits the same way under any provider instead of raising a false tamper alarm. ## Consequences From ea9d4d6ad4d5d77da2ab625c04a31bde5a584ae0 Mon Sep 17 00:00:00 2001 From: Jerry Xiao Date: Mon, 27 Jul 2026 08:46:05 +0000 Subject: [PATCH 08/16] no-mistakes(review): enforce decision-lifecycle invariant on replay, fix baseline reason --- apps/experiment-runner/src/main.rs | 7 ++- apps/experiment-runner/src/runner.rs | 30 +++++++++- apps/experiment-runner/tests/integration.rs | 63 +++++++++++++++++++-- docs/adr/0002-alpha-experiment-journal.md | 5 +- 4 files changed, 93 insertions(+), 12 deletions(-) diff --git a/apps/experiment-runner/src/main.rs b/apps/experiment-runner/src/main.rs index 3dd44b6..ee29cfd 100644 --- a/apps/experiment-runner/src/main.rs +++ b/apps/experiment-runner/src/main.rs @@ -8,9 +8,10 @@ //! A replay that diverges from the journal means the record was tampered with //! or the immutable evaluator drifted, and a replay the policy gate refuses - //! its capability, sample counts, decision bounds, candidate value, baseline -//! ceiling, or the agreement between the record and the spec it carries - means -//! the journaled row is not one this runner would have written. The demo exits -//! non-zero on either rather than reporting it as a successful run. +//! ceiling, the lifecycle fields its decision implies, or the agreement between +//! the record and the spec it carries - means the journaled row is not one this +//! runner would have written. The demo exits non-zero on either rather than +//! reporting it as a successful run. use std::{ num::{NonZeroU32, NonZeroU64}, diff --git a/apps/experiment-runner/src/runner.rs b/apps/experiment-runner/src/runner.rs index 396a394..8eff59c 100644 --- a/apps/experiment-runner/src/runner.rs +++ b/apps/experiment-runner/src/runner.rs @@ -348,7 +348,11 @@ pub fn replay_trial(plane: &ControlPlane, id: i64) -> Result Result<(), RunnerError> { ))); } if record.baseline_value > MAX_MOCK_VALUE { - return Err(RunnerError::BaselineOutOfPolicy(record.baseline_value)); + return Err(RunnerError::InconsistentRecord(format!( + "baseline_value is {}, above the {MAX_MOCK_VALUE} the policy allows", + record.baseline_value + ))); } for (label, samples, declared) in [ ( @@ -390,9 +397,28 @@ fn check_policy(record: &TrialRecord) -> Result<(), RunnerError> { ))); } } + let recorded_lifecycle = record.lifecycle.is_some(); + let recorded_failure = record.lifecycle_error.is_some(); + let holds = match record.verdict.decision { + Decision::Promote => recorded_lifecycle != recorded_failure, + Decision::Reject => !recorded_lifecycle && !recorded_failure, + }; + if !holds { + return Err(RunnerError::InconsistentRecord(format!( + "verdict is {:?} with lifecycle {} and lifecycle_error {}, but a promotion records exactly one of the two and a rejection neither", + record.verdict.decision, + presence(recorded_lifecycle), + presence(recorded_failure) + ))); + } Ok(()) } +/// Renders whether an optional record field was journaled. +fn presence(recorded: bool) -> &'static str { + if recorded { "present" } else { "absent" } +} + /// Whether a recorded measurement set holds exactly the count its spec declared. fn holds_declared_count(samples: &[MetricSample], declared: NonZeroU32) -> bool { u32::try_from(samples.len()).is_ok_and(|counted| counted == declared.get()) diff --git a/apps/experiment-runner/tests/integration.rs b/apps/experiment-runner/tests/integration.rs index a7cc855..68ce6d6 100644 --- a/apps/experiment-runner/tests/integration.rs +++ b/apps/experiment-runner/tests/integration.rs @@ -397,7 +397,7 @@ fn a_replay_reports_bounds_outside_the_policy_envelope() { } #[test] -fn a_replay_reports_a_record_the_run_time_gate_would_refuse() { +fn a_replay_reports_a_record_the_policy_gate_would_refuse() { let journal = NamedTempFile::new().expect("temp journal"); let mut plane = ControlPlane::open(Box::new(MockProvider::new(10)), journal.path()).expect("open"); @@ -405,9 +405,9 @@ fn a_replay_reports_a_record_the_run_time_gate_would_refuse() { let recorded = plane.read_trial(trial.id).expect("trial should read"); // Each of these rows re-evaluates to exactly the verdict it carries, so - // comparing verdicts cannot catch any of them. Replay re-runs the whole - // run-time gate over the journaled spec instead, and cross-checks the - // record against the spec it carries. + // comparing verdicts cannot catch any of them. Replay applies the policy + // gate to the journaled spec instead - deliberately without the run-time + // manifest check - and cross-checks the record against the spec it carries. let refused = [ ( "a capability the model never described", @@ -439,7 +439,7 @@ fn a_replay_reports_a_record_the_run_time_gate_would_refuse() { ), ( "a baseline outside the policy envelope", - "snapshot value", + "baseline_value", { let mut payload = recorded.clone(); payload["baseline_value"] = json!(MAX_MOCK_VALUE + 1); @@ -485,6 +485,59 @@ fn a_replay_reports_a_record_the_run_time_gate_would_refuse() { assert!(outcome.policy_reason.is_none()); } +#[test] +fn a_replay_reports_lifecycle_fields_that_contradict_the_verdict() { + let journal = NamedTempFile::new().expect("temp journal"); + let mut plane = + ControlPlane::open(Box::new(MockProvider::new(10)), journal.path()).expect("open"); + let promoted = run_trial(&mut plane, &spec_for(40, 5.0, 80.0)).expect("run trial"); + let rejected = run_trial(&mut plane, &spec_for(70, 5.0, 80.0)).expect("run trial"); + assert_eq!(rejected.record.verdict.decision, Decision::Reject); + + // A promotion is journaled once its lifecycle has finished, carrying either + // the outcome or the broker error, so a row with neither claims a promotion + // whose fate went unrecorded. + let mut stripped = plane.read_trial(promoted.id).expect("trial should read"); + stripped["lifecycle"] = json!(null); + + // A rejected candidate is never applied, so a lifecycle on that row claims + // the knob was written when it never was - the trial record is the only + // auditable statement that it reached the provider. + let mut fabricated = plane.read_trial(rejected.id).expect("trial should read"); + fabricated["lifecycle"] = json!({ + "provider_id": "mock", + "preview": "set value to 70", + "verified": true, + "rolled_back": true, + }); + + for (rewrite, payload) in [ + ("a promotion recording no lifecycle", stripped), + ("a rejection recording a lifecycle", fabricated), + ] { + let tampered = plane + .record_trial(&payload) + .expect("a rewritten record should append"); + let outcome = replay_trial(&plane, tampered).expect("replay trial"); + assert!( + outcome.is_consistent(), + "{rewrite} still reproduces the recorded verdict" + ); + assert!(!outcome.policy_legal, "{rewrite} must replay as illegal"); + let reason = outcome + .policy_reason + .unwrap_or_else(|| panic!("{rewrite} must carry a reason")); + assert!(reason.contains("lifecycle"), "{rewrite}: {reason}"); + } + + // Both trials as they were actually run stay legal. + for id in [promoted.id, rejected.id] { + let outcome = replay_trial(&plane, id).expect("replay trial"); + assert!(outcome.is_consistent() && outcome.policy_legal); + assert!(outcome.policy_reason.is_none()); + } +} + #[test] fn a_replay_does_not_depend_on_the_attached_provider() { let journal = NamedTempFile::new().expect("temp journal"); diff --git a/docs/adr/0002-alpha-experiment-journal.md b/docs/adr/0002-alpha-experiment-journal.md index 71d02b6..c2b33e5 100644 --- a/docs/adr/0002-alpha-experiment-journal.md +++ b/docs/adr/0002-alpha-experiment-journal.md @@ -25,8 +25,9 @@ A per-stage two-phase protocol would duplicate that machinery for the mock-only Trial records carry a `schema_version` so a future field addition is a version bump a reader can refuse rather than a silent misread of journaled history. The version gate only catches a writer that bumps it, so the record types also reject unknown fields: a row carrying a field this build does not know, under a version it does, means a divergent writer or a rewritten row and fails the replay rather than decoding with that field dropped. Replay re-runs the policy gate over the journaled spec as well - capability, sample counts, decision bounds, and candidate value - and cross-checks the record's redundant fields against that spec, because a rewritten row re-evaluates to the verdict it carries and so is invisible to a verdict comparison alone. -A row whose thresholds were widened, whose target was pointed at a capability the measurement model never described, or whose samples contradict the counts its spec declared is reported as outside policy even when the recomputed verdict matches. -That detection is structural only: replay checks the spec, the capability, the declared sample counts, the candidate value, and the baseline ceiling, but does not re-derive the recorded samples, so a rewrite of the measurements together with the verdict they imply passes both checks. +A row whose thresholds were widened, whose target was pointed at a capability the measurement model never described, whose samples contradict the counts its spec declared, or whose lifecycle fields contradict its own decision is reported as outside policy even when the recomputed verdict matches. +The lifecycle cross-check matters because the trial row is the only auditable statement that a promotion reached the provider: a promotion carries exactly one of the lifecycle outcome and the broker error, and a rejection carries neither. +That detection is structural only: replay checks the spec, the capability, the declared sample counts, the candidate value, the baseline ceiling, and the lifecycle fields the decision implies, but does not re-derive the recorded samples, so a rewrite of the measurements together with the verdict they imply passes both checks. Detecting a coherent rewrite of that kind requires anchoring each row outside itself - a signed or hash-chained journal - which is deliberately out of scope for the alpha. Replay reads the journaled capability against the constant the measurement model describes rather than the attached provider's manifest, so an archived journal audits the same way under any provider instead of raising a false tamper alarm. From 7afc76a0ede58f0cb288a96e872e735d15a8793c Mon Sep 17 00:00:00 2001 From: Jerry Xiao Date: Mon, 27 Jul 2026 09:08:58 +0000 Subject: [PATCH 09/16] no-mistakes(review): gate lease, lifecycle result fields, and record version --- apps/experiment-runner/src/runner.rs | 74 ++++++++-- apps/experiment-runner/tests/integration.rs | 153 ++++++++++++++++++-- crates/control-plane/src/lib.rs | 21 ++- 3 files changed, 220 insertions(+), 28 deletions(-) diff --git a/apps/experiment-runner/src/runner.rs b/apps/experiment-runner/src/runner.rs index 8eff59c..7918dbb 100644 --- a/apps/experiment-runner/src/runner.rs +++ b/apps/experiment-runner/src/runner.rs @@ -40,7 +40,9 @@ use fpsmaxxing_contracts::{ Decision, DecisionBounds, ExperimentSpec, MAX_DECISION_ERRORS, MAX_DECISION_POWER_W, MAX_DECISION_TEMPERATURE_C, MAX_SAMPLES, MetricSample, ProviderManifest, Verdict, }; -use fpsmaxxing_control_plane::{ControlPlane, ControlPlaneError, LifecycleResult, MAX_MOCK_VALUE}; +use fpsmaxxing_control_plane::{ + ControlPlane, ControlPlaneError, LifecycleResult, MAX_LEASE_SECONDS, MAX_MOCK_VALUE, +}; use serde::{Deserialize, Serialize}; use serde_json::Value; use thiserror::Error; @@ -77,6 +79,13 @@ pub enum RunnerError { /// A journaled trial record was written by an unsupported record version. #[error("journaled trial uses unsupported record version {0}")] UnsupportedRecordVersion(u32), + /// A journaled trial record does not carry a readable record version. + /// + /// Distinct from [`UnsupportedRecordVersion`](Self::UnsupportedRecordVersion): + /// the row states no version at all, so it was not written by any runner + /// this format describes rather than by an older one. + #[error("journaled trial does not carry a readable record version")] + MalformedRecordVersion, /// A journaled trial record could not be decoded for replay. #[error(transparent)] Decode(#[from] serde_json::Error), @@ -298,18 +307,20 @@ pub fn run_trial( /// # Errors /// /// Returns an error if the trial cannot be read from the durable journal, its -/// record cannot be decoded, or the record was written by an unsupported -/// [`TRIAL_RECORD_VERSION`]. +/// record cannot be decoded, the record was written by an unsupported +/// [`TRIAL_RECORD_VERSION`], or it states no readable version at all. pub fn replay_trial(plane: &ControlPlane, id: i64) -> Result { let payload = plane.read_trial(id)?; // The version is read off the raw payload so a record whose fields this // build cannot decode still reports the version that wrote it rather than a - // decode error that says nothing about why. + // decode error that says nothing about why. A row stating no readable + // version is not an older record but a foreign or corrupted one, so it is + // reported apart from a version this build merely does not support. let version = payload .get("schema_version") .and_then(Value::as_u64) .and_then(|version| u32::try_from(version).ok()) - .unwrap_or_default(); + .ok_or(RunnerError::MalformedRecordVersion)?; if version != TRIAL_RECORD_VERSION { return Err(RunnerError::UnsupportedRecordVersion(version)); } @@ -352,7 +363,11 @@ pub fn replay_trial(plane: &ControlPlane, id: i64) -> Result Result<(), RunnerError> { presence(recorded_failure) ))); } + if let Some(lifecycle) = &record.lifecycle + && !(lifecycle.verified && lifecycle.rolled_back) + { + return Err(RunnerError::InconsistentRecord(format!( + "lifecycle records verified {} and rolled_back {}, but a completed lifecycle observes the applied value and restores the captured baseline", + lifecycle.verified, lifecycle.rolled_back + ))); + } Ok(()) } @@ -456,11 +479,14 @@ fn validate(manifest: &ProviderManifest, spec: &ExperimentSpec) -> Result Result { "target value is {value}, above the {MAX_MOCK_VALUE} the policy allows" ))); } + let lease = spec.target.lease_seconds.get(); + if lease > MAX_LEASE_SECONDS { + return Err(RunnerError::InvalidSpec(format!( + "lease_seconds is {lease}, above the {MAX_LEASE_SECONDS} the policy allows" + ))); + } Ok(value) } @@ -597,7 +629,7 @@ mod tests { use super::{ ExperimentSpec, MAX_DECISION_ERRORS, MAX_DECISION_POWER_W, MAX_DECISION_TEMPERATURE_C, - MAX_MOCK_VALUE, MAX_SAMPLES, ProviderManifest, RunnerError, validate, + MAX_LEASE_SECONDS, MAX_MOCK_VALUE, MAX_SAMPLES, ProviderManifest, RunnerError, validate, }; /// A manifest advertising only the knob the measurement model describes. @@ -735,6 +767,24 @@ mod tests { assert!(rejection(&spec_for_value(2, 5, 5, 3, u64::MAX)).contains("target value")); } + #[test] + fn rejects_a_lease_above_the_policy_bound() { + // The lease is the TTL bounding how long a mutation may persist, and + // the broker only sees it inside the lifecycle. Measuring first would + // journal an authoritative trial the lifecycle can only ever deny. + let mut leased = spec(2, 5, 5, 3); + leased.target.lease_seconds = + NonZeroU64::new(MAX_LEASE_SECONDS).expect("lease is non-zero"); + assert!(validate(&manifest(), &leased).is_ok()); + + leased.target.lease_seconds = + NonZeroU64::new(MAX_LEASE_SECONDS + 1).expect("lease is non-zero"); + assert!(rejection(&leased).contains("lease_seconds")); + + leased.target.lease_seconds = NonZeroU64::MAX; + assert!(rejection(&leased).contains("lease_seconds")); + } + #[test] fn rejects_a_capability_the_provider_does_not_advertise() { // The measurement model only describes the mock knob, so a target the diff --git a/apps/experiment-runner/tests/integration.rs b/apps/experiment-runner/tests/integration.rs index 68ce6d6..ecb3a9d 100644 --- a/apps/experiment-runner/tests/integration.rs +++ b/apps/experiment-runner/tests/integration.rs @@ -14,7 +14,9 @@ use fpsmaxxing_contracts::{ CapabilityDescriptor, ChangeRequest, Decision, DecisionBounds, ExperimentSpec, Persistence, ProviderManifest, RiskClass, StateSnapshot, VerdictReason, }; -use fpsmaxxing_control_plane::{ControlPlane, ControlPlaneError, MAX_MOCK_VALUE}; +use fpsmaxxing_control_plane::{ + ControlPlane, ControlPlaneError, MAX_LEASE_SECONDS, MAX_MOCK_VALUE, +}; use fpsmaxxing_experiment_runner::{RunnerError, TrialRecord, evaluate, replay_trial, run_trial}; use fpsmaxxing_mock_provider::MockProvider; use fpsmaxxing_provider_sdk::{Provider, ProviderError}; @@ -27,8 +29,7 @@ fn spec_for(candidate: u64, min_fps_improvement: f64, max_temperature_c: f64) -> spec_with_lease(candidate, min_fps_improvement, max_temperature_c, 30) } -/// Builds the same spec with an explicit lease, so a test can drive the broker -/// policy into denying the change the evaluator promoted. +/// Builds the same spec with an explicit TTL lease. fn spec_with_lease( candidate: u64, min_fps_improvement: f64, @@ -103,6 +104,41 @@ impl Provider for ForeignProvider { } } +/// A provider advertising the modeled knob under a risk class the broker's +/// policy refuses, standing in for a lifecycle denied after the evaluator has +/// already promoted on the measurements. +struct ApprovalGatedProvider(MockProvider); + +impl Provider for ApprovalGatedProvider { + fn manifest(&self) -> ProviderManifest { + let mut manifest = self.0.manifest(); + for capability in &mut manifest.capabilities { + capability.risk = RiskClass::ApprovalRequired; + } + manifest + } + + fn snapshot(&self) -> Result { + self.0.snapshot() + } + + fn preview(&self, request: &ChangeRequest) -> Result { + self.0.preview(request) + } + + fn apply(&mut self, request: &ChangeRequest) -> Result<(), ProviderError> { + self.0.apply(request) + } + + fn verify(&self, request: &ChangeRequest) -> Result { + self.0.verify(request) + } + + fn rollback(&mut self, snapshot: &StateSnapshot) -> Result<(), ProviderError> { + self.0.rollback(snapshot) + } +} + /// Reads the mock provider's current knob value through a broker snapshot. fn current_value(plane: &ControlPlane) -> u64 { plane @@ -186,13 +222,17 @@ fn a_rejected_experiment_is_never_applied_and_leaves_the_baseline() { #[test] fn a_promoted_trial_survives_a_lifecycle_the_broker_refuses() { let journal = NamedTempFile::new().expect("temp journal"); - let mut plane = - ControlPlane::open(Box::new(MockProvider::new(10)), journal.path()).expect("open"); + let mut plane = ControlPlane::open( + Box::new(ApprovalGatedProvider(MockProvider::new(10))), + journal.path(), + ) + .expect("open"); - // The evaluator promotes on the measurements, but the 400 second lease is - // outside the broker's policy envelope, so the lifecycle never runs. - let spec = spec_with_lease(40, 5.0, 80.0, 400); - let error = run_trial(&mut plane, &spec).expect_err("policy should deny the lease"); + // The evaluator promotes on the measurements, but this provider advertises + // the knob under a risk class the broker's policy refuses, so the lifecycle + // never runs. + let spec = spec_for(40, 5.0, 80.0); + let error = run_trial(&mut plane, &spec).expect_err("policy should deny the risk class"); assert!(matches!( error, RunnerError::ControlPlane(ControlPlaneError::PolicyDenied(_)) @@ -218,7 +258,13 @@ fn a_promoted_trial_survives_a_lifecycle_the_broker_refuses() { .lifecycle_error .expect("the refused lifecycle is recorded on the trial"); assert_eq!(failure.kind, "policy-denied"); - assert!(failure.error.contains("lease exceeds 300 seconds")); + assert!( + failure + .error + .contains("only reversible mock capabilities are enabled"), + "{}", + failure.error + ); let outcome = replay_trial(&plane, ids[0]).expect("replay trial"); assert!(outcome.is_consistent()); @@ -293,6 +339,77 @@ fn a_candidate_outside_the_policy_bound_is_refused_before_any_measurement() { assert_eq!(current_value(&plane), 10); } +#[test] +fn a_lease_outside_the_policy_bound_is_refused_before_any_measurement() { + let journal = NamedTempFile::new().expect("temp journal"); + let mut plane = + ControlPlane::open(Box::new(MockProvider::new(10)), journal.path()).expect("open"); + + // The lease bounds how long a mutation may persist and the broker only + // checks it inside the lifecycle, so a spec that can only ever be denied is + // refused before it is measured and journaled as an authoritative trial. + let error = run_trial( + &mut plane, + &spec_with_lease(40, 5.0, 80.0, MAX_LEASE_SECONDS + 1), + ) + .expect_err("an out-of-policy lease should be refused"); + assert!(matches!(error, RunnerError::InvalidSpec(_)), "{error:?}"); + + assert!( + plane.trial_ids().expect("trial ids").is_empty(), + "a refused spec journals nothing" + ); + assert_eq!(current_value(&plane), 10); + + // A lease exactly at the ceiling is inside the envelope and still runs. + let trial = run_trial( + &mut plane, + &spec_with_lease(40, 5.0, 80.0, MAX_LEASE_SECONDS), + ) + .expect("run trial"); + assert_eq!(trial.record.verdict.decision, Decision::Promote); +} + +#[test] +fn a_trial_record_without_a_readable_version_fails_closed() { + let journal = NamedTempFile::new().expect("temp journal"); + let mut plane = + ControlPlane::open(Box::new(MockProvider::new(10)), journal.path()).expect("open"); + let trial = run_trial(&mut plane, &spec_for(40, 5.0, 80.0)).expect("run trial"); + let recorded = plane.read_trial(trial.id).expect("trial should read"); + + // A row stating no version this build can read was not written by an older + // runner, so it is reported apart from a version merely unsupported here. + for rewrite in [json!(null), json!("one"), json!(u64::from(u32::MAX) + 1)] { + let mut payload = recorded.clone(); + payload["schema_version"] = rewrite.clone(); + let malformed = plane + .record_trial(&payload) + .expect("a malformed record should append"); + let error = + replay_trial(&plane, malformed).expect_err("an unreadable version should be refused"); + assert!( + matches!(error, RunnerError::MalformedRecordVersion), + "{rewrite}: {error:?}" + ); + } + + // A record that simply omits the field is the same signal. + let mut payload = recorded; + payload + .as_object_mut() + .expect("a trial record is an object") + .remove("schema_version"); + let malformed = plane + .record_trial(&payload) + .expect("a malformed record should append"); + let error = replay_trial(&plane, malformed).expect_err("an absent version should be refused"); + assert!( + matches!(error, RunnerError::MalformedRecordVersion), + "{error:?}" + ); +} + #[test] fn a_trial_record_from_an_unsupported_version_fails_closed() { let journal = NamedTempFile::new().expect("temp journal"); @@ -428,6 +545,11 @@ fn a_replay_reports_a_record_the_policy_gate_would_refuse() { payload }, ), + ("a lease above the policy ceiling", "lease_seconds", { + let mut payload = recorded.clone(); + payload["spec"]["target"]["lease_seconds"] = json!(MAX_LEASE_SECONDS + 1); + payload + }), ( "a candidate value contradicting its own spec", "candidate_value", @@ -511,9 +633,20 @@ fn a_replay_reports_lifecycle_fields_that_contradict_the_verdict() { "rolled_back": true, }); + // The broker returns an outcome only once the applied value verified and + // the captured baseline was restored, so a row claiming a promoted knob was + // left mutated - or was never verified - is one this runner cannot write. + let mut left_mutated = plane.read_trial(promoted.id).expect("trial should read"); + left_mutated["lifecycle"]["rolled_back"] = json!(false); + + let mut unverified = plane.read_trial(promoted.id).expect("trial should read"); + unverified["lifecycle"]["verified"] = json!(false); + for (rewrite, payload) in [ ("a promotion recording no lifecycle", stripped), ("a rejection recording a lifecycle", fabricated), + ("a promotion left un-rolled-back", left_mutated), + ("a promotion recording no verification", unverified), ] { let tampered = plane .record_trial(&payload) diff --git a/crates/control-plane/src/lib.rs b/crates/control-plane/src/lib.rs index e944a94..77aa1e9 100644 --- a/crates/control-plane/src/lib.rs +++ b/crates/control-plane/src/lib.rs @@ -1,6 +1,6 @@ //! Capability registry, policy seam, broker lifecycle, and durable journal. -use std::{num::NonZeroU64, path::Path, time::Duration}; +use std::{path::Path, time::Duration}; use fpsmaxxing_contracts::{ChangeRequest, ProviderManifest, RiskClass, StateSnapshot}; use fpsmaxxing_provider_sdk::{Provider, ProviderError}; @@ -16,6 +16,15 @@ use thiserror::Error; /// lifecycle runs, can refuse the same values this policy would. pub const MAX_MOCK_VALUE: u64 = 100; +/// Inclusive ceiling the bounded alpha policy enforces on a change request's +/// TTL lease, in seconds. +/// +/// [`ControlPlane::run_lifecycle`] rejects any change above it. It is exported +/// alongside [`MAX_MOCK_VALUE`] so the experiment runner, which bounds a spec +/// before the lifecycle runs, holds the lease to the same ceiling this policy +/// does rather than restating it. +pub const MAX_LEASE_SECONDS: u64 = 300; + /// Fail-closed errors from the broker seam. #[derive(Debug, Error)] pub enum ControlPlaneError { @@ -285,10 +294,10 @@ impl ControlPlane { "only reversible mock capabilities are enabled".to_owned(), )); } - if request.lease_seconds > NonZeroU64::new(300).expect("constant is non-zero") { - return Err(ControlPlaneError::PolicyDenied( - "lease exceeds 300 seconds".to_owned(), - )); + if request.lease_seconds.get() > MAX_LEASE_SECONDS { + return Err(ControlPlaneError::PolicyDenied(format!( + "lease exceeds {MAX_LEASE_SECONDS} seconds" + ))); } let value = request .parameters @@ -471,7 +480,7 @@ impl ControlPlane { #[cfg(test)] mod tests { - use std::num::NonZeroU32; + use std::num::{NonZeroU32, NonZeroU64}; use fpsmaxxing_contracts::{CapabilityDescriptor, Persistence}; From 4a8f2220b97cf5400977421c9625d5b3efb3830b Mon Sep 17 00:00:00 2001 From: Jerry Xiao Date: Mon, 27 Jul 2026 09:28:36 +0000 Subject: [PATCH 10/16] no-mistakes(review): test lease gate, share lease cap, carry trial id --- apps/experiment-runner/src/main.rs | 29 +++++++-- apps/experiment-runner/src/runner.rs | 68 +++++++++++++++------ apps/experiment-runner/tests/integration.rs | 37 ++++++----- crates/contracts/src/lib.rs | 39 ++++++++++-- crates/control-plane/src/lib.rs | 55 +++++++++++++---- docs/ARCHITECTURE.md | 4 +- docs/adr/0002-alpha-experiment-journal.md | 4 +- schemas/experiment.schema.json | 2 +- 8 files changed, 178 insertions(+), 60 deletions(-) diff --git a/apps/experiment-runner/src/main.rs b/apps/experiment-runner/src/main.rs index ee29cfd..1acd75d 100644 --- a/apps/experiment-runner/src/main.rs +++ b/apps/experiment-runner/src/main.rs @@ -7,11 +7,13 @@ //! //! A replay that diverges from the journal means the record was tampered with //! or the immutable evaluator drifted, and a replay the policy gate refuses - -//! its capability, sample counts, decision bounds, candidate value, baseline -//! ceiling, the lifecycle fields its decision implies, or the agreement between -//! the record and the spec it carries - means the journaled row is not one this -//! runner would have written. The demo exits non-zero on either rather than -//! reporting it as a successful run. +//! its capability, sample counts, decision bounds, candidate value, TTL lease, +//! baseline ceiling, the lifecycle fields its decision implies, or the +//! agreement between the record and the spec it carries - means the journaled +//! row is not one this runner would have written. A lifecycle that fails after +//! the trial was measured is the third failure: the trial is journaled anyway +//! and the error names the row, which the demo reports. It exits non-zero on +//! any of the three rather than reporting a successful run. use std::{ num::{NonZeroU32, NonZeroU64}, @@ -28,7 +30,22 @@ fn main() -> Result { let mut plane = ControlPlane::open(Box::new(MockProvider::new(10)), ":memory:")?; let spec = demo_spec(); - let trial = run_trial(&mut plane, &spec)?; + let trial = match run_trial(&mut plane, &spec) { + Ok(trial) => trial, + Err(RunnerError::LifecycleFailed { trial_id, source }) => { + // The measurements that authorized the promotion were journaled + // before the broker error surfaced, and the error names that row. + eprintln!( + "fpsmaxxing-experiment-runner: the lifecycle failed after the trial was journaled: {source}" + ); + match trial_id { + Some(id) => eprintln!(" the trial recording it is {id}"), + None => eprintln!(" the trial recording it could not be journaled"), + } + return Ok(ExitCode::FAILURE); + } + Err(error) => return Err(error), + }; let verdict = &trial.record.verdict; println!( "trial {} -> {:?} ({:?}); fps_improvement = {:.1}", diff --git a/apps/experiment-runner/src/runner.rs b/apps/experiment-runner/src/runner.rs index 7918dbb..f78725e 100644 --- a/apps/experiment-runner/src/runner.rs +++ b/apps/experiment-runner/src/runner.rs @@ -14,7 +14,9 @@ //! afterwards. A lifecycle that fails after a promotion - a policy denial, a //! provider fault, or a rollback that could not be verified - is therefore //! still recorded, as a [`LifecycleFailure`] carried by the same record as the -//! measurements that authorized the apply, and is also returned to the caller. +//! measurements that authorized the apply, and is also returned to the caller +//! together with the identifier that record was stored under, so the caller +//! addresses its own row rather than the journal's last one. //! Crash safety for the window between the mutation and that record stays with //! the lifecycle journal's write-ahead `apply-intent` stage (ADR 0002). //! @@ -38,11 +40,10 @@ use std::num::NonZeroU32; use fpsmaxxing_contracts::{ Decision, DecisionBounds, ExperimentSpec, MAX_DECISION_ERRORS, MAX_DECISION_POWER_W, - MAX_DECISION_TEMPERATURE_C, MAX_SAMPLES, MetricSample, ProviderManifest, Verdict, -}; -use fpsmaxxing_control_plane::{ - ControlPlane, ControlPlaneError, LifecycleResult, MAX_LEASE_SECONDS, MAX_MOCK_VALUE, + MAX_DECISION_TEMPERATURE_C, MAX_LEASE_SECONDS, MAX_SAMPLES, MetricSample, ProviderManifest, + Verdict, }; +use fpsmaxxing_control_plane::{ControlPlane, ControlPlaneError, LifecycleResult, MAX_MOCK_VALUE}; use serde::{Deserialize, Serialize}; use serde_json::Value; use thiserror::Error; @@ -61,6 +62,23 @@ pub enum RunnerError { /// The broker or the durable journal rejected an operation. #[error(transparent)] ControlPlane(#[from] ControlPlaneError), + /// A promoted trial's lifecycle failed after its record was journaled. + /// + /// The record carrying the measurements that authorized the promotion is + /// written before the error surfaces, so its identifier travels with the + /// error and the caller can address that exact row. Recovering it by + /// reading back the last identifier the journal holds would be wrong: the + /// trial journal is shared, and a concurrent runner can append between the + /// write and the read. + #[error("lifecycle failed after {}: {source}", journaled_as(*trial_id))] + LifecycleFailed { + /// Identifier of the journaled trial, absent only when journaling the + /// record failed too; that failure is traced to stderr. + trial_id: Option, + /// The broker error that ended the lifecycle. + #[source] + source: ControlPlaneError, + }, /// The experiment specification is outside the bounded alpha envelope. #[error("experiment spec rejected: {0}")] InvalidSpec(String), @@ -237,8 +255,10 @@ impl ReplayOutcome { /// if the provider snapshot lacks one or reports one outside the policy /// envelope, or if the broker or durable journal rejects an operation. A /// lifecycle error is returned only after the trial record carrying it has been -/// journaled; if that write also fails, the lifecycle error still takes -/// precedence and the journal failure is traced to stderr. +/// journaled, as a [`LifecycleFailed`](RunnerError::LifecycleFailed) naming the +/// identifier that record was stored under; if that write also fails, the +/// lifecycle error still takes precedence, the identifier is absent, and the +/// journal failure is traced to stderr. pub fn run_trial( plane: &mut ControlPlane, spec: &ExperimentSpec, @@ -278,13 +298,17 @@ pub fn run_trial( .map(LifecycleFailure::from), }; let journaled = plane.record_trial(&record); - if let Some(Err(error)) = outcome { - if let Err(journal_error) = journaled { - eprintln!( - "fpsmaxxing-experiment-runner: could not journal the trial whose lifecycle failed: {journal_error}" - ); - } - return Err(error.into()); + if let Some(Err(source)) = outcome { + let trial_id = match journaled { + Ok(id) => Some(id), + Err(journal_error) => { + eprintln!( + "fpsmaxxing-experiment-runner: could not journal the trial whose lifecycle failed: {journal_error}" + ); + None + } + }; + return Err(RunnerError::LifecycleFailed { trial_id, source }); } Ok(StoredTrial { id: journaled?, @@ -302,7 +326,7 @@ pub fn run_trial( /// The journaled record is re-checked against the policy gate as well, so a row /// that was rewritten after the fact is reported as `policy_legal = false`, with /// the gate it tripped in `policy_reason`, even when re-evaluating it reproduces -/// the recorded verdict; see [`check_policy`]. +/// the recorded verdict. /// /// # Errors /// @@ -344,9 +368,9 @@ pub fn replay_trial(plane: &ControlPlane, id: i64) -> Result Result<(), RunnerError> { Ok(()) } +/// Renders where a failed lifecycle's trial record came to rest. +fn journaled_as(trial_id: Option) -> String { + match trial_id { + Some(id) => format!("journaling trial {id}"), + None => "failing to journal its trial".to_owned(), + } +} + /// Renders whether an optional record field was journaled. fn presence(recorded: bool) -> &'static str { if recorded { "present" } else { "absent" } diff --git a/apps/experiment-runner/tests/integration.rs b/apps/experiment-runner/tests/integration.rs index ecb3a9d..64a8799 100644 --- a/apps/experiment-runner/tests/integration.rs +++ b/apps/experiment-runner/tests/integration.rs @@ -11,12 +11,10 @@ use std::num::{NonZeroU32, NonZeroU64}; use fpsmaxxing_contracts::{ - CapabilityDescriptor, ChangeRequest, Decision, DecisionBounds, ExperimentSpec, Persistence, - ProviderManifest, RiskClass, StateSnapshot, VerdictReason, -}; -use fpsmaxxing_control_plane::{ - ControlPlane, ControlPlaneError, MAX_LEASE_SECONDS, MAX_MOCK_VALUE, + CapabilityDescriptor, ChangeRequest, Decision, DecisionBounds, ExperimentSpec, + MAX_LEASE_SECONDS, Persistence, ProviderManifest, RiskClass, StateSnapshot, VerdictReason, }; +use fpsmaxxing_control_plane::{ControlPlane, ControlPlaneError, MAX_MOCK_VALUE}; use fpsmaxxing_experiment_runner::{RunnerError, TrialRecord, evaluate, replay_trial, run_trial}; use fpsmaxxing_mock_provider::MockProvider; use fpsmaxxing_provider_sdk::{Provider, ProviderError}; @@ -233,21 +231,30 @@ fn a_promoted_trial_survives_a_lifecycle_the_broker_refuses() { // never runs. let spec = spec_for(40, 5.0, 80.0); let error = run_trial(&mut plane, &spec).expect_err("policy should deny the risk class"); - assert!(matches!( - error, - RunnerError::ControlPlane(ControlPlaneError::PolicyDenied(_)) - )); + let reported = error.to_string(); // The measurements that authorized the promotion are journaled with the - // refusal, in one append, so the trial is still discoverable and replayable. - let ids = plane.trial_ids().expect("trial ids"); + // refusal, in one append, and the error names the row it was written to, so + // the caller addresses its own trial instead of the journal's last one. + let RunnerError::LifecycleFailed { trial_id, source } = error else { + panic!("a refused lifecycle reports the trial it journaled, got {reported}"); + }; + assert!( + matches!(source, ControlPlaneError::PolicyDenied(_)), + "{source:?}" + ); + let trial_id = trial_id.expect("the refused lifecycle journaled its trial"); + assert!( + reported.contains(&format!("trial {trial_id}")), + "the reported error names the journaled row: {reported}" + ); assert_eq!( - ids.len(), - 1, + plane.trial_ids().expect("trial ids"), + [trial_id], "the failed promotion is journaled exactly once" ); let record: TrialRecord = - serde_json::from_value(plane.read_trial(ids[0]).expect("trial should read")) + serde_json::from_value(plane.read_trial(trial_id).expect("trial should read")) .expect("trial should decode"); assert_eq!(record.verdict.decision, Decision::Promote); assert!( @@ -266,7 +273,7 @@ fn a_promoted_trial_survives_a_lifecycle_the_broker_refuses() { failure.error ); - let outcome = replay_trial(&plane, ids[0]).expect("replay trial"); + let outcome = replay_trial(&plane, trial_id).expect("replay trial"); assert!(outcome.is_consistent()); assert_eq!(outcome.recomputed.decision, Decision::Promote); diff --git a/crates/contracts/src/lib.rs b/crates/contracts/src/lib.rs index f6cfd03..bc90f7f 100644 --- a/crates/contracts/src/lib.rs +++ b/crates/contracts/src/lib.rs @@ -64,6 +64,16 @@ pub struct ProviderManifest { pub capabilities: Vec, } +/// Inclusive ceiling on a [`ChangeRequest`]'s TTL lease, in seconds. +/// +/// The lease is what bounds how long a mutation may survive, so the ceiling +/// belongs to the shared request type rather than to any one enforcement point: +/// it is mirrored as `maximum` on `lease_seconds` in +/// `schemas/experiment.schema.json`, applied by the broker policy in +/// `crates/control-plane` before a lifecycle runs, and applied again by the +/// experiment runner, which bounds a spec before it measures anything. +pub const MAX_LEASE_SECONDS: u64 = 300; + /// A requested provider change after policy validation. #[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] #[serde(deny_unknown_fields)] @@ -73,7 +83,8 @@ pub struct ChangeRequest { /// Capability-specific parameters. pub parameters: Value, /// Automatic rollback deadline in seconds; every mutation carries a - /// non-zero TTL lease. + /// non-zero TTL lease, at most [`MAX_LEASE_SECONDS`]. + #[schemars(range(max = MAX_LEASE_SECONDS))] pub lease_seconds: NonZeroU64, } @@ -263,9 +274,9 @@ mod tests { use super::{ CapabilityDescriptor, ChangeRequest, Decision, DecisionBounds, ExperimentSpec, - MAX_DECISION_ERRORS, MAX_DECISION_POWER_W, MAX_DECISION_TEMPERATURE_C, MAX_SAMPLES, - MetricSample, MetricSummary, NonZeroU32, NonZeroU64, Persistence, ProviderManifest, - RiskClass, Verdict, VerdictReason, + MAX_DECISION_ERRORS, MAX_DECISION_POWER_W, MAX_DECISION_TEMPERATURE_C, MAX_LEASE_SECONDS, + MAX_SAMPLES, MetricSample, MetricSummary, NonZeroU32, NonZeroU64, Persistence, + ProviderManifest, RiskClass, Verdict, VerdictReason, }; const CAPABILITY_SCHEMA: &str = include_str!("../../../schemas/capability.schema.json"); @@ -446,6 +457,26 @@ mod tests { assert!(serde_json::from_value::(serialized).is_err()); } + #[test] + fn lease_seconds_is_bounded_like_the_schema() { + // The lease bounds how long a mutation may survive, so the ceiling is + // declared once on the shared request type and mirrored wherever a + // change request is published for an agent to author against. + let checked_in: Value = + serde_json::from_str(EXPERIMENT_SCHEMA).expect("experiment schema should parse"); + assert_eq!( + checked_in["$defs"]["change_request"]["properties"]["lease_seconds"]["maximum"], + json!(MAX_LEASE_SECONDS) + ); + + let generated = serde_json::to_value(schemars::schema_for!(ExperimentSpec)) + .expect("generated schema should serialize"); + assert_eq!( + generated["$defs"]["ChangeRequest"]["properties"]["lease_seconds"]["maximum"], + json!(MAX_LEASE_SECONDS) + ); + } + /// Asserts that two object schemas declare the same fields. fn assert_object_parity(label: &str, generated: &Value, checked_in: &Value) { let generated_properties: BTreeSet = generated["properties"] diff --git a/crates/control-plane/src/lib.rs b/crates/control-plane/src/lib.rs index 77aa1e9..f101b38 100644 --- a/crates/control-plane/src/lib.rs +++ b/crates/control-plane/src/lib.rs @@ -2,7 +2,9 @@ use std::{path::Path, time::Duration}; -use fpsmaxxing_contracts::{ChangeRequest, ProviderManifest, RiskClass, StateSnapshot}; +use fpsmaxxing_contracts::{ + ChangeRequest, MAX_LEASE_SECONDS, ProviderManifest, RiskClass, StateSnapshot, +}; use fpsmaxxing_provider_sdk::{Provider, ProviderError}; use rusqlite::{Connection, OptionalExtension, TransactionBehavior, params}; use serde::Serialize; @@ -13,18 +15,11 @@ use thiserror::Error; /// /// [`ControlPlane::run_lifecycle`] rejects any change above it. It is exported /// so the experiment runner, which acts on a candidate value before the -/// lifecycle runs, can refuse the same values this policy would. +/// lifecycle runs, can refuse the same values this policy would. Callers that +/// publish their own request schema, such as the gateway's advertised tool +/// input, state the bound independently. pub const MAX_MOCK_VALUE: u64 = 100; -/// Inclusive ceiling the bounded alpha policy enforces on a change request's -/// TTL lease, in seconds. -/// -/// [`ControlPlane::run_lifecycle`] rejects any change above it. It is exported -/// alongside [`MAX_MOCK_VALUE`] so the experiment runner, which bounds a spec -/// before the lifecycle runs, holds the lease to the same ceiling this policy -/// does rather than restating it. -pub const MAX_LEASE_SECONDS: u64 = 300; - /// Fail-closed errors from the broker seam. #[derive(Debug, Error)] pub enum ControlPlaneError { @@ -569,10 +564,14 @@ mod tests { ]; fn request(value: u64) -> ChangeRequest { + leased_request(value, 30) + } + + fn leased_request(value: u64, lease_seconds: u64) -> ChangeRequest { ChangeRequest { capability_id: "mock.value".to_owned(), parameters: json!({ "value": value }), - lease_seconds: NonZeroU64::new(30).expect("lease is non-zero"), + lease_seconds: NonZeroU64::new(lease_seconds).expect("lease is non-zero"), } } @@ -874,6 +873,38 @@ mod tests { assert!(matches!(error, ControlPlaneError::PolicyDenied(_))); } + #[test] + fn a_lease_above_the_policy_ceiling_is_denied() { + // The lease is the TTL that bounds how long a mutation may persist, and + // the gateway hands one straight from an agent to this seam, so the + // ceiling is checked before the provider is touched at all. + let mut plane = plane(false); + let error = plane + .run_lifecycle(&leased_request(42, MAX_LEASE_SECONDS + 1)) + .expect_err("an oversized lease should be denied"); + assert!( + matches!(&error, ControlPlaneError::PolicyDenied(message) if message.contains("lease")), + "{error:?}" + ); + assert!( + plane + .journal_stages() + .expect("journal should read") + .is_empty(), + "a denied request opens no experiment" + ); + + // The ceiling is inclusive, so a lease exactly at it still runs. + let result = plane + .run_lifecycle(&leased_request(42, MAX_LEASE_SECONDS)) + .expect("a lease at the ceiling is inside the envelope"); + assert!(result.verified && result.rolled_back); + assert_eq!( + plane.journal_stages().expect("journal should read"), + LIFECYCLE_STAGES + ); + } + #[test] fn trial_records_round_trip_through_the_journal() { let plane = plane(false); diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 0ee2fbb..19ad33c 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -32,8 +32,8 @@ The runner controls workload setup, warmup, repeated measurements, cooldown, cor In the safe alpha (`apps/experiment-runner`) the runner measures a baseline and a candidate against a deterministic model, then a pure immutable evaluator returns a promote or reject verdict from the recorded samples and fixed bounds alone - no clock, no LLM, no I/O. Every trial is journaled as a self-describing, versioned record so it can be replayed and re-evaluated from the journal without the original conversation. The record is appended once, after the broker lifecycle it authorized has finished and carrying either that lifecycle's outcome or the error the broker returned, so a promotion the broker refuses still leaves the measurements that authorized it and no API can rewrite a recorded verdict. -Sample counts are bounded by the spec schema and rechecked by the runner before any measurement runs, the target must name the one capability the measurement model describes and the accepted provider advertises so unknown hardware fails closed before anything is measured or journaled, the candidate and the provider's own baseline are both held to the knob ceiling the broker policy enforces later, and the spec's decision bounds are intersected with a policy-owned envelope - declared once in `crates/contracts`, mirrored in `schemas/experiment.schema.json`, and re-checked on replay - so a spec can tighten its own safety gate but never loosen it. -Replay applies that whole gate again to the journaled record rather than the bounds alone, so a row whose spec, capability, declared sample counts, candidate value, or baseline was rewritten after the fact is reported as outside policy even when re-evaluating it reproduces the recorded verdict; the recorded samples themselves are not re-derived, so detecting a rewrite of the measurements together with the verdict they imply needs a signed or hash-chained journal, which the alpha does not have. +Sample counts are bounded by the spec schema and rechecked by the runner before any measurement runs, the target must name the one capability the measurement model describes and the accepted provider advertises so unknown hardware fails closed before anything is measured or journaled, the candidate and the provider's own baseline are both held to the knob ceiling the broker policy enforces later, the target's TTL lease is held to the ceiling the broker enforces on every change request, and the spec's decision bounds are intersected with a policy-owned envelope - declared once in `crates/contracts`, mirrored in `schemas/experiment.schema.json`, and re-checked on replay - so a spec can tighten its own safety gate but never loosen it. +Replay applies that whole gate again to the journaled record rather than the bounds alone, so a row whose spec, capability, declared sample counts, candidate value, TTL lease, or baseline was rewritten after the fact is reported as outside policy even when re-evaluating it reproduces the recorded verdict; the recorded samples themselves are not re-derived, so detecting a rewrite of the measurements together with the verdict they imply needs a signed or hash-chained journal, which the alpha does not have. Because mock capabilities are leased and the broker lifecycle always rolls back, the verdict gates whether the candidate is applied at all rather than whether it persists; durable keep-or-rollback awaits the privileged broker. ### Provider sidecars diff --git a/docs/adr/0002-alpha-experiment-journal.md b/docs/adr/0002-alpha-experiment-journal.md index c2b33e5..8cb6125 100644 --- a/docs/adr/0002-alpha-experiment-journal.md +++ b/docs/adr/0002-alpha-experiment-journal.md @@ -24,10 +24,10 @@ The write-ahead apply intent makes a crash between mutation and journaling disti A per-stage two-phase protocol would duplicate that machinery for the mock-only alpha before the broker owns the transaction log. Trial records carry a `schema_version` so a future field addition is a version bump a reader can refuse rather than a silent misread of journaled history. The version gate only catches a writer that bumps it, so the record types also reject unknown fields: a row carrying a field this build does not know, under a version it does, means a divergent writer or a rewritten row and fails the replay rather than decoding with that field dropped. -Replay re-runs the policy gate over the journaled spec as well - capability, sample counts, decision bounds, and candidate value - and cross-checks the record's redundant fields against that spec, because a rewritten row re-evaluates to the verdict it carries and so is invisible to a verdict comparison alone. +Replay re-runs the policy gate over the journaled spec as well - capability, sample counts, decision bounds, candidate value, and TTL lease - and cross-checks the record's redundant fields against that spec, because a rewritten row re-evaluates to the verdict it carries and so is invisible to a verdict comparison alone. A row whose thresholds were widened, whose target was pointed at a capability the measurement model never described, whose samples contradict the counts its spec declared, or whose lifecycle fields contradict its own decision is reported as outside policy even when the recomputed verdict matches. The lifecycle cross-check matters because the trial row is the only auditable statement that a promotion reached the provider: a promotion carries exactly one of the lifecycle outcome and the broker error, and a rejection carries neither. -That detection is structural only: replay checks the spec, the capability, the declared sample counts, the candidate value, the baseline ceiling, and the lifecycle fields the decision implies, but does not re-derive the recorded samples, so a rewrite of the measurements together with the verdict they imply passes both checks. +That detection is structural only: replay checks the spec, the capability, the declared sample counts, the candidate value, the TTL lease, the baseline ceiling, and the lifecycle fields the decision implies, but does not re-derive the recorded samples, so a rewrite of the measurements together with the verdict they imply passes both checks. Detecting a coherent rewrite of that kind requires anchoring each row outside itself - a signed or hash-chained journal - which is deliberately out of scope for the alpha. Replay reads the journaled capability against the constant the measurement model describes rather than the attached provider's manifest, so an archived journal audits the same way under any provider instead of raising a false tamper alarm. diff --git a/schemas/experiment.schema.json b/schemas/experiment.schema.json index 9d994d9..11f40f9 100644 --- a/schemas/experiment.schema.json +++ b/schemas/experiment.schema.json @@ -28,7 +28,7 @@ "properties": { "capability_id": { "type": "string", "minLength": 1 }, "parameters": { "type": "object" }, - "lease_seconds": { "type": "integer", "minimum": 1 } + "lease_seconds": { "type": "integer", "minimum": 1, "maximum": 300 } } }, "decision_bounds": { From a5d02ebe71d1afb9815c279a0d3f842a1a5c97b1 Mon Sep 17 00:00:00 2001 From: Jerry Xiao Date: Mon, 27 Jul 2026 09:51:19 +0000 Subject: [PATCH 11/16] no-mistakes(review): carry journal error, bound hypothesis, document policy drift --- Cargo.lock | 1 + apps/experiment-runner/Cargo.toml | 1 + apps/experiment-runner/src/main.rs | 30 +++--- apps/experiment-runner/src/runner.rs | 104 ++++++++++++++------ apps/experiment-runner/tests/integration.rs | 78 ++++++++++++++- crates/contracts/src/lib.rs | 43 +++++++- docs/ARCHITECTURE.md | 5 +- docs/adr/0002-alpha-experiment-journal.md | 7 +- schemas/experiment.schema.json | 2 +- 9 files changed, 218 insertions(+), 53 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 52a6491..1347832 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -117,6 +117,7 @@ dependencies = [ "fpsmaxxing-control-plane", "fpsmaxxing-mock-provider", "fpsmaxxing-provider-sdk", + "rusqlite", "serde", "serde_json", "tempfile", diff --git a/apps/experiment-runner/Cargo.toml b/apps/experiment-runner/Cargo.toml index 01c85b6..30f05c1 100644 --- a/apps/experiment-runner/Cargo.toml +++ b/apps/experiment-runner/Cargo.toml @@ -17,6 +17,7 @@ thiserror.workspace = true [dev-dependencies] fpsmaxxing-provider-sdk.workspace = true +rusqlite.workspace = true tempfile = "3" [lints] diff --git a/apps/experiment-runner/src/main.rs b/apps/experiment-runner/src/main.rs index 1acd75d..da02fec 100644 --- a/apps/experiment-runner/src/main.rs +++ b/apps/experiment-runner/src/main.rs @@ -7,13 +7,14 @@ //! //! A replay that diverges from the journal means the record was tampered with //! or the immutable evaluator drifted, and a replay the policy gate refuses - -//! its capability, sample counts, decision bounds, candidate value, TTL lease, -//! baseline ceiling, the lifecycle fields its decision implies, or the -//! agreement between the record and the spec it carries - means the journaled -//! row is not one this runner would have written. A lifecycle that fails after -//! the trial was measured is the third failure: the trial is journaled anyway -//! and the error names the row, which the demo reports. It exits non-zero on -//! any of the three rather than reporting a successful run. +//! its capability, hypothesis, sample counts, decision bounds, candidate value, +//! TTL lease, baseline ceiling, the lifecycle fields its decision implies, or +//! the agreement between the record and the spec it carries - means the +//! journaled row is not one this runner would write under the policy in force +//! now. A lifecycle that fails after the trial was measured is the third +//! failure: the trial is journaled anyway and the error names the row, or says +//! why the row was lost, which the demo reports. It exits non-zero on any of +//! the three rather than reporting a successful run. use std::{ num::{NonZeroU32, NonZeroU64}, @@ -32,15 +33,22 @@ fn main() -> Result { let trial = match run_trial(&mut plane, &spec) { Ok(trial) => trial, - Err(RunnerError::LifecycleFailed { trial_id, source }) => { + Err(RunnerError::LifecycleFailed { + trial_id, + journal_error, + source, + }) => { // The measurements that authorized the promotion were journaled // before the broker error surfaced, and the error names that row. eprintln!( "fpsmaxxing-experiment-runner: the lifecycle failed after the trial was journaled: {source}" ); - match trial_id { - Some(id) => eprintln!(" the trial recording it is {id}"), - None => eprintln!(" the trial recording it could not be journaled"), + match (trial_id, journal_error) { + (Some(id), _) => eprintln!(" the trial recording it is {id}"), + (None, Some(error)) => { + eprintln!(" the trial recording it could not be journaled: {error}"); + } + (None, None) => eprintln!(" the trial recording it could not be journaled"), } return Ok(ExitCode::FAILURE); } diff --git a/apps/experiment-runner/src/runner.rs b/apps/experiment-runner/src/runner.rs index f78725e..4ff8e3a 100644 --- a/apps/experiment-runner/src/runner.rs +++ b/apps/experiment-runner/src/runner.rs @@ -16,7 +16,9 @@ //! still recorded, as a [`LifecycleFailure`] carried by the same record as the //! measurements that authorized the apply, and is also returned to the caller //! together with the identifier that record was stored under, so the caller -//! addresses its own row rather than the journal's last one. +//! addresses its own row rather than the journal's last one - or, when the +//! journal itself refused the write, with the reason those measurements were +//! lost. //! Crash safety for the window between the mutation and that record stays with //! the lifecycle journal's write-ahead `apply-intent` stage (ADR 0002). //! @@ -40,8 +42,8 @@ use std::num::NonZeroU32; use fpsmaxxing_contracts::{ Decision, DecisionBounds, ExperimentSpec, MAX_DECISION_ERRORS, MAX_DECISION_POWER_W, - MAX_DECISION_TEMPERATURE_C, MAX_LEASE_SECONDS, MAX_SAMPLES, MetricSample, ProviderManifest, - Verdict, + MAX_DECISION_TEMPERATURE_C, MAX_HYPOTHESIS_CHARS, MAX_LEASE_SECONDS, MAX_SAMPLES, MetricSample, + ProviderManifest, Verdict, }; use fpsmaxxing_control_plane::{ControlPlane, ControlPlaneError, LifecycleResult, MAX_MOCK_VALUE}; use serde::{Deserialize, Serialize}; @@ -70,11 +72,22 @@ pub enum RunnerError { /// reading back the last identifier the journal holds would be wrong: the /// trial journal is shared, and a concurrent runner can append between the /// write and the read. - #[error("lifecycle failed after {}: {source}", journaled_as(*trial_id))] + /// + /// Exactly one of `trial_id` and `journal_error` is set: journaling the + /// record either produced an identifier or failed with a reason. Losing the + /// measurements is the graver of the two conditions, so the reason it was + /// lost travels with the error rather than being reported out of band, even + /// though the lifecycle error stays the primary one. + #[error("lifecycle failed after {}: {source}", journaled_as(*trial_id, journal_error.as_deref()))] LifecycleFailed { /// Identifier of the journaled trial, absent only when journaling the - /// record failed too; that failure is traced to stderr. + /// record failed too. trial_id: Option, + /// Why the record could not be journaled, present only when it was not. + /// + /// Boxed so carrying a second broker error does not widen every + /// `Result` this module returns to the size of two of them. + journal_error: Option>, /// The broker error that ended the lifecycle. #[source] source: ControlPlaneError, @@ -219,8 +232,8 @@ pub struct ReplayOutcome { pub recorded: Verdict, /// The verdict recomputed from the journaled samples and bounds. pub recomputed: Verdict, - /// Whether the journaled record passes the policy gate and agrees with the - /// spec it carries; see [`replay_trial`]. + /// Whether the journaled record passes the policy gate as it stands now and + /// agrees with the spec it carries; see [`replay_trial`]. pub policy_legal: bool, /// Which gate the record tripped, absent when `policy_legal` holds. /// @@ -258,7 +271,7 @@ impl ReplayOutcome { /// journaled, as a [`LifecycleFailed`](RunnerError::LifecycleFailed) naming the /// identifier that record was stored under; if that write also fails, the /// lifecycle error still takes precedence, the identifier is absent, and the -/// journal failure is traced to stderr. +/// same error carries why the record was lost. pub fn run_trial( plane: &mut ControlPlane, spec: &ExperimentSpec, @@ -299,16 +312,15 @@ pub fn run_trial( }; let journaled = plane.record_trial(&record); if let Some(Err(source)) = outcome { - let trial_id = match journaled { - Ok(id) => Some(id), - Err(journal_error) => { - eprintln!( - "fpsmaxxing-experiment-runner: could not journal the trial whose lifecycle failed: {journal_error}" - ); - None - } + let (trial_id, journal_error) = match journaled { + Ok(id) => (Some(id), None), + Err(journal_error) => (None, Some(Box::new(journal_error))), }; - return Err(RunnerError::LifecycleFailed { trial_id, source }); + return Err(RunnerError::LifecycleFailed { + trial_id, + journal_error, + source, + }); } Ok(StoredTrial { id: journaled?, @@ -364,13 +376,19 @@ pub fn replay_trial(plane: &ControlPlane, id: i64) -> Result Result<(), RunnerError> { } /// Renders where a failed lifecycle's trial record came to rest. -fn journaled_as(trial_id: Option) -> String { - match trial_id { - Some(id) => format!("journaling trial {id}"), - None => "failing to journal its trial".to_owned(), +fn journaled_as(trial_id: Option, journal_error: Option<&ControlPlaneError>) -> String { + match (trial_id, journal_error) { + (Some(id), _) => format!("journaling trial {id}"), + (None, Some(error)) => format!("failing to journal its trial: {error}"), + (None, None) => "failing to journal its trial".to_owned(), } } @@ -517,8 +536,11 @@ fn validate(manifest: &ProviderManifest, spec: &ExperimentSpec) -> Result Result { return Err(ControlPlaneError::UnknownCapability(spec.target.capability_id.clone()).into()); } validate_bounds(&spec.bounds)?; + let hypothesis = spec.hypothesis.chars().count(); + if hypothesis > MAX_HYPOTHESIS_CHARS as usize { + return Err(RunnerError::InvalidSpec(format!( + "hypothesis is {hypothesis} characters, above the {MAX_HYPOTHESIS_CHARS} ceiling" + ))); + } let min_samples = spec.bounds.min_samples.get(); for (label, count) in [ ("warmup_samples", spec.warmup_samples), @@ -661,7 +689,8 @@ mod tests { use super::{ ExperimentSpec, MAX_DECISION_ERRORS, MAX_DECISION_POWER_W, MAX_DECISION_TEMPERATURE_C, - MAX_LEASE_SECONDS, MAX_MOCK_VALUE, MAX_SAMPLES, ProviderManifest, RunnerError, validate, + MAX_HYPOTHESIS_CHARS, MAX_LEASE_SECONDS, MAX_MOCK_VALUE, MAX_SAMPLES, ProviderManifest, + RunnerError, validate, }; /// A manifest advertising only the knob the measurement model describes. @@ -817,6 +846,25 @@ mod tests { assert!(rejection(&leased).contains("lease_seconds")); } + #[test] + fn rejects_a_hypothesis_above_the_policy_bound() { + // The hypothesis is free text the runner writes verbatim into one + // durable row, so the row is not sized by whatever the author sent. + // The ceiling counts characters, as the schema's maxLength does. + let ceiling = MAX_HYPOTHESIS_CHARS as usize; + let mut verbose = spec(2, 5, 5, 3); + verbose.hypothesis = "\u{e9}".repeat(ceiling); + assert!(validate(&manifest(), &verbose).is_ok()); + + verbose.hypothesis = "\u{e9}".repeat(ceiling + 1); + let message = rejection(&verbose); + assert!(message.contains("hypothesis"), "{message}"); + assert!( + message.contains(&format!("{} characters", ceiling + 1)), + "the ceiling counts characters rather than bytes: {message}" + ); + } + #[test] fn rejects_a_capability_the_provider_does_not_advertise() { // The measurement model only describes the mock knob, so a target the diff --git a/apps/experiment-runner/tests/integration.rs b/apps/experiment-runner/tests/integration.rs index 64a8799..5530562 100644 --- a/apps/experiment-runner/tests/integration.rs +++ b/apps/experiment-runner/tests/integration.rs @@ -4,20 +4,23 @@ //! the mock provider through the broker: a promoted experiment that runs the //! full lifecycle, a rejected experiment that is never applied and leaves the //! baseline untouched, a promoted experiment whose lifecycle the broker refuses -//! but which is still journaled, and a replay from the durable journal alone - -//! reopened as a fresh handle - that reproduces the recorded verdict exactly. +//! but which is still journaled, the same refusal when the journal cannot take +//! the record either, and a replay from the durable journal alone - reopened as +//! a fresh handle - that reproduces the recorded verdict exactly. //! The final test states the MVP acceptance criterion directly. use std::num::{NonZeroU32, NonZeroU64}; use fpsmaxxing_contracts::{ CapabilityDescriptor, ChangeRequest, Decision, DecisionBounds, ExperimentSpec, - MAX_LEASE_SECONDS, Persistence, ProviderManifest, RiskClass, StateSnapshot, VerdictReason, + MAX_HYPOTHESIS_CHARS, MAX_LEASE_SECONDS, Persistence, ProviderManifest, RiskClass, + StateSnapshot, VerdictReason, }; use fpsmaxxing_control_plane::{ControlPlane, ControlPlaneError, MAX_MOCK_VALUE}; use fpsmaxxing_experiment_runner::{RunnerError, TrialRecord, evaluate, replay_trial, run_trial}; use fpsmaxxing_mock_provider::MockProvider; use fpsmaxxing_provider_sdk::{Provider, ProviderError}; +use rusqlite::Connection; use serde_json::json; use tempfile::NamedTempFile; @@ -236,13 +239,22 @@ fn a_promoted_trial_survives_a_lifecycle_the_broker_refuses() { // The measurements that authorized the promotion are journaled with the // refusal, in one append, and the error names the row it was written to, so // the caller addresses its own trial instead of the journal's last one. - let RunnerError::LifecycleFailed { trial_id, source } = error else { + let RunnerError::LifecycleFailed { + trial_id, + journal_error, + source, + } = error + else { panic!("a refused lifecycle reports the trial it journaled, got {reported}"); }; assert!( matches!(source, ControlPlaneError::PolicyDenied(_)), "{source:?}" ); + assert!( + journal_error.is_none(), + "the record was stored, so nothing explains a loss: {journal_error:?}" + ); let trial_id = trial_id.expect("the refused lifecycle journaled its trial"); assert!( reported.contains(&format!("trial {trial_id}")), @@ -281,6 +293,59 @@ fn a_promoted_trial_survives_a_lifecycle_the_broker_refuses() { assert_eq!(current_value(&plane), 10); } +#[test] +fn a_refused_lifecycle_reports_why_its_trial_could_not_be_journaled() { + let journal = NamedTempFile::new().expect("temp journal"); + let mut plane = ControlPlane::open( + Box::new(ApprovalGatedProvider(MockProvider::new(10))), + journal.path(), + ) + .expect("open"); + + // Take the trial table away behind the broker's back, so the append that + // records the refused promotion fails too. Losing the measurements that + // authorized a mutation is the graver of the two failures, so the reason + // travels with the error rather than being reported out of band. + Connection::open(journal.path()) + .expect("second journal handle") + .execute_batch("DROP TABLE experiment_trials") + .expect("the trial table should drop"); + + let error = + run_trial(&mut plane, &spec_for(40, 5.0, 80.0)).expect_err("policy should deny the change"); + let reported = error.to_string(); + let RunnerError::LifecycleFailed { + trial_id, + journal_error, + source, + } = error + else { + panic!("a refused lifecycle reports how its trial was journaled, got {reported}"); + }; + + // The lifecycle error stays the primary one. + assert!( + matches!(source, ControlPlaneError::PolicyDenied(_)), + "{source:?}" + ); + assert!( + trial_id.is_none(), + "no identifier exists for a record that was never stored" + ); + let journal_error = journal_error.expect("a lost record reports why it was lost"); + assert!( + matches!(*journal_error, ControlPlaneError::Journal(_)), + "a storage fault is distinguishable from a serialization one: {journal_error:?}" + ); + assert!( + reported.contains("failing to journal its trial") && reported.contains("experiment_trials"), + "the reported error names both failures: {reported}" + ); + + // Nothing reached the provider, so the baseline is untouched. + assert_eq!(current_value(&plane), 10); +} + #[test] fn a_trial_replays_from_the_journal_alone_with_an_identical_verdict() { let journal = NamedTempFile::new().expect("temp journal"); @@ -557,6 +622,11 @@ fn a_replay_reports_a_record_the_policy_gate_would_refuse() { payload["spec"]["target"]["lease_seconds"] = json!(MAX_LEASE_SECONDS + 1); payload }), + ("a hypothesis above the policy ceiling", "hypothesis", { + let mut payload = recorded.clone(); + payload["spec"]["hypothesis"] = json!("x".repeat(MAX_HYPOTHESIS_CHARS as usize + 1)); + payload + }), ( "a candidate value contradicting its own spec", "candidate_value", diff --git a/crates/contracts/src/lib.rs b/crates/contracts/src/lib.rs index bc90f7f..9b3435b 100644 --- a/crates/contracts/src/lib.rs +++ b/crates/contracts/src/lib.rs @@ -71,7 +71,9 @@ pub struct ProviderManifest { /// it is mirrored as `maximum` on `lease_seconds` in /// `schemas/experiment.schema.json`, applied by the broker policy in /// `crates/control-plane` before a lifecycle runs, and applied again by the -/// experiment runner, which bounds a spec before it measures anything. +/// experiment runner, which bounds a spec before it measures anything. Callers +/// that publish their own request schema, such as the gateway's advertised tool +/// input, state the bound independently. pub const MAX_LEASE_SECONDS: u64 = 300; /// A requested provider change after policy validation. @@ -197,11 +199,22 @@ pub struct DecisionBounds { /// mirrored as `maximum` in `schemas/experiment.schema.json`. pub const MAX_SAMPLES: u32 = 10_000; +/// Inclusive ceiling on an [`ExperimentSpec`]'s hypothesis, in characters. +/// +/// The hypothesis is free text an agent authors and the runner writes verbatim +/// into a single durable trial row, so it is bounded like every other field of +/// the spec rather than sizing that row by whatever the author sends. The +/// ceiling is mirrored as `maxLength` in `schemas/experiment.schema.json`, which +/// counts Unicode characters, so the runner counts them the same way. +pub const MAX_HYPOTHESIS_CHARS: u32 = 4_096; + /// A declarative, typed experiment the runner can execute, journal, and replay. #[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] #[serde(deny_unknown_fields)] pub struct ExperimentSpec { - /// Human-authored hypothesis the trial tests. + /// Human-authored hypothesis the trial tests; at most + /// [`MAX_HYPOTHESIS_CHARS`] characters. + #[schemars(length(max = MAX_HYPOTHESIS_CHARS))] pub hypothesis: String, /// Bounded, policy-checkable capability change under test. pub target: ChangeRequest, @@ -274,9 +287,9 @@ mod tests { use super::{ CapabilityDescriptor, ChangeRequest, Decision, DecisionBounds, ExperimentSpec, - MAX_DECISION_ERRORS, MAX_DECISION_POWER_W, MAX_DECISION_TEMPERATURE_C, MAX_LEASE_SECONDS, - MAX_SAMPLES, MetricSample, MetricSummary, NonZeroU32, NonZeroU64, Persistence, - ProviderManifest, RiskClass, Verdict, VerdictReason, + MAX_DECISION_ERRORS, MAX_DECISION_POWER_W, MAX_DECISION_TEMPERATURE_C, + MAX_HYPOTHESIS_CHARS, MAX_LEASE_SECONDS, MAX_SAMPLES, MetricSample, MetricSummary, + NonZeroU32, NonZeroU64, Persistence, ProviderManifest, RiskClass, Verdict, VerdictReason, }; const CAPABILITY_SCHEMA: &str = include_str!("../../../schemas/capability.schema.json"); @@ -695,6 +708,26 @@ mod tests { } } + #[test] + fn the_hypothesis_is_bounded_like_the_schema() { + // The hypothesis is free text an agent authors and the runner journals + // verbatim, so its ceiling is declared here and mirrored by the schema + // the agent writes against. + let checked_in: Value = + serde_json::from_str(EXPERIMENT_SCHEMA).expect("experiment schema should parse"); + assert_eq!( + checked_in["properties"]["hypothesis"]["maxLength"], + json!(MAX_HYPOTHESIS_CHARS) + ); + + let generated = serde_json::to_value(schemars::schema_for!(ExperimentSpec)) + .expect("generated schema should serialize"); + assert_eq!( + generated["properties"]["hypothesis"]["maxLength"], + json!(MAX_HYPOTHESIS_CHARS) + ); + } + #[test] fn decision_bounds_are_bounded_like_the_schema() { // The evaluator's thresholds arrive in the spec, so the policy envelope diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 19ad33c..81e8282 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -32,8 +32,9 @@ The runner controls workload setup, warmup, repeated measurements, cooldown, cor In the safe alpha (`apps/experiment-runner`) the runner measures a baseline and a candidate against a deterministic model, then a pure immutable evaluator returns a promote or reject verdict from the recorded samples and fixed bounds alone - no clock, no LLM, no I/O. Every trial is journaled as a self-describing, versioned record so it can be replayed and re-evaluated from the journal without the original conversation. The record is appended once, after the broker lifecycle it authorized has finished and carrying either that lifecycle's outcome or the error the broker returned, so a promotion the broker refuses still leaves the measurements that authorized it and no API can rewrite a recorded verdict. -Sample counts are bounded by the spec schema and rechecked by the runner before any measurement runs, the target must name the one capability the measurement model describes and the accepted provider advertises so unknown hardware fails closed before anything is measured or journaled, the candidate and the provider's own baseline are both held to the knob ceiling the broker policy enforces later, the target's TTL lease is held to the ceiling the broker enforces on every change request, and the spec's decision bounds are intersected with a policy-owned envelope - declared once in `crates/contracts`, mirrored in `schemas/experiment.schema.json`, and re-checked on replay - so a spec can tighten its own safety gate but never loosen it. -Replay applies that whole gate again to the journaled record rather than the bounds alone, so a row whose spec, capability, declared sample counts, candidate value, TTL lease, or baseline was rewritten after the fact is reported as outside policy even when re-evaluating it reproduces the recorded verdict; the recorded samples themselves are not re-derived, so detecting a rewrite of the measurements together with the verdict they imply needs a signed or hash-chained journal, which the alpha does not have. +Sample counts and the hypothesis text are bounded by the spec schema and rechecked by the runner before any measurement runs, the target must name the one capability the measurement model describes and the accepted provider advertises so unknown hardware fails closed before anything is measured or journaled, the candidate and the provider's own baseline are both held to the knob ceiling the broker policy enforces later, the target's TTL lease is held to the ceiling the broker enforces on every change request, and the spec's decision bounds are intersected with a policy-owned envelope - declared once in `crates/contracts`, mirrored in `schemas/experiment.schema.json`, and re-checked on replay - so a spec can tighten its own safety gate but never loosen it. +Replay applies that whole gate again to the journaled record rather than the bounds alone, so a row whose spec, capability, hypothesis, declared sample counts, candidate value, TTL lease, or baseline was rewritten after the fact is reported as outside policy even when re-evaluating it reproduces the recorded verdict; the gate applied is the current one, so tightening a policy constant deliberately flags archived rows recorded under the looser ceiling (ADR 0002). +The recorded samples themselves are not re-derived, so detecting a rewrite of the measurements together with the verdict they imply needs a signed or hash-chained journal, which the alpha does not have. Because mock capabilities are leased and the broker lifecycle always rolls back, the verdict gates whether the candidate is applied at all rather than whether it persists; durable keep-or-rollback awaits the privileged broker. ### Provider sidecars diff --git a/docs/adr/0002-alpha-experiment-journal.md b/docs/adr/0002-alpha-experiment-journal.md index 8cb6125..d82e5bb 100644 --- a/docs/adr/0002-alpha-experiment-journal.md +++ b/docs/adr/0002-alpha-experiment-journal.md @@ -24,12 +24,15 @@ The write-ahead apply intent makes a crash between mutation and journaling disti A per-stage two-phase protocol would duplicate that machinery for the mock-only alpha before the broker owns the transaction log. Trial records carry a `schema_version` so a future field addition is a version bump a reader can refuse rather than a silent misread of journaled history. The version gate only catches a writer that bumps it, so the record types also reject unknown fields: a row carrying a field this build does not know, under a version it does, means a divergent writer or a rewritten row and fails the replay rather than decoding with that field dropped. -Replay re-runs the policy gate over the journaled spec as well - capability, sample counts, decision bounds, candidate value, and TTL lease - and cross-checks the record's redundant fields against that spec, because a rewritten row re-evaluates to the verdict it carries and so is invisible to a verdict comparison alone. +Replay re-runs the policy gate over the journaled spec as well - capability, hypothesis, sample counts, decision bounds, candidate value, and TTL lease - and cross-checks the record's redundant fields against that spec, because a rewritten row re-evaluates to the verdict it carries and so is invisible to a verdict comparison alone. A row whose thresholds were widened, whose target was pointed at a capability the measurement model never described, whose samples contradict the counts its spec declared, or whose lifecycle fields contradict its own decision is reported as outside policy even when the recomputed verdict matches. The lifecycle cross-check matters because the trial row is the only auditable statement that a promotion reached the provider: a promotion carries exactly one of the lifecycle outcome and the broker error, and a rejection carries neither. -That detection is structural only: replay checks the spec, the capability, the declared sample counts, the candidate value, the TTL lease, the baseline ceiling, and the lifecycle fields the decision implies, but does not re-derive the recorded samples, so a rewrite of the measurements together with the verdict they imply passes both checks. +That detection is structural only: replay checks the spec, the capability, the hypothesis, the declared sample counts, the candidate value, the TTL lease, the baseline ceiling, and the lifecycle fields the decision implies, but does not re-derive the recorded samples, so a rewrite of the measurements together with the verdict they imply passes both checks. Detecting a coherent rewrite of that kind requires anchoring each row outside itself - a signed or hash-chained journal - which is deliberately out of scope for the alpha. Replay reads the journaled capability against the constant the measurement model describes rather than the attached provider's manifest, so an archived journal audits the same way under any provider instead of raising a false tamper alarm. +The policy constants themselves are treated the other way round: `policy_legal` reports the record against the policy in force now, not the policy in force when it was written. +Tightening a constant - `MAX_LEASE_SECONDS`, `MAX_MOCK_VALUE`, `MAX_SAMPLES`, `MAX_HYPOTHESIS_CHARS`, or a decision-bound ceiling - therefore flags every archived row recorded under the looser ceiling, which is the intended reading: those trials are outside the envelope the alpha now permits, and an auditor wants them surfaced rather than grandfathered. +Because that is a change in what the journal means rather than in what a row contains, a policy-constant change is a `TRIAL_RECORD_VERSION` bump, so a flagged archive is attributable to the constant that moved. ## Consequences diff --git a/schemas/experiment.schema.json b/schemas/experiment.schema.json index 11f40f9..586765d 100644 --- a/schemas/experiment.schema.json +++ b/schemas/experiment.schema.json @@ -13,7 +13,7 @@ "bounds" ], "properties": { - "hypothesis": { "type": "string", "minLength": 1 }, + "hypothesis": { "type": "string", "minLength": 1, "maxLength": 4096 }, "target": { "$ref": "#/$defs/change_request" }, "warmup_samples": { "type": "integer", "minimum": 0, "maximum": 10000 }, "baseline_samples": { "type": "integer", "minimum": 1, "maximum": 10000 }, From da57247df8045686591b0d9b4ef8995136ad9ac6 Mon Sep 17 00:00:00 2001 From: Jerry Xiao Date: Mon, 27 Jul 2026 10:06:13 +0000 Subject: [PATCH 12/16] no-mistakes(review): fix policy-drift docs, hypothesis floor, and lost-record lifecycle --- apps/experiment-runner/src/main.rs | 10 +- apps/experiment-runner/src/runner.rs | 111 ++++++++++++++++---- apps/experiment-runner/tests/integration.rs | 66 +++++++++++- crates/contracts/src/lib.rs | 46 +++++--- docs/ARCHITECTURE.md | 4 +- docs/adr/0002-alpha-experiment-journal.md | 9 +- 6 files changed, 196 insertions(+), 50 deletions(-) diff --git a/apps/experiment-runner/src/main.rs b/apps/experiment-runner/src/main.rs index da02fec..5367a72 100644 --- a/apps/experiment-runner/src/main.rs +++ b/apps/experiment-runner/src/main.rs @@ -7,11 +7,11 @@ //! //! A replay that diverges from the journal means the record was tampered with //! or the immutable evaluator drifted, and a replay the policy gate refuses - -//! its capability, hypothesis, sample counts, decision bounds, candidate value, -//! TTL lease, baseline ceiling, the lifecycle fields its decision implies, or -//! the agreement between the record and the spec it carries - means the -//! journaled row is not one this runner would write under the policy in force -//! now. A lifecycle that fails after the trial was measured is the third +//! its capability, hypothesis length, sample counts, decision bounds, candidate +//! value, TTL lease, baseline ceiling, the lifecycle fields its decision +//! implies, or the agreement between the record and the spec it carries - means +//! the journaled row is not one this runner would write under the policy in +//! force now. A lifecycle that fails after the trial was measured is the third //! failure: the trial is journaled anyway and the error names the row, or says //! why the row was lost, which the demo reports. It exits non-zero on any of //! the three rather than reporting a successful run. diff --git a/apps/experiment-runner/src/runner.rs b/apps/experiment-runner/src/runner.rs index 4ff8e3a..4a56ffa 100644 --- a/apps/experiment-runner/src/runner.rs +++ b/apps/experiment-runner/src/runner.rs @@ -18,7 +18,10 @@ //! together with the identifier that record was stored under, so the caller //! addresses its own row rather than the journal's last one - or, when the //! journal itself refused the write, with the reason those measurements were -//! lost. +//! lost. A journal that refuses the record of a lifecycle that *succeeded* +//! reports the same loss the other way round, carrying the outcome that record +//! would have held, so a promotion that reached the provider is never reported +//! as a bare write failure. //! Crash safety for the window between the mutation and that record stays with //! the lifecycle journal's write-ahead `apply-intent` stage (ADR 0002). //! @@ -42,8 +45,8 @@ use std::num::NonZeroU32; use fpsmaxxing_contracts::{ Decision, DecisionBounds, ExperimentSpec, MAX_DECISION_ERRORS, MAX_DECISION_POWER_W, - MAX_DECISION_TEMPERATURE_C, MAX_HYPOTHESIS_CHARS, MAX_LEASE_SECONDS, MAX_SAMPLES, MetricSample, - ProviderManifest, Verdict, + MAX_DECISION_TEMPERATURE_C, MAX_HYPOTHESIS_CHARS, MAX_LEASE_SECONDS, MAX_SAMPLES, + MIN_HYPOTHESIS_CHARS, MetricSample, ProviderManifest, Verdict, }; use fpsmaxxing_control_plane::{ControlPlane, ControlPlaneError, LifecycleResult, MAX_MOCK_VALUE}; use serde::{Deserialize, Serialize}; @@ -92,6 +95,27 @@ pub enum RunnerError { #[source] source: ControlPlaneError, }, + /// The journal refused the record of a trial that already ran. + /// + /// The trial row is the only auditable statement that a promotion reached + /// the provider, so a lost record carries the lifecycle outcome it would + /// have held. Without it a promotion whose knob was written and restored is + /// indistinguishable from a rejection that never touched the provider, and + /// the lifecycle journal cannot tell them apart either: its terminal + /// `completed` record leaves no dangling apply intent for `doctor` to + /// report. + #[error("{}: {source}", trial_lost(lifecycle.is_some()))] + TrialNotJournaled { + /// The outcome the lost record would have carried, present only when a + /// promotion's lifecycle completed. A rejection never runs one. + /// + /// Boxed for the same reason + /// [`LifecycleFailed`](Self::LifecycleFailed) boxes its journal error. + lifecycle: Option>, + /// Why the record could not be journaled. + #[source] + source: ControlPlaneError, + }, /// The experiment specification is outside the bounded alpha envelope. #[error("experiment spec rejected: {0}")] InvalidSpec(String), @@ -271,7 +295,11 @@ impl ReplayOutcome { /// journaled, as a [`LifecycleFailed`](RunnerError::LifecycleFailed) naming the /// identifier that record was stored under; if that write also fails, the /// lifecycle error still takes precedence, the identifier is absent, and the -/// same error carries why the record was lost. +/// same error carries why the record was lost. A journal that refuses the +/// record of a trial whose lifecycle succeeded is reported as a +/// [`TrialNotJournaled`](RunnerError::TrialNotJournaled) carrying that +/// lifecycle's outcome, so the caller is told the provider was mutated rather +/// than only that a write failed. pub fn run_trial( plane: &mut ControlPlane, spec: &ExperimentSpec, @@ -322,10 +350,16 @@ pub fn run_trial( source, }); } - Ok(StoredTrial { - id: journaled?, - record, - }) + let id = match journaled { + Ok(id) => id, + Err(source) => { + return Err(RunnerError::TrialNotJournaled { + lifecycle: record.lifecycle.map(Box::new), + source, + }); + } + }; + Ok(StoredTrial { id, record }) } /// Re-evaluates a journaled trial from the journal alone. @@ -380,15 +414,16 @@ pub fn replay_trial(plane: &ControlPlane, id: i64) -> Result Result Result<(), RunnerError> { Ok(()) } +/// Renders whether a lost trial record would have recorded a lifecycle. +fn trial_lost(applied: bool) -> &'static str { + if applied { + "trial record was lost after its promotion completed the lifecycle" + } else { + "trial record was lost" + } +} + /// Renders where a failed lifecycle's trial record came to rest. fn journaled_as(trial_id: Option, journal_error: Option<&ControlPlaneError>) -> String { match (trial_id, journal_error) { @@ -538,8 +585,10 @@ fn validate(manifest: &ProviderManifest, spec: &ExperimentSpec) -> Result Result { } validate_bounds(&spec.bounds)?; let hypothesis = spec.hypothesis.chars().count(); + if hypothesis < MIN_HYPOTHESIS_CHARS as usize { + return Err(RunnerError::InvalidSpec(format!( + "hypothesis is {hypothesis} characters, below the {MIN_HYPOTHESIS_CHARS} the schema requires" + ))); + } if hypothesis > MAX_HYPOTHESIS_CHARS as usize { return Err(RunnerError::InvalidSpec(format!( "hypothesis is {hypothesis} characters, above the {MAX_HYPOTHESIS_CHARS} ceiling" @@ -689,8 +743,8 @@ mod tests { use super::{ ExperimentSpec, MAX_DECISION_ERRORS, MAX_DECISION_POWER_W, MAX_DECISION_TEMPERATURE_C, - MAX_HYPOTHESIS_CHARS, MAX_LEASE_SECONDS, MAX_MOCK_VALUE, MAX_SAMPLES, ProviderManifest, - RunnerError, validate, + MAX_HYPOTHESIS_CHARS, MAX_LEASE_SECONDS, MAX_MOCK_VALUE, MAX_SAMPLES, MIN_HYPOTHESIS_CHARS, + ProviderManifest, RunnerError, validate, }; /// A manifest advertising only the knob the measurement model describes. @@ -846,6 +900,21 @@ mod tests { assert!(rejection(&leased).contains("lease_seconds")); } + #[test] + fn rejects_an_empty_hypothesis() { + // The schema publishes a floor of one character, and the trial row is + // the durable statement of what a promotion was for, so a blank + // hypothesis must not be measured and journaled as authoritative. + let mut blank = spec(2, 5, 5, 3); + blank.hypothesis = String::new(); + let message = rejection(&blank); + assert!(message.contains("hypothesis"), "{message}"); + assert!( + message.contains(&format!("below the {MIN_HYPOTHESIS_CHARS}")), + "{message}" + ); + } + #[test] fn rejects_a_hypothesis_above_the_policy_bound() { // The hypothesis is free text the runner writes verbatim into one diff --git a/apps/experiment-runner/tests/integration.rs b/apps/experiment-runner/tests/integration.rs index 5530562..0b1f402 100644 --- a/apps/experiment-runner/tests/integration.rs +++ b/apps/experiment-runner/tests/integration.rs @@ -5,8 +5,9 @@ //! full lifecycle, a rejected experiment that is never applied and leaves the //! baseline untouched, a promoted experiment whose lifecycle the broker refuses //! but which is still journaled, the same refusal when the journal cannot take -//! the record either, and a replay from the durable journal alone - reopened as -//! a fresh handle - that reproduces the recorded verdict exactly. +//! the record either, a completed promotion whose record the journal refuses, +//! and a replay from the durable journal alone - reopened as a fresh handle - +//! that reproduces the recorded verdict exactly. //! The final test states the MVP acceptance criterion directly. use std::num::{NonZeroU32, NonZeroU64}; @@ -346,6 +347,58 @@ fn a_refused_lifecycle_reports_why_its_trial_could_not_be_journaled() { assert_eq!(current_value(&plane), 10); } +#[test] +fn a_lost_record_reports_the_lifecycle_the_promotion_had_already_run() { + let journal = NamedTempFile::new().expect("temp journal"); + let mut plane = + ControlPlane::open(Box::new(MockProvider::new(10)), journal.path()).expect("open"); + + // Take the trial table away behind the broker's back, so the promotion runs + // its whole lifecycle and only the record of it is lost. The trial row is + // the only auditable statement that a promotion reached the provider, and + // the lifecycle journal cannot stand in for it - its terminal `completed` + // record leaves no dangling apply intent for `doctor` to report - so the + // outcome travels with the error. + Connection::open(journal.path()) + .expect("second journal handle") + .execute_batch("DROP TABLE experiment_trials") + .expect("the trial table should drop"); + + let error = + run_trial(&mut plane, &spec_for(40, 5.0, 80.0)).expect_err("the record cannot be stored"); + let reported = error.to_string(); + let RunnerError::TrialNotJournaled { lifecycle, source } = error else { + panic!("a lost record reports the lifecycle it would have carried, got {reported}"); + }; + assert!( + matches!(source, ControlPlaneError::Journal(_)), + "{source:?}" + ); + let lifecycle = lifecycle.expect("a completed promotion reports its lifecycle"); + assert_eq!(lifecycle.provider_id, "mock"); + assert!(lifecycle.verified && lifecycle.rolled_back); + assert!( + reported.contains("completed the lifecycle"), + "the reported error says the provider was reached: {reported}" + ); + + // A rejection never runs a lifecycle, so its lost record carries none - the + // distinction the caller could not otherwise draw. + let error = + run_trial(&mut plane, &spec_for(70, 5.0, 80.0)).expect_err("the record cannot be stored"); + let reported = error.to_string(); + let RunnerError::TrialNotJournaled { lifecycle, .. } = error else { + panic!("a lost record is reported as such, got {reported}"); + }; + assert!( + lifecycle.is_none(), + "a rejected candidate never reached the provider: {lifecycle:?}" + ); + + // The leased lifecycle still restored the pre-state on the way out. + assert_eq!(current_value(&plane), 10); +} + #[test] fn a_trial_replays_from_the_journal_alone_with_an_identical_verdict() { let journal = NamedTempFile::new().expect("temp journal"); @@ -627,6 +680,15 @@ fn a_replay_reports_a_record_the_policy_gate_would_refuse() { payload["spec"]["hypothesis"] = json!("x".repeat(MAX_HYPOTHESIS_CHARS as usize + 1)); payload }), + // Blanking the statement of what a promotion was for is the cheaper + // rewrite of the two, so the floor is gated like the ceiling. Only the + // length is checked, though: a hypothesis rewritten within its bounds + // has no redundant copy in the record to contradict it. + ("a blanked hypothesis", "hypothesis", { + let mut payload = recorded.clone(); + payload["spec"]["hypothesis"] = json!(""); + payload + }), ( "a candidate value contradicting its own spec", "candidate_value", diff --git a/crates/contracts/src/lib.rs b/crates/contracts/src/lib.rs index 9b3435b..e0a2f5a 100644 --- a/crates/contracts/src/lib.rs +++ b/crates/contracts/src/lib.rs @@ -205,16 +205,24 @@ pub const MAX_SAMPLES: u32 = 10_000; /// into a single durable trial row, so it is bounded like every other field of /// the spec rather than sizing that row by whatever the author sends. The /// ceiling is mirrored as `maxLength` in `schemas/experiment.schema.json`, which -/// counts Unicode characters, so the runner counts them the same way. +/// counts Unicode characters, so the runner counts them the same way. The floor +/// is [`MIN_HYPOTHESIS_CHARS`]. pub const MAX_HYPOTHESIS_CHARS: u32 = 4_096; +/// Inclusive floor on an [`ExperimentSpec`]'s hypothesis, in characters. +/// +/// A trial row is the durable statement of what a promotion was for, so the +/// hypothesis that states it may not be blank. The floor is mirrored as +/// `minLength` in `schemas/experiment.schema.json`. +pub const MIN_HYPOTHESIS_CHARS: u32 = 1; + /// A declarative, typed experiment the runner can execute, journal, and replay. #[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] #[serde(deny_unknown_fields)] pub struct ExperimentSpec { - /// Human-authored hypothesis the trial tests; at most - /// [`MAX_HYPOTHESIS_CHARS`] characters. - #[schemars(length(max = MAX_HYPOTHESIS_CHARS))] + /// Human-authored hypothesis the trial tests; from + /// [`MIN_HYPOTHESIS_CHARS`] to [`MAX_HYPOTHESIS_CHARS`] characters. + #[schemars(length(min = MIN_HYPOTHESIS_CHARS, max = MAX_HYPOTHESIS_CHARS))] pub hypothesis: String, /// Bounded, policy-checkable capability change under test. pub target: ChangeRequest, @@ -288,8 +296,9 @@ mod tests { use super::{ CapabilityDescriptor, ChangeRequest, Decision, DecisionBounds, ExperimentSpec, MAX_DECISION_ERRORS, MAX_DECISION_POWER_W, MAX_DECISION_TEMPERATURE_C, - MAX_HYPOTHESIS_CHARS, MAX_LEASE_SECONDS, MAX_SAMPLES, MetricSample, MetricSummary, - NonZeroU32, NonZeroU64, Persistence, ProviderManifest, RiskClass, Verdict, VerdictReason, + MAX_HYPOTHESIS_CHARS, MAX_LEASE_SECONDS, MAX_SAMPLES, MIN_HYPOTHESIS_CHARS, MetricSample, + MetricSummary, NonZeroU32, NonZeroU64, Persistence, ProviderManifest, RiskClass, Verdict, + VerdictReason, }; const CAPABILITY_SCHEMA: &str = include_str!("../../../schemas/capability.schema.json"); @@ -711,21 +720,24 @@ mod tests { #[test] fn the_hypothesis_is_bounded_like_the_schema() { // The hypothesis is free text an agent authors and the runner journals - // verbatim, so its ceiling is declared here and mirrored by the schema - // the agent writes against. + // verbatim, so both its ceiling and its floor are declared here and + // mirrored by the schema the agent writes against. The floor matters as + // much as the ceiling: a blank hypothesis leaves a promoted trial with + // no statement of what it was for. let checked_in: Value = serde_json::from_str(EXPERIMENT_SCHEMA).expect("experiment schema should parse"); - assert_eq!( - checked_in["properties"]["hypothesis"]["maxLength"], - json!(MAX_HYPOTHESIS_CHARS) - ); - let generated = serde_json::to_value(schemars::schema_for!(ExperimentSpec)) .expect("generated schema should serialize"); - assert_eq!( - generated["properties"]["hypothesis"]["maxLength"], - json!(MAX_HYPOTHESIS_CHARS) - ); + for schema in [&checked_in, &generated] { + assert_eq!( + schema["properties"]["hypothesis"]["maxLength"], + json!(MAX_HYPOTHESIS_CHARS) + ); + assert_eq!( + schema["properties"]["hypothesis"]["minLength"], + json!(MIN_HYPOTHESIS_CHARS) + ); + } } #[test] diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 81e8282..616c37b 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -33,8 +33,8 @@ The runner controls workload setup, warmup, repeated measurements, cooldown, cor In the safe alpha (`apps/experiment-runner`) the runner measures a baseline and a candidate against a deterministic model, then a pure immutable evaluator returns a promote or reject verdict from the recorded samples and fixed bounds alone - no clock, no LLM, no I/O. Every trial is journaled as a self-describing, versioned record so it can be replayed and re-evaluated from the journal without the original conversation. The record is appended once, after the broker lifecycle it authorized has finished and carrying either that lifecycle's outcome or the error the broker returned, so a promotion the broker refuses still leaves the measurements that authorized it and no API can rewrite a recorded verdict. Sample counts and the hypothesis text are bounded by the spec schema and rechecked by the runner before any measurement runs, the target must name the one capability the measurement model describes and the accepted provider advertises so unknown hardware fails closed before anything is measured or journaled, the candidate and the provider's own baseline are both held to the knob ceiling the broker policy enforces later, the target's TTL lease is held to the ceiling the broker enforces on every change request, and the spec's decision bounds are intersected with a policy-owned envelope - declared once in `crates/contracts`, mirrored in `schemas/experiment.schema.json`, and re-checked on replay - so a spec can tighten its own safety gate but never loosen it. -Replay applies that whole gate again to the journaled record rather than the bounds alone, so a row whose spec, capability, hypothesis, declared sample counts, candidate value, TTL lease, or baseline was rewritten after the fact is reported as outside policy even when re-evaluating it reproduces the recorded verdict; the gate applied is the current one, so tightening a policy constant deliberately flags archived rows recorded under the looser ceiling (ADR 0002). -The recorded samples themselves are not re-derived, so detecting a rewrite of the measurements together with the verdict they imply needs a signed or hash-chained journal, which the alpha does not have. +Replay applies that whole gate again to the journaled record rather than the bounds alone, so a row whose spec, capability, hypothesis length, declared sample counts, candidate value, TTL lease, or baseline was rewritten after the fact is reported as outside policy even when re-evaluating it reproduces the recorded verdict; the gate applied is the current one, so tightening a policy constant deliberately flags archived rows recorded under the looser ceiling (ADR 0002). +The recorded samples themselves are not re-derived, and the hypothesis is held to its length rather than its content, so detecting a rewrite of the measurements together with the verdict they imply - or a rewrite of the hypothesis text within its bounds - needs a signed or hash-chained journal, which the alpha does not have. Because mock capabilities are leased and the broker lifecycle always rolls back, the verdict gates whether the candidate is applied at all rather than whether it persists; durable keep-or-rollback awaits the privileged broker. ### Provider sidecars diff --git a/docs/adr/0002-alpha-experiment-journal.md b/docs/adr/0002-alpha-experiment-journal.md index d82e5bb..102dc51 100644 --- a/docs/adr/0002-alpha-experiment-journal.md +++ b/docs/adr/0002-alpha-experiment-journal.md @@ -24,15 +24,18 @@ The write-ahead apply intent makes a crash between mutation and journaling disti A per-stage two-phase protocol would duplicate that machinery for the mock-only alpha before the broker owns the transaction log. Trial records carry a `schema_version` so a future field addition is a version bump a reader can refuse rather than a silent misread of journaled history. The version gate only catches a writer that bumps it, so the record types also reject unknown fields: a row carrying a field this build does not know, under a version it does, means a divergent writer or a rewritten row and fails the replay rather than decoding with that field dropped. -Replay re-runs the policy gate over the journaled spec as well - capability, hypothesis, sample counts, decision bounds, candidate value, and TTL lease - and cross-checks the record's redundant fields against that spec, because a rewritten row re-evaluates to the verdict it carries and so is invisible to a verdict comparison alone. +Replay re-runs the policy gate over the journaled spec as well - capability, hypothesis length, sample counts, decision bounds, candidate value, and TTL lease - and cross-checks the record's redundant fields against that spec, because a rewritten row re-evaluates to the verdict it carries and so is invisible to a verdict comparison alone. A row whose thresholds were widened, whose target was pointed at a capability the measurement model never described, whose samples contradict the counts its spec declared, or whose lifecycle fields contradict its own decision is reported as outside policy even when the recomputed verdict matches. The lifecycle cross-check matters because the trial row is the only auditable statement that a promotion reached the provider: a promotion carries exactly one of the lifecycle outcome and the broker error, and a rejection carries neither. -That detection is structural only: replay checks the spec, the capability, the hypothesis, the declared sample counts, the candidate value, the TTL lease, the baseline ceiling, and the lifecycle fields the decision implies, but does not re-derive the recorded samples, so a rewrite of the measurements together with the verdict they imply passes both checks. +That detection is structural only: replay checks the spec, the capability, the hypothesis length, the declared sample counts, the candidate value, the TTL lease, the baseline ceiling, and the lifecycle fields the decision implies, but does not re-derive the recorded samples and holds the hypothesis to its length rather than its content. +A rewrite of the measurements together with the verdict they imply therefore passes both checks, and so does a rewrite of the hypothesis text within its length bounds - every other checked field has a second term to disagree with, while the hypothesis is free text with no redundant copy in the record. Detecting a coherent rewrite of that kind requires anchoring each row outside itself - a signed or hash-chained journal - which is deliberately out of scope for the alpha. Replay reads the journaled capability against the constant the measurement model describes rather than the attached provider's manifest, so an archived journal audits the same way under any provider instead of raising a false tamper alarm. The policy constants themselves are treated the other way round: `policy_legal` reports the record against the policy in force now, not the policy in force when it was written. Tightening a constant - `MAX_LEASE_SECONDS`, `MAX_MOCK_VALUE`, `MAX_SAMPLES`, `MAX_HYPOTHESIS_CHARS`, or a decision-bound ceiling - therefore flags every archived row recorded under the looser ceiling, which is the intended reading: those trials are outside the envelope the alpha now permits, and an auditor wants them surfaced rather than grandfathered. -Because that is a change in what the journal means rather than in what a row contains, a policy-constant change is a `TRIAL_RECORD_VERSION` bump, so a flagged archive is attributable to the constant that moved. +A policy-constant change is deliberately not a `TRIAL_RECORD_VERSION` bump, even though it changes what a flagged row means: the version gate is fail-closed and runs before the policy gate, so bumping it would make every archived row fail replay outright instead of replaying as flagged, which destroys the signal rather than attributing it. +The version tracks what a row contains, and a constant change contains nothing new. +Attributing a flagged archive to the constant that moved needs the policy envelope journaled alongside each record, which is a durable-format change deferred with the rest of the journal work to broker promotion; until then the constant's own history is the attribution. ## Consequences From bde4fcf9c89150f714ed0db7bcef968ff84b57b9 Mon Sep 17 00:00:00 2001 From: Jerry Xiao Date: Mon, 27 Jul 2026 10:23:13 +0000 Subject: [PATCH 13/16] no-mistakes(review): bound target parameters, correct sample-integrity docs, report lost record --- apps/experiment-runner/src/main.rs | 36 ++++++-- apps/experiment-runner/src/model.rs | 7 ++ apps/experiment-runner/src/runner.rs | 94 +++++++++++++++++---- apps/experiment-runner/tests/integration.rs | 30 +++++++ docs/ARCHITECTURE.md | 8 +- docs/adr/0002-alpha-experiment-journal.md | 8 +- 6 files changed, 151 insertions(+), 32 deletions(-) diff --git a/apps/experiment-runner/src/main.rs b/apps/experiment-runner/src/main.rs index 5367a72..c33893d 100644 --- a/apps/experiment-runner/src/main.rs +++ b/apps/experiment-runner/src/main.rs @@ -7,14 +7,17 @@ //! //! A replay that diverges from the journal means the record was tampered with //! or the immutable evaluator drifted, and a replay the policy gate refuses - -//! its capability, hypothesis length, sample counts, decision bounds, candidate -//! value, TTL lease, baseline ceiling, the lifecycle fields its decision -//! implies, or the agreement between the record and the spec it carries - means -//! the journaled row is not one this runner would write under the policy in -//! force now. A lifecycle that fails after the trial was measured is the third -//! failure: the trial is journaled anyway and the error names the row, or says -//! why the row was lost, which the demo reports. It exits non-zero on any of -//! the three rather than reporting a successful run. +//! its capability, hypothesis length, sample counts, decision bounds, target +//! parameters, candidate value, TTL lease, baseline ceiling, the lifecycle +//! fields its decision implies, or the agreement between the record and the +//! spec it carries - means the journaled row is not one this runner would write +//! under the policy in force now. The remaining two failures are the trial's +//! own: a lifecycle that fails after the trial was measured, which is journaled +//! anyway so the error names the row or says why the row was lost, and a +//! journal that refuses the record of a trial that already ran, which reports +//! whether the promotion had reached the provider before its record was lost. +//! The demo reports each of the four rather than reporting a successful run, +//! and exits non-zero. use std::{ num::{NonZeroU32, NonZeroU64}, @@ -52,6 +55,23 @@ fn main() -> Result { } return Ok(ExitCode::FAILURE); } + Err(error @ RunnerError::TrialNotJournaled { .. }) => { + // The trial row is the only auditable statement that a promotion + // reached the provider, so the crafted message - which says whether + // one did - is what the demo reports for a lost record. + eprintln!("fpsmaxxing-experiment-runner: {error}"); + if let RunnerError::TrialNotJournaled { + lifecycle: Some(lifecycle), + .. + } = &error + { + eprintln!( + " the lost record would have held: provider {}, verified = {}, rolled_back = {}", + lifecycle.provider_id, lifecycle.verified, lifecycle.rolled_back + ); + } + return Ok(ExitCode::FAILURE); + } Err(error) => return Err(error), }; let verdict = &trial.record.verdict; diff --git a/apps/experiment-runner/src/model.rs b/apps/experiment-runner/src/model.rs index f00c969..b808feb 100644 --- a/apps/experiment-runner/src/model.rs +++ b/apps/experiment-runner/src/model.rs @@ -21,6 +21,13 @@ use fpsmaxxing_contracts::MetricSample; /// about it. The runner refuses such a target rather than measuring it. pub(crate) const MODELED_CAPABILITY_ID: &str = "mock.value"; +/// The change-request parameters [`MODELED_CAPABILITY_ID`] takes. +/// +/// [`measure`] reads the knob value and nothing else, so a target carrying any +/// other key is asking for a change this model does not describe. The runner +/// refuses it rather than passing it through unread. +pub(crate) const MODELED_PARAMETERS: [&str; 1] = ["value"]; + /// Frames per second reported while warming up, before steady state. const COLD_FPS: f64 = 60.0; /// Steady-state frames per second at the lowest knob value. diff --git a/apps/experiment-runner/src/runner.rs b/apps/experiment-runner/src/runner.rs index 4a56ffa..3c0be75 100644 --- a/apps/experiment-runner/src/runner.rs +++ b/apps/experiment-runner/src/runner.rs @@ -414,9 +414,9 @@ pub fn replay_trial(plane: &ControlPlane, id: i64) -> Result Result Result Result { ))); } } + validate_parameters(&spec.target.parameters)?; let value = candidate_value(spec)?; if value > MAX_MOCK_VALUE { return Err(RunnerError::InvalidSpec(format!( @@ -651,6 +664,32 @@ fn validate_spec(spec: &ExperimentSpec) -> Result { Ok(value) } +/// Rejects target parameters the modeled capability does not take. +/// +/// The change request's parameter object is free-form on the wire, and the +/// measurement path reads exactly one key out of it, so every other key would be +/// carried unread into two durable rows: the trial record itself and the +/// lifecycle journal's write-ahead apply intent, which holds the whole request. +/// A spec author would then choose how large those rows are. Holding the object +/// to [`MODELED_PARAMETERS`](model::MODELED_PARAMETERS) bounds it by the same +/// term that decides whether the capability is measurable at all, so the +/// unbounded field is refused rather than sized. +/// +/// A parameter object that is not an object carries no value either, so it is +/// reported as the missing target value it is. +fn validate_parameters(parameters: &Value) -> Result<(), RunnerError> { + let fields = parameters.as_object().ok_or(RunnerError::InvalidTarget)?; + for key in fields.keys() { + if !model::MODELED_PARAMETERS.contains(&key.as_str()) { + return Err(RunnerError::InvalidSpec(format!( + "target parameter {key:?} is not one {} takes", + model::MODELED_CAPABILITY_ID + ))); + } + } + Ok(()) +} + /// Rejects decision bounds that are looser than the policy envelope. /// /// The evaluator is immutable, but its thresholds arrive in the spec, so a spec @@ -985,4 +1024,23 @@ mod tests { Err(RunnerError::InvalidTarget) )); } + + #[test] + fn rejects_target_parameters_the_model_does_not_take() { + // The measurement path reads one key out of a free-form object, so any + // other key is carried unread into the trial row and the lifecycle + // journal's apply intent, sizing both by whatever the author sent. + let mut padded = spec(2, 5, 5, 3); + padded.target.parameters = json!({ "value": 40, "pad": "\u{e9}".repeat(4096) }); + let message = rejection(&padded); + assert!(message.contains("pad"), "{message}"); + + // A parameter object that is not an object carries no value either. + let mut malformed = spec(2, 5, 5, 3); + malformed.target.parameters = json!([{ "value": 40 }]); + assert!(matches!( + validate(&manifest(), &malformed), + Err(RunnerError::InvalidTarget) + )); + } } diff --git a/apps/experiment-runner/tests/integration.rs b/apps/experiment-runner/tests/integration.rs index 0b1f402..7d3ca3a 100644 --- a/apps/experiment-runner/tests/integration.rs +++ b/apps/experiment-runner/tests/integration.rs @@ -464,6 +464,31 @@ fn a_candidate_outside_the_policy_bound_is_refused_before_any_measurement() { assert_eq!(current_value(&plane), 10); } +#[test] +fn target_parameters_the_model_does_not_take_are_refused_before_any_measurement() { + let journal = NamedTempFile::new().expect("temp journal"); + let mut plane = + ControlPlane::open(Box::new(MockProvider::new(10)), journal.path()).expect("open"); + + // The measurement path reads only the knob value out of the target's + // free-form parameter object. An unread key would be written verbatim into + // both the trial row and the lifecycle journal's write-ahead apply intent, + // so the spec author would choose how large those durable rows are. + let mut spec = spec_for(40, 5.0, 80.0); + spec.target.parameters = json!({ "value": 40, "pad": "x".repeat(4096) }); + let error = run_trial(&mut plane, &spec).expect_err("an unread parameter should be refused"); + let RunnerError::InvalidSpec(message) = &error else { + panic!("an unbounded parameter object is refused as an invalid spec, got {error:?}"); + }; + assert!(message.contains("pad"), "{message}"); + + assert!( + plane.trial_ids().expect("trial ids").is_empty(), + "a refused spec journals nothing" + ); + assert_eq!(current_value(&plane), 10); +} + #[test] fn a_lease_outside_the_policy_bound_is_refused_before_any_measurement() { let journal = NamedTempFile::new().expect("temp journal"); @@ -675,6 +700,11 @@ fn a_replay_reports_a_record_the_policy_gate_would_refuse() { payload["spec"]["target"]["lease_seconds"] = json!(MAX_LEASE_SECONDS + 1); payload }), + ("a parameter the model does not take", "pad", { + let mut payload = recorded.clone(); + payload["spec"]["target"]["parameters"]["pad"] = json!("x".repeat(4096)); + payload + }), ("a hypothesis above the policy ceiling", "hypothesis", { let mut payload = recorded.clone(); payload["spec"]["hypothesis"] = json!("x".repeat(MAX_HYPOTHESIS_CHARS as usize + 1)); diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 616c37b..ab4c873 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -32,9 +32,11 @@ The runner controls workload setup, warmup, repeated measurements, cooldown, cor In the safe alpha (`apps/experiment-runner`) the runner measures a baseline and a candidate against a deterministic model, then a pure immutable evaluator returns a promote or reject verdict from the recorded samples and fixed bounds alone - no clock, no LLM, no I/O. Every trial is journaled as a self-describing, versioned record so it can be replayed and re-evaluated from the journal without the original conversation. The record is appended once, after the broker lifecycle it authorized has finished and carrying either that lifecycle's outcome or the error the broker returned, so a promotion the broker refuses still leaves the measurements that authorized it and no API can rewrite a recorded verdict. -Sample counts and the hypothesis text are bounded by the spec schema and rechecked by the runner before any measurement runs, the target must name the one capability the measurement model describes and the accepted provider advertises so unknown hardware fails closed before anything is measured or journaled, the candidate and the provider's own baseline are both held to the knob ceiling the broker policy enforces later, the target's TTL lease is held to the ceiling the broker enforces on every change request, and the spec's decision bounds are intersected with a policy-owned envelope - declared once in `crates/contracts`, mirrored in `schemas/experiment.schema.json`, and re-checked on replay - so a spec can tighten its own safety gate but never loosen it. -Replay applies that whole gate again to the journaled record rather than the bounds alone, so a row whose spec, capability, hypothesis length, declared sample counts, candidate value, TTL lease, or baseline was rewritten after the fact is reported as outside policy even when re-evaluating it reproduces the recorded verdict; the gate applied is the current one, so tightening a policy constant deliberately flags archived rows recorded under the looser ceiling (ADR 0002). -The recorded samples themselves are not re-derived, and the hypothesis is held to its length rather than its content, so detecting a rewrite of the measurements together with the verdict they imply - or a rewrite of the hypothesis text within its bounds - needs a signed or hash-chained journal, which the alpha does not have. +Sample counts and the hypothesis text are bounded by the spec schema and rechecked by the runner before any measurement runs, the target must name the one capability the measurement model describes and the accepted provider advertises so unknown hardware fails closed before anything is measured or journaled, the free-form parameter object that capability is invoked with is held to the keys it actually takes so nothing unread is journaled, the candidate and the provider's own baseline are both held to the knob ceiling the broker policy enforces later, the target's TTL lease is held to the ceiling the broker enforces on every change request, and the spec's decision bounds are intersected with a policy-owned envelope - declared once in `crates/contracts`, mirrored in `schemas/experiment.schema.json`, and re-checked on replay - so a spec can tighten its own safety gate but never loosen it. +Replay applies that whole gate again to the journaled record rather than the bounds alone, so a row whose spec, capability, hypothesis length, declared sample counts, target parameters, candidate value, TTL lease, or baseline was rewritten after the fact is reported as outside policy even when re-evaluating it reproduces the recorded verdict; the gate applied is the current one, so tightening a policy constant deliberately flags archived rows recorded under the looser ceiling (ADR 0002). +The recorded samples themselves are not re-derived, and the hypothesis is held to its length rather than its content, so a rewrite of the measurements together with the verdict they imply - or of the hypothesis text within its bounds - passes replay. +Re-deriving the samples would in fact work today, because the stand-in model is a pure function of values the record already carries, and it is deliberately not done: real telemetry is not reproducible, which is why samples are journaled verbatim in the first place, so a gate built on the model's reproducibility would have to be deleted the moment that model is replaced. +Measurement-content integrity needs a signed or hash-chained journal instead, which the alpha defers rather than approximates. Because mock capabilities are leased and the broker lifecycle always rolls back, the verdict gates whether the candidate is applied at all rather than whether it persists; durable keep-or-rollback awaits the privileged broker. ### Provider sidecars diff --git a/docs/adr/0002-alpha-experiment-journal.md b/docs/adr/0002-alpha-experiment-journal.md index 102dc51..307c165 100644 --- a/docs/adr/0002-alpha-experiment-journal.md +++ b/docs/adr/0002-alpha-experiment-journal.md @@ -24,12 +24,14 @@ The write-ahead apply intent makes a crash between mutation and journaling disti A per-stage two-phase protocol would duplicate that machinery for the mock-only alpha before the broker owns the transaction log. Trial records carry a `schema_version` so a future field addition is a version bump a reader can refuse rather than a silent misread of journaled history. The version gate only catches a writer that bumps it, so the record types also reject unknown fields: a row carrying a field this build does not know, under a version it does, means a divergent writer or a rewritten row and fails the replay rather than decoding with that field dropped. -Replay re-runs the policy gate over the journaled spec as well - capability, hypothesis length, sample counts, decision bounds, candidate value, and TTL lease - and cross-checks the record's redundant fields against that spec, because a rewritten row re-evaluates to the verdict it carries and so is invisible to a verdict comparison alone. +Replay re-runs the policy gate over the journaled spec as well - capability, hypothesis length, sample counts, decision bounds, target parameters, candidate value, and TTL lease - and cross-checks the record's redundant fields against that spec, because a rewritten row re-evaluates to the verdict it carries and so is invisible to a verdict comparison alone. A row whose thresholds were widened, whose target was pointed at a capability the measurement model never described, whose samples contradict the counts its spec declared, or whose lifecycle fields contradict its own decision is reported as outside policy even when the recomputed verdict matches. The lifecycle cross-check matters because the trial row is the only auditable statement that a promotion reached the provider: a promotion carries exactly one of the lifecycle outcome and the broker error, and a rejection carries neither. -That detection is structural only: replay checks the spec, the capability, the hypothesis length, the declared sample counts, the candidate value, the TTL lease, the baseline ceiling, and the lifecycle fields the decision implies, but does not re-derive the recorded samples and holds the hypothesis to its length rather than its content. +That detection is structural only: replay checks the spec, the capability, the hypothesis length, the declared sample counts, the target parameters, the candidate value, the TTL lease, the baseline ceiling, and the lifecycle fields the decision implies, but does not re-derive the recorded samples and holds the hypothesis to its length rather than its content. A rewrite of the measurements together with the verdict they imply therefore passes both checks, and so does a rewrite of the hypothesis text within its length bounds - every other checked field has a second term to disagree with, while the hypothesis is free text with no redundant copy in the record. -Detecting a coherent rewrite of that kind requires anchoring each row outside itself - a signed or hash-chained journal - which is deliberately out of scope for the alpha. +Re-deriving the samples is available in this alpha and deliberately unused: the stand-in model is a pure function of the knob value and the two counts, all of which the record carries, so the recorded sets could be regenerated and compared outright. +That check cannot survive what the model stands in for - real `PresentMon` telemetry is not reproducible, which is exactly why samples are journaled verbatim - so it would have to be deleted when the model is replaced, leaving an archive audited under it with no gate at all. +Measurement-content integrity is therefore left to anchoring each row outside itself - a signed or hash-chained journal - which is deliberately out of scope for the alpha rather than approximated by a check with a shorter life than the journal it guards. Replay reads the journaled capability against the constant the measurement model describes rather than the attached provider's manifest, so an archived journal audits the same way under any provider instead of raising a false tamper alarm. The policy constants themselves are treated the other way round: `policy_legal` reports the record against the policy in force now, not the policy in force when it was written. Tightening a constant - `MAX_LEASE_SECONDS`, `MAX_MOCK_VALUE`, `MAX_SAMPLES`, `MAX_HYPOTHESIS_CHARS`, or a decision-bound ceiling - therefore flags every archived row recorded under the looser ceiling, which is the intended reading: those trials are outside the envelope the alpha now permits, and an auditor wants them surfaced rather than grandfathered. From 3b3d81338e56b0b2cc82f04db30da99a789a4ec2 Mon Sep 17 00:00:00 2001 From: Jerry Xiao Date: Mon, 27 Jul 2026 10:35:51 +0000 Subject: [PATCH 14/16] no-mistakes(review): correct replay-gate docs on candidate and baseline detection --- apps/experiment-runner/src/main.rs | 13 ++++++--- apps/experiment-runner/src/runner.rs | 32 +++++++++++++++-------- docs/ARCHITECTURE.md | 5 ++-- docs/adr/0002-alpha-experiment-journal.md | 5 ++-- 4 files changed, 36 insertions(+), 19 deletions(-) diff --git a/apps/experiment-runner/src/main.rs b/apps/experiment-runner/src/main.rs index c33893d..09a1174 100644 --- a/apps/experiment-runner/src/main.rs +++ b/apps/experiment-runner/src/main.rs @@ -8,10 +8,15 @@ //! A replay that diverges from the journal means the record was tampered with //! or the immutable evaluator drifted, and a replay the policy gate refuses - //! its capability, hypothesis length, sample counts, decision bounds, target -//! parameters, candidate value, TTL lease, baseline ceiling, the lifecycle -//! fields its decision implies, or the agreement between the record and the -//! spec it carries - means the journaled row is not one this runner would write -//! under the policy in force now. The remaining two failures are the trial's +//! parameters, TTL lease, the candidate value the spec's own copy of it +//! implies, the ceiling the baseline is held under, the lifecycle fields its +//! decision implies, or the agreement between the record and the spec it +//! carries - means the journaled row is not one this runner would write under +//! the policy in force now. A clean replay is not the converse: the samples are +//! not re-derived and nothing ties them to the values they were taken at, so a +//! rewrite of the measurements with the verdict they imply, of the hypothesis +//! within its length bounds, or of the candidate and baseline values within +//! policy passes both reads. The remaining two failures are the trial's //! own: a lifecycle that fails after the trial was measured, which is journaled //! anyway so the error names the row or says why the row was lost, and a //! journal that refuses the record of a trial that already ran, which reports diff --git a/apps/experiment-runner/src/runner.rs b/apps/experiment-runner/src/runner.rs index 3c0be75..27721c9 100644 --- a/apps/experiment-runner/src/runner.rs +++ b/apps/experiment-runner/src/runner.rs @@ -436,22 +436,32 @@ pub fn replay_trial(plane: &ControlPlane, id: i64) -> Result Date: Mon, 27 Jul 2026 10:55:23 +0000 Subject: [PATCH 15/16] no-mistakes(document): sync docs with deterministic experiment engine and replay gate --- AGENTS.md | 1 + README.md | 6 +- .../tests/acceptance_transcript.rs | 256 ++++++++++++++++++ docs/ARCHITECTURE.md | 1 + docs/IMPLEMENTATION_PLAN.html | 4 +- docs/IMPLEMENTATION_PLAN.md | 7 + docs/README.md | 2 +- docs/threat-model/README.md | 5 + 8 files changed, 279 insertions(+), 3 deletions(-) create mode 100644 apps/experiment-runner/tests/acceptance_transcript.rs diff --git a/AGENTS.md b/AGENTS.md index eef66c7..b8bdb48 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -18,6 +18,7 @@ Start with `docs/README.md`. Read `docs/IMPLEMENTATION_PLAN.md`, `docs/ARCHITECT - Keep provider lifecycle behavior in `crates/provider-sdk`. - Keep the capability registry, policy, broker lifecycle, and experiment journal in `crates/control-plane`. - Keep the independent crash and lease recovery path in `apps/watchdog`; it reads the journal owned by `crates/control-plane` and writes only its own restore-outcome records, never the schema. +- Keep the measurement model, immutable evaluator, and replayable trial records in `apps/experiment-runner`; the evaluator stays a pure function of recorded samples and fixed bounds. - Put provider-specific code in one `sidecars/` package; sidecars may not import each other. - Put non-Rust compatibility processes under `bridges/` and isolate them behind the sidecar protocol. - Do not vendor third-party binaries without confirmed redistribution rights. diff --git a/README.md b/README.md index 2991b4e..259c289 100644 --- a/README.md +++ b/README.md @@ -99,6 +99,7 @@ Try the read-only alpha: cargo test --workspace cargo run -p fpsmaxxing-cli -- doctor cargo run -p fpsmaxxing-mock-provider +cargo run -p fpsmaxxing-experiment-runner printf '%s\n' \ '{"jsonrpc":"2.0","id":1,"method":"tools/list","params":{}}' \ '{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"fpsmaxxing.run_mock_lifecycle","arguments":{"value":42,"lease_seconds":30}}}' \ @@ -111,6 +112,9 @@ Override the journal location with `--journal ` or the `FPSMAXXING_JOURNAL Run the watchdog against the same journal to reclaim leaked experiments: `cargo run -p fpsmaxxing-watchdog -- --once` performs a single expired-lease pass and `--recover-all` rolls back every unclosed experiment after a crash. It accepts the same `--journal ` and `FPSMAXXING_JOURNAL_PATH` overrides, plus `--interval ` for its steady-state poll loop. +The experiment runner measures a baseline and a candidate against a deterministic stand-in for live telemetry, gates the candidate's lifecycle on the immutable evaluator's verdict, journals the trial with its spec, samples, and verdict, then replays it from the journal alone and checks the re-evaluated verdict against the recorded one. +It is a demonstration binary rather than an MCP tool, takes no arguments, and journals to an in-memory SQLite database, so it leaves nothing on disk and exits non-zero if a replay diverges from the journal, falls outside the policy gate, or the broker refuses a promoted lifecycle. + ## Architecture ```text @@ -134,7 +138,7 @@ Start with the [documentation index](docs/README.md). The core references are th ### Can Claude optimize my PC for higher FPS? -That is the intended workflow. A Claude or Codex agent should be able to inspect available capabilities, propose a bounded change, run a controlled game or benchmark workload, and keep the change only when frame time, latency, thermals, and correctness remain within policy. The alpha already runs capability discovery and the bounded snapshot-to-rollback lifecycle over MCP against a mock provider; the measurement-driven keep-or-rollback decision and real hardware providers are not implemented yet. +That is the intended workflow. A Claude or Codex agent should be able to inspect available capabilities, propose a bounded change, run a controlled game or benchmark workload, and keep the change only when frame time, latency, thermals, and correctness remain within policy. The alpha already runs capability discovery and the bounded snapshot-to-rollback lifecycle over MCP against a mock provider, and it promotes or rejects one measured experiment through an immutable evaluator that decides from recorded samples and fixed bounds alone. Real hardware providers, live frame-time measurement, and a promotion that survives its lease are not implemented yet: the alpha measures a deterministic stand-in for telemetry, and the verdict gates whether a candidate is applied at all rather than whether it persists. ### Can an AI safely overclock a GPU? diff --git a/apps/experiment-runner/tests/acceptance_transcript.rs b/apps/experiment-runner/tests/acceptance_transcript.rs new file mode 100644 index 0000000..c8b4fb2 --- /dev/null +++ b/apps/experiment-runner/tests/acceptance_transcript.rs @@ -0,0 +1,256 @@ +//! The MVP acceptance criterion as one auditable transcript. +//! +//! `tests/integration.rs` asserts each behaviour of the experiment engine in +//! isolation. This test walks the whole story once, in the order an operator +//! experiences it, over a single durable journal file, and prints what it +//! observes at each step so the run is readable as evidence rather than only as +//! a pass: a measured experiment is promoted through the broker lifecycle, a +//! second measured experiment is rejected and never reaches the provider, the +//! rows both trials left behind are shown as they are stored in `SQLite`, and a +//! fresh handle on that file alone re-evaluates both trials to the identical +//! verdicts without the specs, the provider state, or any chat history that +//! produced them. + +use std::num::{NonZeroU32, NonZeroU64}; + +use fpsmaxxing_contracts::{ChangeRequest, Decision, DecisionBounds, ExperimentSpec, VerdictReason}; +use fpsmaxxing_control_plane::ControlPlane; +use fpsmaxxing_experiment_runner::{StoredTrial, TrialRecord, replay_trial, run_trial}; +use fpsmaxxing_mock_provider::MockProvider; +use rusqlite::Connection; +use serde_json::json; +use tempfile::NamedTempFile; + +/// The knob value the provider is parked at before either trial runs. +const BASELINE_VALUE: u64 = 10; + +/// Builds a spec driving the mock knob to `candidate` under a fixed envelope. +fn spec_for(candidate: u64) -> ExperimentSpec { + ExperimentSpec { + hypothesis: format!( + "raising mock.value from {BASELINE_VALUE} to {candidate} improves FPS within thermal and power limits" + ), + target: ChangeRequest { + capability_id: "mock.value".to_owned(), + parameters: json!({ "value": candidate }), + lease_seconds: NonZeroU64::new(30).expect("lease is non-zero"), + }, + warmup_samples: 2, + baseline_samples: NonZeroU32::new(5).expect("baseline count is non-zero"), + candidate_samples: NonZeroU32::new(5).expect("candidate count is non-zero"), + bounds: DecisionBounds { + min_samples: NonZeroU32::new(3).expect("min samples is non-zero"), + min_fps_improvement: 5.0, + max_temperature_c: 80.0, + max_power_w: 200.0, + max_errors: 0, + }, + } +} + +/// Reads the mock provider's current knob value through a broker snapshot. +fn current_value(plane: &ControlPlane) -> u64 { + plane + .snapshot() + .expect("snapshot") + .state + .get("value") + .and_then(serde_json::Value::as_u64) + .expect("mock value") +} + +/// Prints the measurements and verdict a trial recorded. +fn report(label: &str, trial: &StoredTrial) { + let verdict = &trial.record.verdict; + println!("[{label}] trial id {}", trial.id); + println!(" hypothesis : {}", trial.record.spec.hypothesis); + println!( + " baseline value : {} -> mean {:.1} fps, {:.1} C, {:.1} W, {} errors over {} samples", + trial.record.baseline_value, + verdict.baseline.mean_fps, + verdict.baseline.max_temperature_c, + verdict.baseline.max_power_w, + verdict.baseline.total_errors, + verdict.baseline.samples + ); + println!( + " candidate value : {} -> mean {:.1} fps, {:.1} C, {:.1} W, {} errors over {} samples", + trial.record.candidate_value, + verdict.candidate.mean_fps, + verdict.candidate.max_temperature_c, + verdict.candidate.max_power_w, + verdict.candidate.total_errors, + verdict.candidate.samples + ); + println!( + " bounds : >= {:.1} fps gain, <= {:.1} C, <= {:.1} W, <= {} errors, >= {} samples", + trial.record.spec.bounds.min_fps_improvement, + trial.record.spec.bounds.max_temperature_c, + trial.record.spec.bounds.max_power_w, + trial.record.spec.bounds.max_errors, + trial.record.spec.bounds.min_samples + ); + println!( + " verdict : {:?} ({:?}), fps_improvement = {:+.1}", + verdict.decision, verdict.reason, verdict.fps_improvement + ); + match (&trial.record.lifecycle, &trial.record.lifecycle_error) { + (Some(lifecycle), _) => println!( + " lifecycle : provider {}, preview {:?}, verified = {}, rolled_back = {}", + lifecycle.provider_id, lifecycle.preview, lifecycle.verified, lifecycle.rolled_back + ), + (None, Some(failure)) => println!( + " lifecycle : refused ({}): {}", + failure.kind, failure.error + ), + (None, None) => println!(" lifecycle : none - the candidate was never applied"), + } +} + +#[test] +fn one_measured_experiment_is_promoted_and_another_rejected_then_both_replay() { + let journal = NamedTempFile::new().expect("temp journal"); + println!("journal file: an on-disk SQLite database\n"); + + let promoted; + let rejected; + { + let mut plane = + ControlPlane::open(Box::new(MockProvider::new(BASELINE_VALUE)), journal.path()) + .expect("open"); + println!("provider parked at mock.value = {}\n", current_value(&plane)); + + // A measured experiment the evaluator promotes: the candidate gains + // 30 fps and stays inside every safety ceiling, so the broker runs the + // full snapshot/preview/apply/verify/rollback lifecycle. + promoted = run_trial(&mut plane, &spec_for(40)).expect("run promoted trial"); + report("promoted", &promoted); + assert_eq!(promoted.record.verdict.decision, Decision::Promote); + assert_eq!(promoted.record.verdict.reason, VerdictReason::Promoted); + let lifecycle = promoted + .record + .lifecycle + .clone() + .expect("a promotion records its lifecycle"); + assert!(lifecycle.verified && lifecycle.rolled_back); + println!( + " provider after : mock.value = {} (the lease restored the pre-state)\n", + current_value(&plane) + ); + assert_eq!(current_value(&plane), BASELINE_VALUE); + + // A measured experiment the evaluator rejects: the candidate gains even + // more fps but drives modeled temperature to 85 C, past the ceiling, so + // it is never applied and the baseline is left exactly as it was. + rejected = run_trial(&mut plane, &spec_for(70)).expect("run rejected trial"); + report("rejected", &rejected); + assert_eq!(rejected.record.verdict.decision, Decision::Reject); + assert_eq!( + rejected.record.verdict.reason, + VerdictReason::TemperatureExceeded + ); + assert!(rejected.record.lifecycle.is_none()); + assert!(rejected.record.lifecycle_error.is_none()); + println!( + " provider after : mock.value = {} (untouched - nothing was applied)\n", + current_value(&plane) + ); + assert_eq!(current_value(&plane), BASELINE_VALUE); + } + + // The rows as the journal actually holds them, read with a plain SQLite + // connection rather than through the control plane, so the durable state is + // shown rather than described. + let raw = Connection::open(journal.path()).expect("open journal directly"); + let mut statement = raw + .prepare("SELECT id, recorded_at, length(payload) FROM experiment_trials ORDER BY id") + .expect("query trials"); + let rows: Vec<(i64, String, i64)> = statement + .query_map([], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?))) + .expect("read trials") + .collect::>() + .expect("read trials"); + println!("persisted experiment_trials rows:"); + for (id, recorded_at, payload_bytes) in &rows { + println!(" id {id}, recorded_at {recorded_at}, payload {payload_bytes} bytes"); + } + assert_eq!( + rows.iter().map(|(id, ..)| *id).collect::>(), + [promoted.id, rejected.id] + ); + + // The stored payload of the promotion, verbatim. Everything re-evaluation + // needs is in this one row: the spec that was run, both measurement sets, + // and the verdict, under a record version a reader can refuse. + let stored_payload: String = raw + .query_row( + "SELECT payload FROM experiment_trials WHERE id = ?1", + [promoted.id], + |row| row.get(0), + ) + .expect("read the promoted payload"); + println!("\nstored payload of trial {}:\n {stored_payload}", promoted.id); + + // Reopen the file as a brand new handle, with a provider parked at an + // unrelated value, and re-evaluate both trials from the journal alone. + let archive = + ControlPlane::open(Box::new(MockProvider::new(0)), journal.path()).expect("reopen"); + println!( + "\nreplay from a fresh handle (provider now at mock.value = {}, no spec in hand, no chat history):", + current_value(&archive) + ); + for (label, trial) in [("promoted", &promoted), ("rejected", &rejected)] { + let stored: TrialRecord = + serde_json::from_value(archive.read_trial(trial.id).expect("read trial")) + .expect("decode trial"); + assert_eq!( + stored, trial.record, + "the journaled record must survive the round trip verbatim" + ); + + let outcome = replay_trial(&archive, trial.id).expect("replay trial"); + println!( + " [{label}] trial {}: recorded {:?} ({:?}) -> recomputed {:?} ({:?}); identical = {}, policy legal = {}", + outcome.trial_id, + outcome.recorded.decision, + outcome.recorded.reason, + outcome.recomputed.decision, + outcome.recomputed.reason, + outcome.is_consistent(), + outcome.policy_legal + ); + assert!( + outcome.is_consistent(), + "replay must reproduce the recorded verdict exactly" + ); + assert!(outcome.policy_legal, "{:?}", outcome.policy_reason); + assert_eq!(outcome.recomputed, trial.record.verdict); + } + + // A clean replay is a real check rather than a rubber stamp: append a copy + // of the promotion whose temperature ceiling was widened after the fact. + // It re-evaluates to the very verdict it carries, so only the policy gate + // replay re-applies can catch it. + let mut widened = archive.read_trial(promoted.id).expect("read trial"); + widened["spec"]["bounds"]["max_temperature_c"] = json!(200.0); + let widened_id = archive + .record_trial(&widened) + .expect("append the widened row"); + let outcome = replay_trial(&archive, widened_id).expect("replay the widened row"); + println!( + " [tampered] trial {}: bounds widened to 200 C; recomputed verdict still identical = {}, but policy legal = {} ({})", + outcome.trial_id, + outcome.is_consistent(), + outcome.policy_legal, + outcome.policy_reason.as_deref().unwrap_or("no reason given") + ); + assert!(outcome.is_consistent()); + assert!(!outcome.policy_legal); + + println!( + "\nacceptance criterion: {} measured experiments reached a decision through the immutable evaluator - {:?} and {:?} - and both re-evaluate identically from the journal alone", + rows.len(), + promoted.record.verdict.decision, + rejected.record.verdict.decision + ); +} diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index dd8f9a0..2576fb3 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -39,6 +39,7 @@ The recorded samples themselves are not re-derived and nothing ties them to the Re-deriving the samples would in fact work today, because the stand-in model is a pure function of values the record already carries, and it is deliberately not done: real telemetry is not reproducible, which is why samples are journaled verbatim in the first place, so a gate built on the model's reproducibility would have to be deleted the moment that model is replaced. Measurement-content integrity needs a signed or hash-chained journal instead, which the alpha defers rather than approximates. Because mock capabilities are leased and the broker lifecycle always rolls back, the verdict gates whether the candidate is applied at all rather than whether it persists; durable keep-or-rollback awaits the privileged broker. +The candidate is also measured before it is applied, which only the pure stand-in model permits: real `PresentMon` or hardware telemetry cannot observe a candidate that was never written, so swapping it in moves the candidate measurement inside the apply-and-lease window and runs the gate after it. ### Provider sidecars diff --git a/docs/IMPLEMENTATION_PLAN.html b/docs/IMPLEMENTATION_PLAN.html index b11b2cb..5dc2018 100644 --- a/docs/IMPLEMENTATION_PLAN.html +++ b/docs/IMPLEMENTATION_PLAN.html @@ -391,7 +391,9 @@

The first vertical slice

"│ ├── capability.schema.json", "│ ├── sidecar.schema.json", "│ ├── change.schema.json", - "│ └── experiment.schema.json", + "│ ├── experiment.schema.json", + "│ ├── verdict.schema.json", + "│ └── metric-sample.schema.json", "├── policies/", "│ ├── risk/", "│ ├── conflicts/", diff --git a/docs/IMPLEMENTATION_PLAN.md b/docs/IMPLEMENTATION_PLAN.md index 710263b..e388750 100644 --- a/docs/IMPLEMENTATION_PLAN.md +++ b/docs/IMPLEMENTATION_PLAN.md @@ -99,6 +99,13 @@ observe Use the LLM for hypothesis generation and explanation. Use deterministic search and statistics for numeric optimization. +The alpha implements this loop deterministically in `apps/experiment-runner` against the mock provider. +A typed spec (`schemas/experiment.schema.json`) declares the hypothesis, the target capability change, the warmup and repeated baseline/candidate sample counts, and the decision bounds. +The immutable evaluator applies one fixed, ordered threshold rule - minimum sample count, then the temperature, power, and error ceilings, then the mean-FPS improvement threshold - and returns a verdict (`schemas/verdict.schema.json`) that gates whether the candidate reaches the broker lifecycle at all. +Every trial is journaled with its spec, recorded samples, and verdict, so it replays and re-evaluates from the journal alone. +Measurement is a deterministic stand-in for `PresentMon` and hardware telemetry, and a leased mock change cannot outlive its lifecycle, so live telemetry, statistical search, and durable promotion remain ahead of this phase. +Because that stand-in depends only on the knob value, the alpha measures the candidate before applying it; real telemetry moves that measurement inside the apply-and-lease window shown above. + ## Safety invariants - No generic shell or raw Registry path reaches the broker. diff --git a/docs/README.md b/docs/README.md index 07fb374..8e10d2f 100644 --- a/docs/README.md +++ b/docs/README.md @@ -17,6 +17,6 @@ Use this page as the documentation entry point. The Markdown documents are canon | --- | --- | | [Provider guide](providers/README.md) | Rules and lifecycle for provider implementations | | [Rust-first ADR](adr/0001-rust-first.md) | Why the control plane uses Rust with one isolated .NET bridge | -| [Alpha journal ADR](adr/0002-alpha-experiment-journal.md) | Write-ahead apply intent, terminal outcomes, and deferred two-phase journaling | +| [Alpha journal ADR](adr/0002-alpha-experiment-journal.md) | Write-ahead apply intent, terminal outcomes, append-only trial records, and deferred two-phase journaling | Repository-wide contributor, security, support, and governance documents remain at the project root so GitHub can discover them automatically. diff --git a/docs/threat-model/README.md b/docs/threat-model/README.md index f45f0df..1d737cb 100644 --- a/docs/threat-model/README.md +++ b/docs/threat-model/README.md @@ -18,6 +18,8 @@ - Process termination leaving persistent changes behind - Malicious or incompatible third-party binaries - Reboot before a candidate configuration is blessed +- A proposed experiment declaring decision thresholds that disarm its own promotion gate +- Rewritten experiment history crediting a change with measurements it never produced ## Required mitigations @@ -28,4 +30,7 @@ - Durable pre-state journal and TTL leases - Independent watchdog and last-known-good baseline - Fail-closed behavior on unknown state or missing telemetry +- A policy-owned decision envelope a proposed experiment may tighten but never loosen +- Append-only trial records re-evaluated and re-gated against current policy on replay +- A signed or hash-chained journal before recorded measurement content itself is trusted - Hardware-in-the-loop fault tests before enabling writes From ef260eacfe754522833ac8f0453ddf47dfcc3522 Mon Sep 17 00:00:00 2001 From: Jerry Xiao Date: Mon, 27 Jul 2026 11:01:53 +0000 Subject: [PATCH 16/16] no-mistakes(lint): format acceptance transcript and split over-long trial tests --- .../tests/acceptance_transcript.rs | 154 ++++++++++-------- apps/experiment-runner/tests/integration.rs | 37 +++-- 2 files changed, 109 insertions(+), 82 deletions(-) diff --git a/apps/experiment-runner/tests/acceptance_transcript.rs b/apps/experiment-runner/tests/acceptance_transcript.rs index c8b4fb2..24610c4 100644 --- a/apps/experiment-runner/tests/acceptance_transcript.rs +++ b/apps/experiment-runner/tests/acceptance_transcript.rs @@ -12,8 +12,11 @@ //! produced them. use std::num::{NonZeroU32, NonZeroU64}; +use std::path::Path; -use fpsmaxxing_contracts::{ChangeRequest, Decision, DecisionBounds, ExperimentSpec, VerdictReason}; +use fpsmaxxing_contracts::{ + ChangeRequest, Decision, DecisionBounds, ExperimentSpec, VerdictReason, +}; use fpsmaxxing_control_plane::ControlPlane; use fpsmaxxing_experiment_runner::{StoredTrial, TrialRecord, replay_trial, run_trial}; use fpsmaxxing_mock_provider::MockProvider; @@ -107,56 +110,90 @@ fn report(label: &str, trial: &StoredTrial) { } } +/// Measures both experiments over one journal, returning the trials they stored. +/// +/// The control plane is dropped when this returns, so the journal file is closed +/// before the transcript reads it back. +fn measure_both_experiments(journal_path: &Path) -> (StoredTrial, StoredTrial) { + let mut plane = ControlPlane::open(Box::new(MockProvider::new(BASELINE_VALUE)), journal_path) + .expect("open"); + println!( + "provider parked at mock.value = {}\n", + current_value(&plane) + ); + + // A measured experiment the evaluator promotes: the candidate gains 30 fps + // and stays inside every safety ceiling, so the broker runs the full + // snapshot/preview/apply/verify/rollback lifecycle. + let promoted = run_trial(&mut plane, &spec_for(40)).expect("run promoted trial"); + report("promoted", &promoted); + assert_eq!(promoted.record.verdict.decision, Decision::Promote); + assert_eq!(promoted.record.verdict.reason, VerdictReason::Promoted); + let lifecycle = promoted + .record + .lifecycle + .clone() + .expect("a promotion records its lifecycle"); + assert!(lifecycle.verified && lifecycle.rolled_back); + println!( + " provider after : mock.value = {} (the lease restored the pre-state)\n", + current_value(&plane) + ); + assert_eq!(current_value(&plane), BASELINE_VALUE); + + // A measured experiment the evaluator rejects: the candidate gains even more + // fps but drives modeled temperature to 85 C, past the ceiling, so it is + // never applied and the baseline is left exactly as it was. + let rejected = run_trial(&mut plane, &spec_for(70)).expect("run rejected trial"); + report("rejected", &rejected); + assert_eq!(rejected.record.verdict.decision, Decision::Reject); + assert_eq!( + rejected.record.verdict.reason, + VerdictReason::TemperatureExceeded + ); + assert!(rejected.record.lifecycle.is_none()); + assert!(rejected.record.lifecycle_error.is_none()); + println!( + " provider after : mock.value = {} (untouched - nothing was applied)\n", + current_value(&plane) + ); + assert_eq!(current_value(&plane), BASELINE_VALUE); + + (promoted, rejected) +} + +/// Shows that a clean replay is a real check rather than a rubber stamp. +/// +/// Appends a copy of the promotion whose temperature ceiling was widened after +/// the fact. It re-evaluates to the very verdict it carries, so only the policy +/// gate replay re-applies can catch it. +fn replay_catches_widened_bounds(archive: &ControlPlane, promoted_id: i64) { + let mut widened = archive.read_trial(promoted_id).expect("read trial"); + widened["spec"]["bounds"]["max_temperature_c"] = json!(200.0); + let widened_id = archive + .record_trial(&widened) + .expect("append the widened row"); + let outcome = replay_trial(archive, widened_id).expect("replay the widened row"); + println!( + " [tampered] trial {}: bounds widened to 200 C; recomputed verdict still identical = {}, but policy legal = {} ({})", + outcome.trial_id, + outcome.is_consistent(), + outcome.policy_legal, + outcome + .policy_reason + .as_deref() + .unwrap_or("no reason given") + ); + assert!(outcome.is_consistent()); + assert!(!outcome.policy_legal); +} + #[test] fn one_measured_experiment_is_promoted_and_another_rejected_then_both_replay() { let journal = NamedTempFile::new().expect("temp journal"); println!("journal file: an on-disk SQLite database\n"); - let promoted; - let rejected; - { - let mut plane = - ControlPlane::open(Box::new(MockProvider::new(BASELINE_VALUE)), journal.path()) - .expect("open"); - println!("provider parked at mock.value = {}\n", current_value(&plane)); - - // A measured experiment the evaluator promotes: the candidate gains - // 30 fps and stays inside every safety ceiling, so the broker runs the - // full snapshot/preview/apply/verify/rollback lifecycle. - promoted = run_trial(&mut plane, &spec_for(40)).expect("run promoted trial"); - report("promoted", &promoted); - assert_eq!(promoted.record.verdict.decision, Decision::Promote); - assert_eq!(promoted.record.verdict.reason, VerdictReason::Promoted); - let lifecycle = promoted - .record - .lifecycle - .clone() - .expect("a promotion records its lifecycle"); - assert!(lifecycle.verified && lifecycle.rolled_back); - println!( - " provider after : mock.value = {} (the lease restored the pre-state)\n", - current_value(&plane) - ); - assert_eq!(current_value(&plane), BASELINE_VALUE); - - // A measured experiment the evaluator rejects: the candidate gains even - // more fps but drives modeled temperature to 85 C, past the ceiling, so - // it is never applied and the baseline is left exactly as it was. - rejected = run_trial(&mut plane, &spec_for(70)).expect("run rejected trial"); - report("rejected", &rejected); - assert_eq!(rejected.record.verdict.decision, Decision::Reject); - assert_eq!( - rejected.record.verdict.reason, - VerdictReason::TemperatureExceeded - ); - assert!(rejected.record.lifecycle.is_none()); - assert!(rejected.record.lifecycle_error.is_none()); - println!( - " provider after : mock.value = {} (untouched - nothing was applied)\n", - current_value(&plane) - ); - assert_eq!(current_value(&plane), BASELINE_VALUE); - } + let (promoted, rejected) = measure_both_experiments(journal.path()); // The rows as the journal actually holds them, read with a plain SQLite // connection rather than through the control plane, so the durable state is @@ -189,7 +226,10 @@ fn one_measured_experiment_is_promoted_and_another_rejected_then_both_replay() { |row| row.get(0), ) .expect("read the promoted payload"); - println!("\nstored payload of trial {}:\n {stored_payload}", promoted.id); + println!( + "\nstored payload of trial {}:\n {stored_payload}", + promoted.id + ); // Reopen the file as a brand new handle, with a provider parked at an // unrelated value, and re-evaluate both trials from the journal alone. @@ -227,25 +267,7 @@ fn one_measured_experiment_is_promoted_and_another_rejected_then_both_replay() { assert_eq!(outcome.recomputed, trial.record.verdict); } - // A clean replay is a real check rather than a rubber stamp: append a copy - // of the promotion whose temperature ceiling was widened after the fact. - // It re-evaluates to the very verdict it carries, so only the policy gate - // replay re-applies can catch it. - let mut widened = archive.read_trial(promoted.id).expect("read trial"); - widened["spec"]["bounds"]["max_temperature_c"] = json!(200.0); - let widened_id = archive - .record_trial(&widened) - .expect("append the widened row"); - let outcome = replay_trial(&archive, widened_id).expect("replay the widened row"); - println!( - " [tampered] trial {}: bounds widened to 200 C; recomputed verdict still identical = {}, but policy legal = {} ({})", - outcome.trial_id, - outcome.is_consistent(), - outcome.policy_legal, - outcome.policy_reason.as_deref().unwrap_or("no reason given") - ); - assert!(outcome.is_consistent()); - assert!(!outcome.policy_legal); + replay_catches_widened_bounds(&archive, promoted.id); println!( "\nacceptance criterion: {} measured experiments reached a decision through the immutable evaluator - {:?} and {:?} - and both re-evaluate identically from the journal alone", diff --git a/apps/experiment-runner/tests/integration.rs b/apps/experiment-runner/tests/integration.rs index 7d3ca3a..c2e7390 100644 --- a/apps/experiment-runner/tests/integration.rs +++ b/apps/experiment-runner/tests/integration.rs @@ -22,7 +22,7 @@ use fpsmaxxing_experiment_runner::{RunnerError, TrialRecord, evaluate, replay_tr use fpsmaxxing_mock_provider::MockProvider; use fpsmaxxing_provider_sdk::{Provider, ProviderError}; use rusqlite::Connection; -use serde_json::json; +use serde_json::{Value, json}; use tempfile::NamedTempFile; /// Builds a spec that drives the mock knob to `candidate` under the given @@ -663,19 +663,15 @@ fn a_replay_reports_bounds_outside_the_policy_envelope() { assert!(outcome.policy_reason.is_none()); } -#[test] -fn a_replay_reports_a_record_the_policy_gate_would_refuse() { - let journal = NamedTempFile::new().expect("temp journal"); - let mut plane = - ControlPlane::open(Box::new(MockProvider::new(10)), journal.path()).expect("open"); - let trial = run_trial(&mut plane, &spec_for(40, 5.0, 80.0)).expect("run trial"); - let recorded = plane.read_trial(trial.id).expect("trial should read"); - - // Each of these rows re-evaluates to exactly the verdict it carries, so - // comparing verdicts cannot catch any of them. Replay applies the policy - // gate to the journaled spec instead - deliberately without the run-time - // manifest check - and cross-checks the record against the spec it carries. - let refused = [ +/// Rewrites of a journaled promotion that the policy gate must refuse. +/// +/// Each row re-evaluates to exactly the verdict it carries, so comparing +/// verdicts cannot catch any of them. Replay applies the policy gate to the +/// journaled spec instead - deliberately without the run-time manifest check - +/// and cross-checks the record against the spec it carries. Every entry is a +/// rewrite label, the gate name its reason must mention, and the payload. +fn rewrites_the_policy_gate_refuses(recorded: &Value) -> Vec<(&'static str, &'static str, Value)> { + vec![ ( "a capability the model never described", "unknown capability", @@ -746,9 +742,18 @@ fn a_replay_reports_a_record_the_policy_gate_would_refuse() { payload }, ), - ]; + ] +} + +#[test] +fn a_replay_reports_a_record_the_policy_gate_would_refuse() { + let journal = NamedTempFile::new().expect("temp journal"); + let mut plane = + ControlPlane::open(Box::new(MockProvider::new(10)), journal.path()).expect("open"); + let trial = run_trial(&mut plane, &spec_for(40, 5.0, 80.0)).expect("run trial"); + let recorded = plane.read_trial(trial.id).expect("trial should read"); - for (rewrite, expected_reason, payload) in refused { + for (rewrite, expected_reason, payload) in rewrites_the_policy_gate_refuses(&recorded) { let tampered = plane .record_trial(&payload) .expect("a rewritten record should append");