diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 82ab4b8a32..1ced816086 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -335,7 +335,8 @@ jobs: # read the source to check: the retry ladder drew its jitter from the # OS entropy pool and the hook bus stamped events from SystemTime while # both files' headers said otherwise. A floor over the I/O surfaces and - # the manifest, plus a down-only count of Instant::now() reads. + # the manifest, plus a down-only count of Instant::now() reads that + # reached zero on 2026-09-10 and may not rise. - name: no I/O in stella-core, and no new clock read if: ${{ !cancelled() && steps.checkout.outcome == 'success' }} run: python3 ./scripts/check-core-no-io.py diff --git a/AGENTS.md b/AGENTS.md index ca717bc7a5..18e2b566fa 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -130,8 +130,8 @@ make gate # = no-scratch + no-secrets + design-refs # + core-reachability (a stella-core module is # reachable from the engine's step path; down-only) # + core-no-io (no shipping stella-core source or - # dependency names an I/O surface; Instant::now() - # reads are a down-only count) + # dependency names an I/O surface, a timer or a + # clock; Instant::now() is a down-only count, at 0) # + typed-errors # + tool-error-class (#3167 unclassified-ToolOutput::error ratchet) # + dead-code-allows @@ -947,16 +947,19 @@ Append; do not renumber. `scripts/check-invariants.sh` enforces both halves. directories and `tests.rs` files stripped — and its `[dependencies]`. Three questions. A **floor**: no line names the filesystem, a process, the network, the environment, a standard stream, the wall clock - (`SystemTime::now`), a sleep, an entropy source, or a print macro, and - no dependency is an I/O or entropy crate (`tokio` may take only - `sync`, `time`, `macros` and `rt`). There is no baseline for the floor: - the tree has none and gains none. A **ratchet**: `Instant::now()` reads - are counted per file in `scripts/core-no-io-baseline.txt`, down-only, - because the monotonic clock is ambient state a replay cannot reproduce - even though it is not I/O; `make core-no-io-update` refuses to add a - file or raise a count, so a red run is cleared by reading the clock once - at the edge and passing `now` in, or by taking `ports::Clock`. The - guard was written on 2026-09-09 over two breaches this rule had + (`SystemTime::now`), a thread sleep, tokio's timer, a hidden clock read + (`.elapsed()`), an entropy source, or a print macro, and no dependency + is an I/O or entropy crate (`tokio` may take only `sync`, `macros` and + `rt` — not `time`). There is no baseline for the floor: the tree has + none and gains none. A **ratchet**: `Instant::now()` reads are counted + per file in `scripts/core-no-io-baseline.txt`, down-only, because the + monotonic clock is ambient state a replay cannot reproduce even though + it is not I/O. The count reached zero on 2026-09-10 and + `make core-no-io-update` refuses to add a file or raise it, so any read + fails. The engine's time comes through one port, `retry::Sleeper`: `now` + for every deadline and every elapsed time, `sleep` for every wait, and + `retry::bounded` — the port's sleep racing a call — for every timeout. + The guard was written on 2026-09-09 over two breaches this rule had carried unread: the retry ladder drew its jitter from `rand::rng()`, and the hook bus stamped every event from `SystemTime::now()`, while each file's header said the crate reads nothing directly. Both take a port diff --git a/crates/stella-cli/src/agent/resume.rs b/crates/stella-cli/src/agent/resume.rs index b735f4b313..05911bd68c 100644 --- a/crates/stella-cli/src/agent/resume.rs +++ b/crates/stella-cli/src/agent/resume.rs @@ -194,7 +194,11 @@ pub(crate) async fn run_resume(cfg: &Config, id: Option<&str>) -> Result<(), Cli ); let hook_runner = HostHookRunner; let engine_config = engine_config_for(cfg); - let state = stella_core::step::TurnState::from_checkpoint(checkpoint, &engine_config); + let state = stella_core::step::TurnState::from_checkpoint( + checkpoint, + &engine_config, + std::time::Instant::now(), + ); // Assembled, not built up by optional builders. This turn is the // `Resume` lane and says so. `lane_capabilities::resume` answers // every seam, so nothing is left to a chain. @@ -496,7 +500,8 @@ mod tests { checkpoint_sink: Some(sink.clone()), ..EngineConfig::default() }; - let state = TurnState::from_checkpoint(killed_mid_turn(), &config); + let state = + TurnState::from_checkpoint(killed_mid_turn(), &config, std::time::Instant::now()); let seams = TurnCapabilities::none(); let engine = Engine::assemble( &provider, @@ -578,7 +583,7 @@ mod tests { }; let mut at_cap = killed_mid_turn(); at_cap.step = HOST_CAP; - let state = TurnState::from_checkpoint(at_cap, &config); + let state = TurnState::from_checkpoint(at_cap, &config, std::time::Instant::now()); let seams = TurnCapabilities::none(); let engine = Engine::assemble( &provider, @@ -633,7 +638,7 @@ mod tests { }; let mut at_cap = killed_mid_turn(); at_cap.step = HOST_CAP; - let state = TurnState::from_checkpoint(at_cap, &config); + let state = TurnState::from_checkpoint(at_cap, &config, std::time::Instant::now()); let seams = TurnCapabilities::none(); let engine = Engine::assemble( &provider, diff --git a/crates/stella-cli/src/runtime.rs b/crates/stella-cli/src/runtime.rs index 16b4d74eab..ac0f85bb81 100644 --- a/crates/stella-cli/src/runtime.rs +++ b/crates/stella-cli/src/runtime.rs @@ -69,6 +69,10 @@ impl Sleeper for TokioSleeper { tokio::time::sleep(std::time::Duration::from_millis(duration_ms)).await; } + fn now(&self) -> std::time::Instant { + std::time::Instant::now() + } + fn jitter(&self, upper: u64) -> u64 { rand::rng().random_range(0..=upper) } diff --git a/crates/stella-core/Cargo.toml b/crates/stella-core/Cargo.toml index 961339e65e..a11be42137 100644 --- a/crates/stella-core/Cargo.toml +++ b/crates/stella-core/Cargo.toml @@ -14,7 +14,7 @@ serde = { workspace = true } serde_json = { workspace = true } serde_json_canonicalizer = { workspace = true } thiserror = { workspace = true } -tokio = { workspace = true, features = ["sync", "time"] } +tokio = { workspace = true, features = ["sync"] } async-trait = { workspace = true } futures-util = { workspace = true } sha2 = { workspace = true } diff --git a/crates/stella-core/README.md b/crates/stella-core/README.md index fb9657a29b..be4c5ee855 100644 --- a/crates/stella-core/README.md +++ b/crates/stella-core/README.md @@ -14,9 +14,11 @@ outside world is a trait the caller implements: `ToolExecutor`, `Clock`, ([`src/retry.rs`](src/retry.rs)), `HookRunner` ([`src/hooks.rs`](src/hooks.rs)) — plus `Provider` from `stella-protocol`. `make core-no-io` (`scripts/check-core-no-io.py`) reads the shipping source and the manifest -and fails on any of those surfaces named directly; the `Instant::now()` reads -the deadline arithmetic still makes are a down-only count in -`scripts/core-no-io-baseline.txt`, cleared by passing `now` in from the edge. +and fails on any of those surfaces named directly — a hidden clock read +(`.elapsed()`) and a tokio timer included. Every deadline is an `Instant` +from `Sleeper::now`, every timeout is `retry::bounded` racing the port's +sleep, and `scripts/core-no-io-baseline.txt` holds the `Instant::now()` +count at zero. Even the working directory is passed in (`EngineConfig::cwd`) rather than read from `std::env`. That is what makes compaction, eviction, loop detection and budget arithmetic plain synchronous functions over owned data, testable against diff --git a/crates/stella-core/src/accounted_call.rs b/crates/stella-core/src/accounted_call.rs index 089bd32aa1..0bd3355ead 100644 --- a/crates/stella-core/src/accounted_call.rs +++ b/crates/stella-core/src/accounted_call.rs @@ -1,13 +1,12 @@ //! I/O-free one-shot provider accounting shared by non-engine callers. use std::sync::atomic::{AtomicBool, Ordering}; -use std::time::{Duration, Instant}; +use std::time::Duration; use stella_protocol::{ AgentEvent, CompletionRequest, CompletionResult, ModelCallRole, Provider, ProviderError, ToolCall, ToolCallObserver, UsageIncompleteReason, }; -use tokio::time::timeout; use crate::budget::{BudgetGuard, BudgetOutcome, DeadlineOutcome}; use crate::event_sender::EventSender; @@ -157,7 +156,7 @@ pub async fn run_accounted_call( events: &EventSender, sleeper: &dyn Sleeper, ) -> Result { - let started = Instant::now(); + let started = sleeper.now(); // The task's wall clock, checked BEFORE the dispatch (#2238). This seam is // between model calls by construction — an `AccountedCall` carries no // tools, so there is never anything in flight to interrupt — which makes @@ -245,12 +244,12 @@ pub async fn run_accounted_call( let mut future = std::pin::pin!(future); let mut seen = progress.count(); loop { - match timeout(limit, &mut future).await { - Ok(Ok(outcome)) => break outcome, - Ok(Err(error)) => { + match crate::retry::bounded(sleeper, limit, &mut future).await { + Some(Ok(outcome)) => break outcome, + Some(Err(error)) => { return Err(AccountedCallError::Provider(error)); } - Err(_) => { + None => { // The window elapsed with the call still unresolved. // Whether that is the deadline this bound exists for // depends on what arrived during it: any fragment at @@ -271,7 +270,13 @@ pub async fn run_accounted_call( // so nothing was salvaged: the stream is // still open and its usage frame may yet // have been in flight. - emit_incomplete(&call, events, started.elapsed(), None, None); + emit_incomplete( + &call, + events, + sleeper.now().duration_since(started), + None, + None, + ); } return Err(AccountedCallError::Timeout); } @@ -353,7 +358,7 @@ pub async fn run_accounted_call( reasoning_tokens: result.usage.reasoning_tokens, estimated_input_tokens: call.estimated_input_tokens, cost_usd: result.cost_usd, - duration_ms: started.elapsed().as_millis() as u64, + duration_ms: sleeper.now().duration_since(started).as_millis() as u64, retries: outcome.retries.len() as u32, tool_calls: result.tool_calls.len(), complete: result.usage.is_complete(), @@ -372,7 +377,7 @@ pub async fn run_accounted_call( task_id: None, }); let budget_outcome = budget.record_spend(result.cost_usd); - let _ = events.send(budget.tick_event(Instant::now())); + let _ = events.send(budget.tick_event(sleeper.now())); if let BudgetOutcome::Warn { spent_usd, limit_usd, @@ -455,6 +460,7 @@ mod tests { use stella_protocol::{BudgetMode, CompletionMessage, CompletionRequestRef, CompletionUsage}; use super::*; + use std::time::Instant; struct NoopSleeper; @@ -463,6 +469,10 @@ mod tests { async fn sleep(&self, _duration_ms: u64) {} // The floor: a test that asserts on retry timing wants no spread in it. + fn now(&self) -> std::time::Instant { + std::time::Instant::now() + } + fn jitter(&self, _upper: u64) -> u64 { 0 } @@ -984,6 +994,10 @@ mod tests { // The floor: the timeout under test is placed against the exact // backoff, so the draw must not move it. + fn now(&self) -> std::time::Instant { + std::time::Instant::now() + } + fn jitter(&self, _upper: u64) -> u64 { 0 } @@ -1156,7 +1170,7 @@ mod tests { }, &mut budget, &EventSender::new(tx), - &NoopSleeper, + &TokioSleeper, ) .await .expect("the trailing gap must not abandon a call that was actively answering"); diff --git a/crates/stella-core/src/bus.rs b/crates/stella-core/src/bus.rs index 696ce9f5c5..9ffbcd5e9d 100644 --- a/crates/stella-core/src/bus.rs +++ b/crates/stella-core/src/bus.rs @@ -61,15 +61,16 @@ //! event comes from the [`Clock`] the host hands [`HookBus::new`], never //! from `SystemTime` — a hook script reads that stamp on the far side of a //! process boundary, so the host's wall clock is the right one and the -//! host owns it. Observer dispatch still reads a monotonic `Instant` to -//! enforce the per-handler latency budget (#459); `make core-no-io` counts -//! that read and lets it go no higher. +//! host owns it. Observer dispatch times each handler off that same clock +//! to enforce the per-handler latency budget (#459), so a wall-clock step +//! can misjudge one dispatch; quarantine needs three in a row, which one +//! step cannot supply. use std::collections::VecDeque; use std::panic::{AssertUnwindSafe, catch_unwind}; use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}; use std::sync::{Arc, Mutex, Weak}; -use std::time::{Duration, Instant}; +use std::time::Duration; use serde::{Deserialize, Serialize}; use serde_json::Value; @@ -645,9 +646,9 @@ impl HookBus { if quarantined.load(Ordering::Relaxed) { continue; } - let started = Instant::now(); + let started = self.inner.clock.now_ms(); let outcome = catch_unwind(AssertUnwindSafe(|| handler(event))); - let elapsed = started.elapsed(); + let elapsed = Duration::from_millis(self.inner.clock.now_ms().saturating_sub(started)); match outcome { Ok(Ok(())) => {} Ok(Err(message)) => self.report_failure(&pattern, event, message), @@ -1711,18 +1712,31 @@ mod tests { /// every later event, so it can't keep stalling the emitting (tool) thread. /// Positive direction only (a tiny budget + a handler that sleeps ~10x past /// it), to stay non-flaky under CI load. + /// A [`Clock`] the test advances by hand. + struct SteppingClock(Arc); + + impl Clock for SteppingClock { + fn now_ms(&self) -> u64 { + self.0.load(Ordering::Relaxed) + } + } + #[test] fn a_persistently_slow_observer_is_quarantined_then_skipped() { + // The bus reads its clock before and after each dispatch, so a + // handler that moves the clock is a slow handler — with no real + // sleep to flake under CI load. + let clock = Arc::new(AtomicU64::new(0)); let bus = HookBus::with_slow_observer_budget( "s", - crate::ports::FixedClock(0), + SteppingClock(clock.clone()), Duration::from_millis(10), ); let slow_calls = Arc::new(AtomicU32::new(0)); let sc = slow_calls.clone(); bus.on("file.created", move |_event| { sc.fetch_add(1, Ordering::Relaxed); - std::thread::sleep(Duration::from_millis(100)); + clock.fetch_add(100, Ordering::Relaxed); Ok(()) }) .detach(); diff --git a/crates/stella-core/src/driver.rs b/crates/stella-core/src/driver.rs index 01b13ddefc..8641cd5412 100644 --- a/crates/stella-core/src/driver.rs +++ b/crates/stella-core/src/driver.rs @@ -535,7 +535,7 @@ impl<'a> Engine<'a> { // caller's borrows when it drops — including on the hard-cancel path, // where the future is dropped mid-step and there is no exit to copy // back from (see `BorrowedTurn`). - let mut turn = BorrowedTurn::adopt(messages, budget, &self.config); + let mut turn = BorrowedTurn::adopt(messages, budget, &self.config, self.sleeper.now()); self.drive(&mut turn.state, events).await } @@ -677,7 +677,7 @@ impl<'a> Engine<'a> { } // The boundary's host consults — steering drain, soft stop, and the // #3243 Phase 3 re-query — in the order `step_boundary` documents. - deadline_notice::push_if_due(state, std::time::Instant::now()); + deadline_notice::push_if_due(state, self.sleeper.now()); if let Some(outcome) = step_boundary::consult_hosts(self.steering, self.requery, state, events).await { @@ -755,7 +755,7 @@ impl<'a> Engine<'a> { // Wall clock around the whole call including its retries, because that // is what a continuation would actually cost again — not the duration // of the one attempt that happened to succeed. - let step_started = std::time::Instant::now(); + let step_started = self.sleeper.now(); let committed = match self .run_model_call( state.model_call_shape(self.configured_output_ceiling()), @@ -781,7 +781,9 @@ impl<'a> Engine<'a> { self.emit_lifecycle(bus::names::MODEL_REQUEST_COMPLETED, || { lifecycle::model_request_completed_payload(state.step, &committed.result) }); - state.pace.observe_model(step_started.elapsed()); + state + .pace + .observe_model(self.sleeper.now().duration_since(step_started)); state.calibration_model = Some(committed.result.model.clone()); // Anchor the context measure to what the provider just attested for // this exact prefix — before dispatch appends the reply to it. @@ -839,7 +841,9 @@ impl<'a> Engine<'a> { } // The whole step, tools and park included: the reserve (`step_pace`). - state.pace.observe_step(step_started.elapsed()); + state + .pace + .observe_step(self.sleeper.now().duration_since(step_started)); // Advanced only by a step that committed and continued, so the index // a checkpoint carries is always "the step that runs next". @@ -852,7 +856,7 @@ impl<'a> Engine<'a> { /// and a meter. #[must_use] pub fn new_turn(&self, messages: Vec, budget: BudgetGuard) -> TurnState { - TurnState::new(messages, budget, &self.config) + TurnState::new(messages, budget, &self.config, self.sleeper.now()) } /// A [`TurnState`] resumed from a durable snapshot. See @@ -876,7 +880,7 @@ impl<'a> Engine<'a> { /// `stella-engine`'s test suite. #[must_use] pub fn resume_turn(&self, checkpoint: crate::step::Checkpoint) -> TurnState { - TurnState::from_checkpoint(checkpoint, &self.config) + TurnState::from_checkpoint(checkpoint, &self.config, self.sleeper.now()) } /// The calibrated compaction budget and the factor that produced it — @@ -1177,7 +1181,7 @@ impl<'a> Engine<'a> { // path, where AGENTS.md #5 (no panics on runtime data) // outranks asserting a structural claim (#618 item 17). biased; - result = deadline_bounded_generation(self.config.model_timeout, task_deadline, &progress, &mut complete) => result, + result = deadline_bounded_generation(self.sleeper, self.config.model_timeout, task_deadline, &progress, &mut complete) => result, _ = &mut pump => Err(ProviderError::Terminal( "speculation pump ended before the model call that feeds it; \ the speculation gate holds the channel open for the whole call, \ @@ -1204,7 +1208,7 @@ impl<'a> Engine<'a> { .. } = outcome; // One boundary read: the call's duration and the tick's clock axis. - let now = std::time::Instant::now(); + let now = self.sleeper.now(); let call_duration_ms = now.duration_since(call_started).as_millis() as u64; let budget_outcome = record_settled_cost(budget, result.cost_usd, warnings, events, now); @@ -1281,16 +1285,21 @@ impl<'a> Engine<'a> { events: EventSender, ) -> SpeculationPool { let announced = futures_util::stream::poll_fn(move |cx| rx.poll_recv(cx)); + let sleeper = self.sleeper; let mut in_flight = announced .map(|call| async move { - let started = std::time::Instant::now(); + let started = sleeper.now(); // `read_only: true` is exact, not a guess: only tools whose // schemas declare `read_only` (AND `speculation_safe`) are // ever announced to this pool, and hooked tools are fenced // out entirely (`tool_has_matching_hook`), so no hook reads // this bit off a speculative dispatch anyway. let output = self.execute_with_repair(&call, true, None).await; - (call, output, started.elapsed().as_millis() as u64) + ( + call, + output, + sleeper.now().duration_since(started).as_millis() as u64, + ) }) .buffer_unordered(MAX_CONCURRENT_TOOL_CALLS); @@ -1498,9 +1507,15 @@ impl<'a> Engine<'a> { let Some(limit) = self.config.tool_timeout else { return self.dispatch_tool_call(call, read_only, events).await; }; - match tokio::time::timeout(limit, self.dispatch_tool_call(call, read_only, events)).await { - Ok(output) => output, - Err(_) => ToolOutput::classified_error( + match crate::retry::bounded( + self.sleeper, + limit, + self.dispatch_tool_call(call, read_only, events), + ) + .await + { + Some(output) => output, + None => ToolOutput::classified_error( stella_protocol::ErrorClass::Timeout, format!( "tool `{}` exceeded the engine's {}s dispatch ceiling and was abandoned \ diff --git a/crates/stella-core/src/driver/capabilities.rs b/crates/stella-core/src/driver/capabilities.rs index 679e3e0fd3..3d9b0ecc04 100644 --- a/crates/stella-core/src/driver/capabilities.rs +++ b/crates/stella-core/src/driver/capabilities.rs @@ -431,7 +431,7 @@ mod tests { let provider = crate::subagent::tests::ScriptedProvider::new(vec![]); let tools = crate::subagent::tests::MixedTools::default(); - let sleeper = crate::subagent::tests::NoSleep; + let sleeper = crate::subagent::tests::TokioSleeper; let owned = built_elsewhere(); let seams = owned.as_borrowed(); @@ -473,7 +473,7 @@ mod tests { fn assemble_carries_the_bare_capability_set_onto_the_engine() { let provider = crate::subagent::tests::ScriptedProvider::new(vec![]); let tools = crate::subagent::tests::MixedTools::default(); - let sleeper = crate::subagent::tests::NoSleep; + let sleeper = crate::subagent::tests::TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &sleeper, seams); diff --git a/crates/stella-core/src/driver/completion.rs b/crates/stella-core/src/driver/completion.rs index 2a8dd1c64c..3744e1f9c2 100644 --- a/crates/stella-core/src/driver/completion.rs +++ b/crates/stella-core/src/driver/completion.rs @@ -92,7 +92,7 @@ impl<'a> Engine<'a> { &result.text, result.usage.output_tokens, *length_continuations, - clock.continuation_budget(std::time::Instant::now()), + clock.continuation_budget(self.sleeper.now()), ) { ContinuationPlan::Continue(plan) => { *length_continuations += 1; diff --git a/crates/stella-core/src/driver/dispatch.rs b/crates/stella-core/src/driver/dispatch.rs index c9c694e639..919298ac64 100644 --- a/crates/stella-core/src/driver/dispatch.rs +++ b/crates/stella-core/src/driver/dispatch.rs @@ -144,7 +144,7 @@ impl<'a> Engine<'a> { // than inside the futures below so a refusal lands before any // `ToolStart` fires, which is the shape `close_open_tool_calls` // established for a synthetic closure. - let now = std::time::Instant::now(); + let now = self.sleeper.now(); let mut admitted: Vec<(usize, &ToolCall)> = Vec::with_capacity(group_end - group_start); for (offset, call) in calls[group_start..group_end].iter().enumerate() { let index = group_start + offset; @@ -198,11 +198,12 @@ impl<'a> Engine<'a> { match harvested { Some(s) => (index, call, s.output, s.duration_ms, true), None => { - let started = std::time::Instant::now(); + let started = self.sleeper.now(); let output = self .execute_with_repair(call, read_only, Some(events)) .await; - let duration_ms = started.elapsed().as_millis() as u64; + let duration_ms = + self.sleeper.now().duration_since(started).as_millis() as u64; (index, call, output, duration_ms, false) } } @@ -455,6 +456,10 @@ mod tests { async fn sleep(&self, _duration_ms: u64) {} // The floor: a test that asserts on retry timing wants no spread in it. + fn now(&self) -> std::time::Instant { + std::time::Instant::now() + } + fn jitter(&self, _upper: u64) -> u64 { 0 } @@ -520,7 +525,7 @@ mod tests { turn_halt: Some(Arc::new(AfterFlag { flag })), ..EngineConfig::default() }; - let mut state = TurnState::from_checkpoint(step_one(), &config); + let mut state = TurnState::from_checkpoint(step_one(), &config, std::time::Instant::now()); let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, config, &NoopSleeper, seams); let (tx, mut rx) = mpsc::unbounded_channel(); @@ -575,7 +580,7 @@ mod tests { }; let tools = FlagAndHang { flag, hang: false }; let config = EngineConfig::default(); - let mut state = TurnState::from_checkpoint(step_one(), &config); + let mut state = TurnState::from_checkpoint(step_one(), &config, std::time::Instant::now()); let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, config, &NoopSleeper, seams); let (tx, mut rx) = mpsc::unbounded_channel(); diff --git a/crates/stella-core/src/driver/drive.rs b/crates/stella-core/src/driver/drive.rs index a25de75593..e86a1a1759 100644 --- a/crates/stella-core/src/driver/drive.rs +++ b/crates/stella-core/src/driver/drive.rs @@ -285,6 +285,10 @@ mod tests { async fn sleep(&self, _duration_ms: u64) {} // The floor: a test that asserts on retry timing wants no spread in it. + fn now(&self) -> std::time::Instant { + std::time::Instant::now() + } + fn jitter(&self, _upper: u64) -> u64 { 0 } @@ -341,7 +345,8 @@ mod tests { turn_halt: Some(Arc::new(AlwaysHalt)), ..EngineConfig::default() }; - let mut state = TurnState::from_checkpoint(restored_at_step_one(), &config); + let mut state = + TurnState::from_checkpoint(restored_at_step_one(), &config, std::time::Instant::now()); let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, config, &NoopSleeper, seams); let (tx, _rx) = mpsc::unbounded_channel(); @@ -371,7 +376,8 @@ mod tests { }; let tools = OkTool; let config = EngineConfig::default(); - let mut state = TurnState::from_checkpoint(restored_at_step_one(), &config); + let mut state = + TurnState::from_checkpoint(restored_at_step_one(), &config, std::time::Instant::now()); let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, config, &NoopSleeper, seams); let (tx, _rx) = mpsc::unbounded_channel(); diff --git a/crates/stella-core/src/driver/rate_limit.rs b/crates/stella-core/src/driver/rate_limit.rs index 8aec02e1d7..0621762e36 100644 --- a/crates/stella-core/src/driver/rate_limit.rs +++ b/crates/stella-core/src/driver/rate_limit.rs @@ -74,7 +74,7 @@ struct RateLimitPark<'e, 'a> { impl ParkSupervisor for RateLimitPark<'_, '_> { fn wait_allowance_ms(&mut self, waited_ms: u64) -> u64 { let cap = MAX_PARKED_WAIT_MS.saturating_sub(waited_ms); - match self.budget.deadline_remaining(std::time::Instant::now()) { + match self.budget.deadline_remaining(self.engine.sleeper.now()) { Some(remaining) => { let remaining_ms = u64::try_from(remaining.as_millis()).unwrap_or(u64::MAX); cap.min(remaining_ms.saturating_sub(PARK_DEADLINE_RESERVE_MS)) @@ -166,7 +166,7 @@ impl<'a> Engine<'a> { ), ModelCallFailure, > { - let call_started = std::time::Instant::now(); + let call_started = self.sleeper.now(); // Armed for exactly the interval where a paid attempt may be in // flight: a caller-side hard cancel that drops this future mid-await // still leaves one content-free `Cancelled` envelope behind. @@ -186,6 +186,7 @@ impl<'a> Engine<'a> { .unwrap_or(stella_protocol::UNKNOWN_MODEL) .to_string(), started: call_started, + sleeper: self.sleeper, armed: true, attempt_in_flight, }; diff --git a/crates/stella-core/src/driver/restore.rs b/crates/stella-core/src/driver/restore.rs index 7abece0543..e04190db5f 100644 --- a/crates/stella-core/src/driver/restore.rs +++ b/crates/stella-core/src/driver/restore.rs @@ -585,6 +585,10 @@ mod tests { async fn sleep(&self, _duration_ms: u64) {} // The floor: a test that asserts on retry timing wants no spread in it. + fn now(&self) -> std::time::Instant { + std::time::Instant::now() + } + fn jitter(&self, _upper: u64) -> u64 { 0 } diff --git a/crates/stella-core/src/driver/settlement.rs b/crates/stella-core/src/driver/settlement.rs index cfa4512092..f4d248e927 100644 --- a/crates/stella-core/src/driver/settlement.rs +++ b/crates/stella-core/src/driver/settlement.rs @@ -141,7 +141,7 @@ impl super::Engine<'_> { // the settlement tick below, so the boundary's two answers — what is // left on the clock, and whether that is already too little — cannot // disagree about when "now" was. - let now = std::time::Instant::now(); + let now = self.sleeper.now(); let child_spend = self.tools.drain_sub_agent_spend_usd(); if child_spend > 0.0 { record_settled_cost( diff --git a/crates/stella-core/src/driver/tests.rs b/crates/stella-core/src/driver/tests.rs index a65fa240e0..086053f76e 100644 --- a/crates/stella-core/src/driver/tests.rs +++ b/crates/stella-core/src/driver/tests.rs @@ -13,12 +13,22 @@ use crate::TurnCapabilities; use crate::hooks::{HookAction, HookExecError, HookExecResult, HookMatcher}; use crate::retry::Sleeper; -/// A `Sleeper` that records but never actually waits. +/// A `Sleeper` on tokio's clock, which every test here runs paused: a sleep +/// costs nothing while the runtime is idle and still lets a pending call +/// finish first, and `now` reads the same virtual timeline. A sleeper that +/// returned at once would make every engine timeout fire the moment a +/// provider future waited on another task, which is not what a timeout is. #[derive(Default)] -struct NoopSleeper; +struct TokioSleeper; #[async_trait] -impl Sleeper for NoopSleeper { - async fn sleep(&self, _duration_ms: u64) {} +impl Sleeper for TokioSleeper { + async fn sleep(&self, duration_ms: u64) { + tokio::time::sleep(std::time::Duration::from_millis(duration_ms)).await; + } + + fn now(&self) -> std::time::Instant { + tokio::time::Instant::now().into_std() + } // The floor: a test that asserts on retry timing wants no spread in it. fn jitter(&self, _upper: u64) -> u64 { @@ -242,7 +252,7 @@ async fn run_speculation_turn( provider: &SpeculatingProvider, tools: &dyn ToolExecutor, ) -> (TurnOutcome, Vec) { - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(provider, tools, EngineConfig::default(), &sleeper, seams); let (tx, mut rx) = mpsc::unbounded_channel(); @@ -257,7 +267,7 @@ async fn run_speculation_turn( (outcome, drain_events(&mut rx)) } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn read_only_calls_execute_during_the_stream_and_are_harvested_not_rerun() { let executed = Arc::new(tokio::sync::Notify::new()); let calls = Arc::new(AtomicU32::new(0)); @@ -299,7 +309,7 @@ async fn read_only_calls_execute_during_the_stream_and_are_harvested_not_rerun() /// out of speculation, with no hook attached. Its announced call must not /// run during the stream — it executes exactly once, at dispatch, so a /// retried attempt could never have billed it twice. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_read_only_but_speculation_unsafe_call_runs_once_at_dispatch() { let executed = Arc::new(tokio::sync::Notify::new()); let calls = Arc::new(AtomicU32::new(0)); @@ -344,7 +354,7 @@ async fn a_read_only_but_speculation_unsafe_call_runs_once_at_dispatch() { ); } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_divergent_committed_call_is_re_executed_not_harvested() { let executed = Arc::new(tokio::sync::Notify::new()); let calls = Arc::new(AtomicU32::new(0)); @@ -378,7 +388,7 @@ async fn a_divergent_committed_call_is_re_executed_not_harvested() { ); } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_divergent_committed_call_emits_a_harvest_mismatch_discard() { // The announced read (a.rs) is speculated and runs real I/O; the // committed call (b.rs) diverges, so the pooled result is rejected at @@ -416,7 +426,7 @@ async fn a_divergent_committed_call_emits_a_harvest_mismatch_discard() { /// already executed would drop silently on the abort unwind; it must instead /// emit `SpeculationDiscarded(budget_abort)` so #370's accounting holds on the /// abort path too. Witness: this event is absent before the fix. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn budget_abort_after_speculation_discards_the_pool() { let executed = Arc::new(tokio::sync::Notify::new()); let calls = Arc::new(AtomicU32::new(0)); @@ -434,7 +444,7 @@ async fn budget_abort_after_speculation_discards_the_pool() { executed, }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &sleeper, seams); let (tx, mut rx) = mpsc::unbounded_channel(); @@ -528,7 +538,7 @@ impl Provider for FlakySpeculatingProvider { } } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_failed_attempts_speculative_pool_emits_discarded_events() { // No hooks: the read IS speculated, so the failed first attempt runs it // for real (the wait returns the moment it does) and then drops the @@ -547,7 +557,7 @@ async fn a_failed_attempts_speculative_pool_emits_discarded_events() { calls: calls.clone(), executed, }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &sleeper, seams); let mut messages = vec![CompletionMessage::user("read a.rs")]; @@ -601,13 +611,13 @@ impl Provider for StreamingTextProvider { } } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn text_deltas_precede_the_authoritative_text_and_concatenate_to_it() { let provider = StreamingTextProvider; let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &sleeper, seams); let (tx, mut rx) = mpsc::unbounded_channel(); @@ -648,7 +658,7 @@ async fn text_deltas_precede_the_authoritative_text_and_concatenate_to_it() { } } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn mutating_calls_are_never_speculated() { let executed = Arc::new(tokio::sync::Notify::new()); let calls = Arc::new(AtomicU32::new(0)); @@ -718,7 +728,7 @@ impl Provider for WedgedProvider { /// The call count is the required assertion. A deadline that tripped as /// `Transport` would be retried and this would read 4, multiplying the very /// window the deadline exists to close. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_wedged_generation_trips_the_deadline_once_instead_of_burning_the_retry_budget() { let calls = Arc::new(AtomicU32::new(0)); let provider = WedgedProvider { @@ -727,7 +737,7 @@ async fn a_wedged_generation_trips_the_deadline_once_instead_of_burning_the_retr let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let config = EngineConfig { model_timeout: Some(Duration::from_millis(50)), ..EngineConfig::default() @@ -758,7 +768,7 @@ async fn a_wedged_generation_trips_the_deadline_once_instead_of_burning_the_retr drain_events(&mut rx); } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn simple_turn_with_no_tool_calls_completes() { let provider = ScriptedProvider { id: "scripted".into(), @@ -768,7 +778,7 @@ async fn simple_turn_with_no_tool_calls_completes() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &sleeper, seams); let mut messages = vec![ @@ -822,7 +832,7 @@ impl crate::driver::TurnHalt for NeverHalt { /// `Aborted` is the arm that matters here. It reaches the CLI as a non-zero /// exit, which Harbor scores identically to the agent crashing — so a turn /// that stopped BECAUSE it succeeded must not take that exit. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_halt_ends_the_turn_at_the_next_step_boundary_as_completed() { // Three tool-calling steps queued. Without a halt this turn runs all // three; with one it must stop after the first. @@ -838,7 +848,7 @@ async fn a_halt_ends_the_turn_at_the_next_step_boundary_as_completed() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let config = EngineConfig { turn_halt: Some(Arc::new(AlwaysHalt)), ..EngineConfig::default() @@ -872,7 +882,7 @@ async fn a_halt_ends_the_turn_at_the_next_step_boundary_as_completed() { /// The control: the same script with a halt that never fires runs to its /// scripted end. Without this, the test above would also pass if the seam /// simply broke every turn after one step. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_halt_that_never_fires_leaves_the_turn_exactly_as_it_was() { let provider = ScriptedProvider { id: "scripted".into(), @@ -886,7 +896,7 @@ async fn a_halt_that_never_fires_leaves_the_turn_exactly_as_it_was() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let config = EngineConfig { turn_halt: Some(Arc::new(NeverHalt)), ..EngineConfig::default() @@ -975,7 +985,7 @@ impl crate::ports::TurnSteering for TestSteering { } } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn steered_messages_inject_before_the_next_model_call() { let provider = ScriptedProvider { id: "scripted".into(), @@ -985,7 +995,7 @@ async fn steered_messages_inject_before_the_next_model_call() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let steering = TestSteering { queue: std::sync::Mutex::new(vec!["also check the tests".into()]), stop_after_drains: None, @@ -1028,7 +1038,7 @@ async fn steered_messages_inject_before_the_next_model_call() { ); } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn soft_stop_ends_the_turn_keeping_completed_steps() { let provider = ScriptedProvider { id: "scripted".into(), @@ -1043,7 +1053,7 @@ async fn soft_stop_ends_the_turn_keeping_completed_steps() { let tools = CountingTools { calls: tool_calls.clone(), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; // Stop latches after the first boundary: step 0 runs fully (model // call + tool), step 1's boundary honors the stop. let steering = TestSteering { @@ -1077,7 +1087,7 @@ async fn soft_stop_ends_the_turn_keeping_completed_steps() { ); } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn overflow_of_protected_content_is_summarized_and_metered() { let provider = ScriptedProvider { id: "scripted".into(), @@ -1090,7 +1100,7 @@ async fn overflow_of_protected_content_is_summarized_and_metered() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, overflow_config(), &sleeper, seams); let mut messages = vec![ @@ -1144,7 +1154,7 @@ async fn overflow_of_protected_content_is_summarized_and_metered() { ); } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn summarization_disabled_leaves_history_untouched() { let provider = ScriptedProvider { id: "scripted".into(), @@ -1155,7 +1165,7 @@ async fn summarization_disabled_leaves_history_untouched() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let config = EngineConfig { summarize_overflow: false, ..overflow_config() @@ -1182,7 +1192,7 @@ async fn summarization_disabled_leaves_history_untouched() { ); } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn summarizer_failure_is_non_fatal_and_leaves_history() { let provider = ScriptedProvider { id: "scripted".into(), @@ -1195,7 +1205,7 @@ async fn summarizer_failure_is_non_fatal_and_leaves_history() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, overflow_config(), &sleeper, seams); let mut messages = vec![ @@ -1221,7 +1231,7 @@ async fn summarizer_failure_is_non_fatal_and_leaves_history() { ); } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn summarization_never_orphans_tool_results_at_the_span_edge() { let provider = ScriptedProvider { id: "scripted".into(), @@ -1234,7 +1244,7 @@ async fn summarization_never_orphans_tool_results_at_the_span_edge() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, overflow_config(), &sleeper, seams); // The naive span end (len - keep_recent) lands ON the tool-result @@ -1305,7 +1315,7 @@ fn empty_result(finish_reason: Option) -> CompletionResultAlias { } } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn empty_completion_aborts_with_a_visible_message_not_a_silent_success() { // A turn that yields no text AND no tool calls — e.g. the model spent // its whole output budget on reasoning and was cut off at @@ -1320,7 +1330,7 @@ async fn empty_completion_aborts_with_a_visible_message_not_a_silent_success() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &sleeper, seams); let mut messages = vec![ @@ -1349,7 +1359,7 @@ async fn empty_completion_aborts_with_a_visible_message_not_a_silent_success() { ); } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_step_out_of_time_completes_with_a_truthful_partial_instead_of_aborting() { // The same empty length-truncated shape as the test above, and the opposite // ending, because the reason for stopping is opposite: above, the model @@ -1371,7 +1381,7 @@ async fn a_step_out_of_time_completes_with_a_truthful_partial_instead_of_abortin let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble( &provider, @@ -1444,7 +1454,7 @@ fn length_text_result(text: &str) -> CompletionResultAlias { } } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_length_truncated_tool_less_step_continues_the_turn_instead_of_completing() { // Step 0 is cut off at the output limit mid-"reasoning" with no tool // call. That is not a finished turn: the engine must record the partial, @@ -1465,7 +1475,7 @@ async fn a_length_truncated_tool_less_step_continues_the_turn_instead_of_complet let tools = CountingTools { calls: tool_calls.clone(), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &sleeper, seams); let mut messages = vec![ @@ -1509,7 +1519,7 @@ async fn a_length_truncated_tool_less_step_continues_the_turn_instead_of_complet assert_tool_pairing(&messages); } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn length_continuations_are_bounded_per_turn() { // The model truncates tool-less on EVERY step (the scripted provider // loops its last entry). The engine spends its whole continuation @@ -1525,7 +1535,7 @@ async fn length_continuations_are_bounded_per_turn() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &sleeper, seams); let mut messages = vec![ @@ -1568,7 +1578,7 @@ async fn length_continuations_are_bounded_per_turn() { ); } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn tool_calls_execute_and_feed_back_into_history() { let provider = ScriptedProvider { id: "scripted".into(), @@ -1582,7 +1592,7 @@ async fn tool_calls_execute_and_feed_back_into_history() { let tools = CountingTools { calls: tool_calls.clone(), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &sleeper, seams); let mut messages = vec![ @@ -1615,7 +1625,7 @@ async fn tool_calls_execute_and_feed_back_into_history() { ); } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn retry_never_re_executes_a_tool_call() { // Property: a step's tool call is executed exactly once, even when // the model call surrounding it needed retries elsewhere in the @@ -1635,7 +1645,7 @@ async fn retry_never_re_executes_a_tool_call() { let tools = CountingTools { calls: tool_calls.clone(), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &sleeper, seams); let mut messages = vec![ @@ -1670,7 +1680,7 @@ async fn retry_never_re_executes_a_tool_call() { assert_eq!(retry_events, 2); } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn malformed_tool_call_input_is_repaired_not_executed_blindly() { let mut malformed_call = tool_call_result("call_1", "bash"); malformed_call.tool_calls[0].input = Value::Null; @@ -1683,7 +1693,7 @@ async fn malformed_tool_call_input_is_repaired_not_executed_blindly() { let tools = CountingTools { calls: tool_calls.clone(), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &sleeper, seams); let mut messages = vec![ @@ -1711,7 +1721,7 @@ async fn malformed_tool_call_input_is_repaired_not_executed_blindly() { } } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn stuck_loop_aborts_the_turn_cleanly_before_the_step_cap() { // Every call returns the identical tool call and the tool answers with // identical output — well past the default exact-repeat threshold (3) @@ -1727,7 +1737,7 @@ async fn stuck_loop_aborts_the_turn_cleanly_before_the_step_cap() { let tools = CountingTools { calls: tool_calls.clone(), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &sleeper, seams); let mut messages = vec![ @@ -1749,7 +1759,7 @@ async fn stuck_loop_aborts_the_turn_cleanly_before_the_step_cap() { assert!(events.iter().any(|e| matches!(e, AgentEvent::Error { .. }))); } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn stuck_loop_steers_once_then_aborts_on_re_detection() { // The exact steer-then-abort sequencing: three identical no-progress // calls earn a steering warning, the model ignores it with a fourth @@ -1766,7 +1776,7 @@ async fn stuck_loop_steers_once_then_aborts_on_re_detection() { let tools = CountingTools { calls: tool_calls.clone(), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &sleeper, seams); let mut messages = vec![ @@ -1870,7 +1880,7 @@ impl ToolExecutor for PollingTools { } } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn identical_polls_with_changing_output_complete_without_abort() { // Six byte-identical calls (same name, same input, no cursor field) — // but every poll returns new output. That is visible progress, not a @@ -1888,7 +1898,7 @@ async fn identical_polls_with_changing_output_complete_without_abort() { let tools = PollingTools { calls: tool_calls.clone(), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &sleeper, seams); let mut messages = vec![ @@ -1913,7 +1923,7 @@ async fn identical_polls_with_changing_output_complete_without_abort() { ); } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn period_three_cycle_with_no_progress_steers_then_aborts() { // The common real stuck signature: read → failing edit → failing test, // with byte-identical outputs every cycle — invisible to exact-repeat @@ -1950,7 +1960,7 @@ async fn period_three_cycle_with_no_progress_steers_then_aborts() { let tools = CountingTools { calls: tool_calls.clone(), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let config = EngineConfig { loop_detection: LoopDetectionConfig { exact_repeat_threshold: 3, @@ -1994,7 +2004,7 @@ async fn period_three_cycle_with_no_progress_steers_then_aborts() { ); } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn enforced_budget_aborts_the_turn_cleanly_between_steps() { let provider = ScriptedProvider { id: "scripted".into(), @@ -2004,7 +2014,7 @@ async fn enforced_budget_aborts_the_turn_cleanly_between_steps() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &sleeper, seams); let mut messages = vec![ @@ -2131,10 +2141,10 @@ async fn run_synthetic_survival_turn(dialect: &str, id_style: fn(u32) -> String) } } let tools = GrowingTools; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let config = EngineConfig { // Keep the retry backoff floor at 0 so 200 steps with injected - // 429s/drops still runs near-instantly under NoopSleeper. + // 429s/drops still runs near-instantly under TokioSleeper. retry_policy: RetryPolicy::new(3, 0, 0), // A tight-ish compaction budget so the growing tool output // actually forces multiple compaction passes over 200 steps. @@ -2153,7 +2163,7 @@ async fn run_synthetic_survival_turn(dialect: &str, id_style: fn(u32) -> String) engine.run_turn(&mut messages, &mut budget, &tx).await } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn synthetic_200_step_turn_survives_glm_shape() { let outcome = run_synthetic_survival_turn("glm", |i| format!("call_{i}")).await; assert!( @@ -2162,7 +2172,7 @@ async fn synthetic_200_step_turn_survives_glm_shape() { ); } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn synthetic_200_step_turn_survives_anthropic_shape() { // Anthropic's tool_use ids are its own `toolu_...` convention — // varying the id shape alone is enough to prove the driver never @@ -2174,7 +2184,7 @@ async fn synthetic_200_step_turn_survives_anthropic_shape() { ); } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn synthetic_200_step_turn_survives_openai_shape() { let outcome = run_synthetic_survival_turn("openai", |i| format!("call_{i:016x}")).await; assert!( @@ -2237,7 +2247,7 @@ impl ToolExecutor for BarrierTools { } } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn read_only_calls_in_one_step_execute_concurrently() { let provider = ScriptedProvider { id: "scripted".into(), @@ -2253,7 +2263,7 @@ async fn read_only_calls_in_one_step_execute_concurrently() { let tools = BarrierTools { barrier: tokio::sync::Barrier::new(2), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &sleeper, seams); let mut messages = vec![ @@ -2330,7 +2340,7 @@ impl ToolExecutor for RecordingTools { } } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn mutating_calls_are_barriers_and_history_keeps_call_order() { let provider = ScriptedProvider { id: "scripted".into(), @@ -2351,7 +2361,7 @@ async fn mutating_calls_are_barriers_and_history_keeps_call_order() { read1_started: tokio::sync::Notify::new(), read2_done: tokio::sync::Notify::new(), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &sleeper, seams); let mut messages = vec![ @@ -2420,7 +2430,7 @@ async fn mutating_calls_are_barriers_and_history_keeps_call_order() { // ---- StepUsage telemetry ---------------------------------------------- -#[tokio::test] +#[tokio::test(start_paused = true)] async fn every_committed_step_emits_exactly_one_step_usage_record() { let with_usage = |text: &str, calls: &[(&str, &str)]| { let mut result = if calls.is_empty() { @@ -2454,7 +2464,7 @@ async fn every_committed_step_emits_exactly_one_step_usage_record() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &sleeper, seams); let mut messages = vec![ @@ -2545,7 +2555,7 @@ async fn a_wedged_tool_trips_the_dispatch_ceiling_instead_of_hanging() { ]), calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let config = EngineConfig { tool_timeout: Some(Duration::from_secs(900)), ..EngineConfig::default() @@ -2601,7 +2611,7 @@ async fn a_none_ceiling_leaves_tool_dispatch_unbounded() { ]), calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let config = EngineConfig { tool_timeout: None, ..EngineConfig::default() @@ -2669,7 +2679,7 @@ impl crate::step::CheckpointSink for RecordingSink { } } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_turn_checkpoints_at_every_step_boundary_and_clears_when_it_ends() { // The durability contract in one test. Script: a tool call (step 0 // continues) then text (step 1 completes), so the turn crosses exactly @@ -2694,7 +2704,7 @@ async fn a_turn_checkpoints_at_every_step_boundary_and_clears_when_it_ends() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let sink = Arc::new(RecordingSink::default()); let config = EngineConfig { checkpoint_sink: Some(sink.clone() as Arc), @@ -2743,7 +2753,7 @@ async fn a_turn_checkpoints_at_every_step_boundary_and_clears_when_it_ends() { drain_events(&mut rx); } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_turn_without_a_sink_is_unchanged() { // The default config attaches no sink, and that path must stay entirely // free of checkpoint work — this is what keeps durability opt-in for @@ -2756,7 +2766,7 @@ async fn a_turn_without_a_sink_is_unchanged() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; assert!( EngineConfig::default().checkpoint_sink.is_none(), "durability is opt-in: a default engine writes no checkpoints" diff --git a/crates/stella-core/src/driver/tests/audit_fixes.rs b/crates/stella-core/src/driver/tests/audit_fixes.rs index 61f528c939..4877b53fcc 100644 --- a/crates/stella-core/src/driver/tests/audit_fixes.rs +++ b/crates/stella-core/src/driver/tests/audit_fixes.rs @@ -9,7 +9,7 @@ use super::*; /// F1: `summarize_keep_recent: 0` is a legal config — the tail walk must /// not index one past the end (this test panicked with "index out of /// bounds" before the guard). -#[tokio::test] +#[tokio::test(start_paused = true)] async fn summarize_keep_recent_zero_does_not_panic() { let provider = ScriptedProvider { id: "scripted".into(), @@ -19,7 +19,7 @@ async fn summarize_keep_recent_zero_does_not_panic() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let config = EngineConfig { summarize_keep_recent: 0, ..overflow_config() @@ -51,7 +51,7 @@ async fn summarize_keep_recent_zero_does_not_panic() { /// F2: `BudgetOutcome::Warn`'s contract is that the driver surfaces it — /// an Observed-mode breach must emit a visible warning (exactly once per /// settled call, so the twice-per-step gate checks cannot spam it). -#[tokio::test] +#[tokio::test(start_paused = true)] async fn observed_budget_breach_emits_a_warning_event() { let provider = ScriptedProvider { id: "scripted".into(), @@ -61,7 +61,7 @@ async fn observed_budget_breach_emits_a_warning_event() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &sleeper, seams); let mut messages = vec![ @@ -105,7 +105,7 @@ async fn observed_budget_breach_emits_a_warning_event() { /// F5: the budget-abort path's synthetic tool results must reach the event /// stream, not just `messages` — StepUsage already announced the calls, so /// a transcript reconstructed from events must resolve them too. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn budget_abort_synthetic_results_are_visible_in_the_event_stream() { let provider = ScriptedProvider { id: "scripted".into(), @@ -116,7 +116,7 @@ async fn budget_abort_synthetic_results_are_visible_in_the_event_stream() { let tools = CountingTools { calls: tool_calls.clone(), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &sleeper, seams); let mut messages = vec![ @@ -150,7 +150,7 @@ async fn budget_abort_synthetic_results_are_visible_in_the_event_stream() { /// F6: a whitespace-only response is the empty-turn defect, not an answer — /// it must abort without first streaming a blank `Text` event. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn whitespace_only_completion_aborts_without_a_text_event() { let provider = ScriptedProvider { id: "scripted".into(), @@ -160,7 +160,7 @@ async fn whitespace_only_completion_aborts_without_a_text_event() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &sleeper, seams); let mut messages = vec![ @@ -326,7 +326,7 @@ impl Provider for HangingProvider { /// F9: a hard cancel that drops the turn while a paid attempt is mid-stream /// must leave exactly one content-free `Cancelled` usage envelope — the /// call may have real server-side cost and must not vanish from accounting. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn hard_cancel_mid_stream_emits_a_cancelled_usage_envelope() { let started = Arc::new(tokio::sync::Notify::new()); let provider = HangingProvider { @@ -335,7 +335,7 @@ async fn hard_cancel_mid_stream_emits_a_cancelled_usage_envelope() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &sleeper, seams); let mut messages = vec![CompletionMessage::user("secret prompt text")]; @@ -396,6 +396,10 @@ impl crate::retry::Sleeper for HangingSleeper { } // The floor: this double never wakes, so the draw decides nothing. + fn now(&self) -> std::time::Instant { + std::time::Instant::now() + } + fn jitter(&self, _upper: u64) -> u64 { 0 } @@ -407,7 +411,7 @@ impl crate::retry::Sleeper for HangingSleeper { /// `ProviderError` envelope, and a second envelope for the same single /// dispatch double-reports it (the guard's armed window used to span the /// sleeps; `attempt_in_flight` is what narrowed it). -#[tokio::test] +#[tokio::test(start_paused = true)] async fn hard_cancel_during_a_backoff_sleep_emits_no_phantom_cancelled_envelope() { let provider = ScriptedProvider { id: "scripted".into(), @@ -479,7 +483,7 @@ fn summary_markers(messages: &[CompletionMessage]) -> usize { /// #368.2: the summarizer is the last line of defense before a terminal /// context overflow, so a transient blip must be retried (standard policy), /// not fast-failed (deterministic policy, which discarded the recovery). -#[tokio::test] +#[tokio::test(start_paused = true)] async fn overflow_summarizer_retries_a_transient_error() { let provider = ScriptedProvider { id: "scripted".into(), @@ -493,7 +497,7 @@ async fn overflow_summarizer_retries_a_transient_error() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, overflow_config(), &sleeper, seams); let mut messages = overflow_messages(); @@ -537,7 +541,7 @@ async fn overflow_summarizer_retries_a_transient_error() { /// #368.3: a summary generated and paid for right as the budget trips must /// still be spliced in — applying it only shrinks the context the resumed /// session reloads. Discarding it lost paid work for no benefit. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn budget_aborted_summary_is_applied_not_discarded() { let provider = ScriptedProvider { id: "scripted".into(), @@ -547,7 +551,7 @@ async fn budget_aborted_summary_is_applied_not_discarded() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, overflow_config(), &sleeper, seams); let mut messages = overflow_messages(); @@ -601,7 +605,7 @@ async fn budget_aborted_summary_is_applied_not_discarded() { /// `summarized_blocks` was hard-coded empty, so the one compaction path that /// most changes context reported no block identities. Witness: a tool-result /// block in the folded span leaves context and is named in the receipt. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn overflow_summary_names_the_folded_tool_result_blocks() { let provider = ScriptedProvider { id: "scripted".into(), @@ -611,7 +615,7 @@ async fn overflow_summary_names_the_folded_tool_result_blocks() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, overflow_config(), &sleeper, seams); @@ -701,7 +705,7 @@ async fn overflow_summary_names_the_folded_tool_result_blocks() { /// #368.4: a summarizer that keeps failing must surface each failure and, /// after enough consecutive misses, latch — a persistently-timing-out cheap /// summarizer can't be allowed to re-fire (and re-pay) every remaining step. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn repeated_summarizer_failures_emit_events_and_latch() { let provider = ScriptedProvider { id: "scripted".into(), @@ -712,7 +716,7 @@ async fn repeated_summarizer_failures_emit_events_and_latch() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, overflow_config(), &sleeper, seams); let mut budget = BudgetGuard::new(BudgetMode::Off, None, None); @@ -794,7 +798,7 @@ async fn repeated_summarizer_failures_emit_events_and_latch() { /// A single observed-mode breach persists across every remaining settled /// call of the turn, but it must warn once per axis, not once per call — /// otherwise a session-limit breach on a many-step turn floods the stream. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn observed_budget_breach_warns_once_per_axis_per_turn() { let provider = ScriptedProvider { id: "scripted".into(), @@ -810,7 +814,7 @@ async fn observed_budget_breach_warns_once_per_axis_per_turn() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &sleeper, seams); let mut messages = vec![ @@ -857,7 +861,7 @@ async fn observed_budget_breach_warns_once_per_axis_per_turn() { /// An enforced session breach that trips as the just-landed call settles /// aborts through `handle_committed_result` — its reason must name the /// session axis so the user knows which cap they hit. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn enforced_session_breach_abort_reason_names_the_axis() { let provider = ScriptedProvider { id: "scripted".into(), @@ -867,7 +871,7 @@ async fn enforced_session_breach_abort_reason_names_the_axis() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &sleeper, seams); let mut messages = vec![ @@ -891,7 +895,7 @@ async fn enforced_session_breach_abort_reason_names_the_axis() { /// A session already over budget at the turn's opening safe-boundary aborts /// through `check_budget` (before any call is dispatched) — that reason must /// name the session axis too. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn enforced_session_breach_at_step_boundary_names_the_axis() { let provider = ScriptedProvider { id: "scripted".into(), @@ -902,7 +906,7 @@ async fn enforced_session_breach_at_step_boundary_names_the_axis() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &sleeper, seams); let mut messages = vec![ @@ -1242,7 +1246,7 @@ impl ToolExecutor for CountingReadTools { /// them). A second announcement under the same id therefore EVICTS the first /// pool entry — whose tool had already run real I/O — and before the fix that /// eviction was silent: the execution happened and nothing on the wire said so. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_recycled_speculation_call_id_reports_the_execution_it_displaces() { let read = |path: &str| ToolCall { call_id: "call_0".into(), @@ -1263,7 +1267,7 @@ async fn a_recycled_speculation_call_id_reports_the_execution_it_displaces() { let tools = CountingReadTools { executions: executions.clone(), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &sleeper, seams); let (tx, mut rx) = mpsc::unbounded_channel(); @@ -1374,14 +1378,14 @@ impl ToolExecutor for BulkyTools { /// and the overflow summarizer's span opens *after* the first user message. This /// pins all three at the driver level: several steps of real compaction and a /// summarization pass, and the bytes at index 0 never change. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn the_system_prefix_stays_byte_stable_across_a_compacting_turn() { const SYSTEM: &str = "You are Stella. Follow the workspace rules. Prefer small diffs."; let provider = PrefixRecordingProvider { prefixes: std::sync::Mutex::new(Vec::new()), step: AtomicU32::new(0), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let config = EngineConfig { // Small enough that every step after the first compacts, and small // enough that the pure passes cannot get under it alone — so the diff --git a/crates/stella-core/src/driver/tests/budget_boundaries.rs b/crates/stella-core/src/driver/tests/budget_boundaries.rs index 75faf2c7a8..d5c8d354da 100644 --- a/crates/stella-core/src/driver/tests/budget_boundaries.rs +++ b/crates/stella-core/src/driver/tests/budget_boundaries.rs @@ -58,7 +58,7 @@ fn provider_that_must_not_be_called() -> ScriptedProvider { } } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn an_over_cap_budget_abort_hands_back_a_well_paired_transcript() { // The dollar arm of `settlement::check_budget` fires at // the top of the step, before any model call, on a transcript that is @@ -71,7 +71,7 @@ async fn an_over_cap_budget_abort_hands_back_a_well_paired_transcript() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &sleeper, seams); let mut messages = transcript_with_an_unanswered_tool_call(); @@ -98,7 +98,7 @@ async fn an_over_cap_budget_abort_hands_back_a_well_paired_transcript() { ); } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_past_deadline_abort_hands_back_a_well_paired_transcript() { // The same witness for the deadline arm, which is checked first and has // its own reason string — and had the same missing repair. @@ -107,7 +107,7 @@ async fn a_past_deadline_abort_hands_back_a_well_paired_transcript() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &sleeper, seams); let mut messages = transcript_with_an_unanswered_tool_call(); @@ -197,7 +197,7 @@ impl ToolExecutor for ForeverRead { } } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn summary_induced_budget_breach_aborts_with_cost_before_next_provider_call() { let provider = ScriptedProvider { id: "scripted".into(), @@ -211,7 +211,7 @@ async fn summary_induced_budget_breach_aborts_with_cost_before_next_provider_cal let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, overflow_config(), &sleeper, seams); let mut messages = vec![ @@ -252,7 +252,7 @@ async fn summary_induced_budget_breach_aborts_with_cost_before_next_provider_cal ); } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn an_existing_budget_breach_stops_before_paid_compaction() { let provider = ScriptedProvider { id: "scripted".into(), @@ -263,7 +263,7 @@ async fn an_existing_budget_breach_stops_before_paid_compaction() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, overflow_config(), &sleeper, seams); let mut messages = vec![ @@ -290,7 +290,7 @@ async fn an_existing_budget_breach_stops_before_paid_compaction() { ); } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_past_task_deadline_stops_the_turn_before_the_next_call_with_partial_work() { // The witness for #1481: the dollar budget is per turn/session, but a // benchmark's limit is per TASK — several turns that each honestly fit @@ -310,7 +310,7 @@ async fn a_past_task_deadline_stops_the_turn_before_the_next_call_with_partial_w let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &sleeper, seams); let mut messages = vec![ @@ -356,14 +356,14 @@ async fn a_past_task_deadline_stops_the_turn_before_the_next_call_with_partial_w ); } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn cancellation_after_billed_completion_before_speculation_finishes_keeps_the_cost() { let provider_completed = Arc::new(tokio::sync::Notify::new()); let provider = BilledResultWithBlockedSpeculation { provider_completed: provider_completed.clone(), }; let tools = ForeverRead; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &sleeper, seams); let mut messages = vec![CompletionMessage::user("read")]; @@ -396,7 +396,7 @@ async fn cancellation_after_billed_completion_before_speculation_finishes_keeps_ ); } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_normal_completion_charges_the_budget_exactly_once() { let provider = ScriptedProvider { id: "scripted".into(), @@ -406,7 +406,7 @@ async fn a_normal_completion_charges_the_budget_exactly_once() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &sleeper, seams); let mut messages = vec![CompletionMessage::user("answer")]; @@ -461,7 +461,7 @@ impl ToolExecutor for SlowTool { } } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_slow_tool_stops_the_turn_before_the_deadline() { // The scripted model answers at once and its tool then runs for a // second, so the step costs a second and the call inside it costs @@ -479,7 +479,7 @@ async fn a_slow_tool_stops_the_turn_before_the_deadline() { }; let provider_calls = provider.calls.clone(); let tools = SlowTool { took: tool_time }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &sleeper, seams); let mut messages = vec![ diff --git a/crates/stella-core/src/driver/tests/calibration.rs b/crates/stella-core/src/driver/tests/calibration.rs index 868db1fcfa..034d4c71d4 100644 --- a/crates/stella-core/src/driver/tests/calibration.rs +++ b/crates/stella-core/src/driver/tests/calibration.rs @@ -51,7 +51,7 @@ fn compactable_history() -> Vec { /// the raw estimate runs low against this model's tokenizer — the /// compaction decision demonstrably consumes the calibrated estimate, /// not the raw one. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn calibrated_estimate_changes_the_compaction_decision() { let run = |calibrate: bool| async move { let provider = ScriptedProvider { @@ -62,7 +62,7 @@ async fn calibrated_estimate_changes_the_compaction_decision() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let mut messages = compactable_history(); // A budget the RAW estimate just fits under: uncalibrated, no // compaction can fire. @@ -108,7 +108,7 @@ async fn calibrated_estimate_changes_the_compaction_decision() { /// records its (estimated, actual) pair into the attached calibration — /// keyed by the model that served it — and emits the raw estimate on /// `StepUsage` for persistence. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn each_committed_step_feeds_the_calibration_and_reports_its_estimate() { let with_real_usage = |result: CompletionResultAlias| { let mut result = result; @@ -141,7 +141,7 @@ async fn each_committed_step_feeds_the_calibration_and_reports_its_estimate() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let calibration = CalibrationMap::new(); let seams = TurnCapabilities { calibration: Some(&calibration), @@ -196,7 +196,7 @@ async fn each_committed_step_feeds_the_calibration_and_reports_its_estimate() { /// session's first call — nearly its whole prompt a cache write — recorded /// a near-zero ratio that dragged the factor toward the floor and inflated /// the effective compaction budget past the provider's context window. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn cache_write_tokens_count_toward_the_calibration_actual() { let with_cache_write_usage = |result: CompletionResultAlias| { let mut result = result; @@ -228,7 +228,7 @@ async fn cache_write_tokens_count_toward_the_calibration_actual() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let calibration = CalibrationMap::new(); let seams = TurnCapabilities { calibration: Some(&calibration), @@ -263,7 +263,7 @@ async fn cache_write_tokens_count_toward_the_calibration_actual() { /// pressure, poison for calibration, where one screenshot-bearing step /// clamped the ratio to the sample floor and doubled the effective /// compaction budget for the rest of the session. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn attachment_weight_is_excluded_from_the_drift_sample_estimate() { let provider = ScriptedProvider { id: "scripted".into(), @@ -273,7 +273,7 @@ async fn attachment_weight_is_excluded_from_the_drift_sample_estimate() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let mut messages = vec![ CompletionMessage::system("sys"), CompletionMessage::user_with_attachments( @@ -325,7 +325,7 @@ async fn attachment_weight_is_excluded_from_the_drift_sample_estimate() { /// model-known read — one sample into a three-sample warm-up — and served it /// for the rest of the turn, so the 40+ samples the turn itself recorded were /// never read back into any decision. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_fresh_sessions_calibration_factor_leaves_identity_within_the_turn() { // Actuals that grow with the transcript, all far above the tiny history // estimates — the drift shape the bench trace measured. @@ -359,7 +359,7 @@ async fn a_fresh_sessions_calibration_factor_leaves_identity_within_the_turn() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; // Unseeded, exactly like a bench container's first run: warm-up happens // (or fails to matter) entirely inside this one turn. let calibration = CalibrationMap::new(); diff --git a/crates/stella-core/src/driver/tests/compute_passes.rs b/crates/stella-core/src/driver/tests/compute_passes.rs index 709c703d12..007400c0b6 100644 --- a/crates/stella-core/src/driver/tests/compute_passes.rs +++ b/crates/stella-core/src/driver/tests/compute_passes.rs @@ -24,7 +24,7 @@ use std::collections::HashMap; /// before every model call. The remaining two are the honest ones — compaction /// must measure before deciding, and the step must know its input size — and this /// asserts the count so a future refactor cannot quietly reintroduce a third. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_step_walks_the_transcript_to_estimate_it_at_most_twice() { const STEPS: usize = 3; let provider = ScriptedProvider { @@ -39,7 +39,7 @@ async fn a_step_walks_the_transcript_to_estimate_it_at_most_twice() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &sleeper, seams); let mut messages = vec![ @@ -81,7 +81,7 @@ async fn a_step_walks_the_transcript_to_estimate_it_at_most_twice() { /// re-derives the manifest's estimate from something else (post-compaction /// messages, a calibrated figure) would silently desynchronize the pair that /// `StepUsage`'s drift sampling compares. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn the_receipt_and_the_usage_record_report_one_estimate_per_step() { let provider = ScriptedProvider { id: "scripted".into(), @@ -94,7 +94,7 @@ async fn the_receipt_and_the_usage_record_report_one_estimate_per_step() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &sleeper, seams); let mut messages = vec![ @@ -152,7 +152,7 @@ async fn the_receipt_and_the_usage_record_report_one_estimate_per_step() { /// A block's bytes do not change unless something rewrites them, so the memo /// makes the total linear. This asserts the SHAPE rather than a magic number: /// doubling the steps must not come close to quadrupling the hashing. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn per_step_hashing_grows_with_the_turn_not_with_its_square() { async fn hashes_for(steps: usize) -> usize { let mut script: Vec<_> = (0..steps) @@ -197,7 +197,7 @@ async fn per_step_hashing_grows_with_the_turn_not_with_its_square() { } } let tools = EchoingTools; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &sleeper, seams); let mut messages = vec![ @@ -239,7 +239,7 @@ async fn per_step_hashing_grows_with_the_turn_not_with_its_square() { /// rewrite one result, declare the new revision, emit again, and the manifest must /// name the new bytes. Disabling the invalidation makes this fail — the second /// manifest keeps citing the pre-rewrite id. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_rewritten_block_is_never_served_from_the_digest_memo() { use crate::receipts::{ReceiptLedger, ServedBy, TranscriptRevision}; use stella_protocol::{ToolOutput, ToolResult}; @@ -366,7 +366,7 @@ impl ToolExecutor for BigOutputTools { /// A `Compaction` event names the blocks it stubbed (`evicted_blocks`, captured /// BEFORE mutation). Once those bytes are the eviction stub, no later manifest can /// still be citing their original ids — if one does, the memo outlived them. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn compaction_mid_turn_invalidates_the_receipt_ledgers_digest_memo() { let mut script: Vec<_> = (0..5) .map(|i| { @@ -384,7 +384,7 @@ async fn compaction_mid_turn_invalidates_the_receipt_ledgers_digest_memo() { calls: Arc::new(AtomicU32::new(0)), }; let tools = BigOutputTools { filler: 4_000 }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let config = EngineConfig { // Small enough that a couple of 4 KB results blow it, and no summarizer so // the rewrite comes purely from eviction/aging. @@ -456,7 +456,7 @@ async fn compaction_mid_turn_invalidates_the_receipt_ledgers_digest_memo() { assert!(checked > 0, "manifests must cite blocks"); } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_compaction_pass_journals_the_replacement_bytes_it_wrote() { // The wiring witness for #1667, on the same fixture as the memo test // above: the driver must forward the pass's replacement records onto the @@ -481,7 +481,7 @@ async fn a_compaction_pass_journals_the_replacement_bytes_it_wrote() { calls: Arc::new(AtomicU32::new(0)), }; let tools = BigOutputTools { filler: 4_000 }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let config = EngineConfig { compaction_budget_tokens: 800, summarize_overflow: false, diff --git a/crates/stella-core/src/driver/tests/context_efficiency.rs b/crates/stella-core/src/driver/tests/context_efficiency.rs index 0c4b34c1ec..4b8b2c8983 100644 --- a/crates/stella-core/src/driver/tests/context_efficiency.rs +++ b/crates/stella-core/src/driver/tests/context_efficiency.rs @@ -33,7 +33,7 @@ impl ToolExecutor for BigOutputTools { } } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_long_turn_ages_old_tool_results_far_below_the_compaction_budget() { // The #1285 step-loop witness. Thirteen tool-bearing steps at ~5 KB each // is ~16k estimated tokens, so before the retention pass the step loop @@ -59,7 +59,7 @@ async fn a_long_turn_ages_old_tool_results_far_below_the_compaction_budget() { let tools = BigOutputTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let config = EngineConfig { compaction_budget_tokens: 20_000, ..EngineConfig::default() @@ -129,7 +129,7 @@ fn huge_partial() -> String { ) } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_spent_allowance_retains_an_elided_partial_not_the_scratchpad() { // Post-mortem §3.3 lever 2, the terminal half. The continuation path // already elides what it retains; the path that ENDS the turn after the @@ -147,7 +147,7 @@ async fn a_spent_allowance_retains_an_elided_partial_not_the_scratchpad() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &sleeper, seams); let mut messages = vec![ @@ -186,7 +186,7 @@ async fn a_spent_allowance_retains_an_elided_partial_not_the_scratchpad() { drain_events(&mut rx); } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_truncated_step_with_tool_calls_retains_elided_narration() { // The other residual hole: a step cut at the output limit that still // carried a tool call proceeds normally — and used to retain its whole @@ -213,7 +213,7 @@ async fn a_truncated_step_with_tool_calls_retains_elided_narration() { let tools = CountingTools { calls: tool_calls.clone(), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &sleeper, seams); let mut messages = vec![ diff --git a/crates/stella-core/src/driver/tests/context_overflow.rs b/crates/stella-core/src/driver/tests/context_overflow.rs index 0143ade702..c7805aaeef 100644 --- a/crates/stella-core/src/driver/tests/context_overflow.rs +++ b/crates/stella-core/src/driver/tests/context_overflow.rs @@ -108,6 +108,10 @@ impl crate::retry::Sleeper for NoSleep { async fn sleep(&self, _duration_ms: u64) {} // The floor: a test that asserts on retry timing wants no spread in it. + fn now(&self) -> std::time::Instant { + std::time::Instant::now() + } + fn jitter(&self, _upper: u64) -> u64 { 0 } diff --git a/crates/stella-core/src/driver/tests/deadline_notice.rs b/crates/stella-core/src/driver/tests/deadline_notice.rs index 8a6580b37d..ef2af88b07 100644 --- a/crates/stella-core/src/driver/tests/deadline_notice.rs +++ b/crates/stella-core/src/driver/tests/deadline_notice.rs @@ -66,6 +66,10 @@ impl crate::retry::Sleeper for NoSleep { async fn sleep(&self, _duration_ms: u64) {} // The floor: a test that asserts on retry timing wants no spread in it. + fn now(&self) -> std::time::Instant { + std::time::Instant::now() + } + fn jitter(&self, _upper: u64) -> u64 { 0 } diff --git a/crates/stella-core/src/driver/tests/lifecycle_bus.rs b/crates/stella-core/src/driver/tests/lifecycle_bus.rs index c2d9c44447..b5c3c5533c 100644 --- a/crates/stella-core/src/driver/tests/lifecycle_bus.rs +++ b/crates/stella-core/src/driver/tests/lifecycle_bus.rs @@ -136,6 +136,10 @@ impl crate::retry::Sleeper for NoSleep { async fn sleep(&self, _duration_ms: u64) {} // The floor: a test that asserts on retry timing wants no spread in it. + fn now(&self) -> std::time::Instant { + std::time::Instant::now() + } + fn jitter(&self, _upper: u64) -> u64 { 0 } diff --git a/crates/stella-core/src/driver/tests/live_services.rs b/crates/stella-core/src/driver/tests/live_services.rs index 58fe0df16b..45cb2e98e1 100644 --- a/crates/stella-core/src/driver/tests/live_services.rs +++ b/crates/stella-core/src/driver/tests/live_services.rs @@ -98,7 +98,7 @@ async fn run( script: TokioMutex::new(script.into_iter().map(Ok).collect()), calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, tools, EngineConfig::default(), &sleeper, seams); let mut messages = vec![ @@ -120,7 +120,7 @@ async fn run( /// done — and the engine sends it back once with the handle named. On `main` /// the executor is never asked and the turn completes on the first /// declaration, having said nothing about the process it left listening. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_turn_declaring_done_with_a_service_up_is_asked_about_it() { let tools = Serving::with(vec![service("proc-1", Some("docs"))]); let (outcome, messages, events) = run( @@ -176,7 +176,7 @@ async fn a_turn_declaring_done_with_a_service_up_is_asked_about_it() { /// services, so nothing is appended and nothing is emitted. The executor is /// still asked — the engine cannot know the answer without asking — which is /// why the port must be a peek. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_turn_with_nothing_running_is_left_untouched() { let tools = Serving::with(Vec::new()); let (outcome, messages, events) = run( @@ -215,7 +215,7 @@ async fn a_turn_with_nothing_running_is_left_untouched() { /// everything history keeps of a truncated step), not the raw cut-off /// narration. Before the fix the raw partial rode into history verbatim, /// re-sent on every later step of the turn. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_truncated_declaration_reaches_history_elided_not_raw() { let tools = Serving::with(vec![service("proc-1", Some("docs"))]); let long_partial = "the fix is coming ".repeat(200); // > 2000 chars, elidable @@ -259,7 +259,7 @@ async fn a_truncated_declaration_reaches_history_elided_not_raw() { /// `gate-ab` shape (#2663) the shared nudge window exists to prevent, on a /// gate whose condition, unlike the prove-it gate's, the model may /// legitimately choose not to clear. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_confirmed_service_never_re_arms_the_question() { let tools = Serving::with(vec![service("proc-1", None)]); let (outcome, messages, _) = run( diff --git a/crates/stella-core/src/driver/tests/loop_abort.rs b/crates/stella-core/src/driver/tests/loop_abort.rs index e399d5bc84..48a3e39c48 100644 --- a/crates/stella-core/src/driver/tests/loop_abort.rs +++ b/crates/stella-core/src/driver/tests/loop_abort.rs @@ -101,7 +101,7 @@ fn grep_call() -> CompletionResultAlias { /// with one different call — and then falls straight into a second loop. The /// turn already spent its one warning, so the second loop's first detection /// must abort rather than grind to the step cap. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_loop_that_resumes_after_a_successful_course_correction_still_aborts() { // grep ×3 (detect + steer) → read_file (the course correction) → grep // forever (ScriptedProvider repeats its last entry). @@ -124,7 +124,7 @@ async fn a_loop_that_resumes_after_a_successful_course_correction_still_aborts() let tools = ConstantTools { calls: tool_calls.clone(), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; // A low cap, so a broken ladder shows up as "ground to the cap" in a // second. let config = EngineConfig { @@ -204,7 +204,7 @@ fn exact_repeat_only(max_steps: usize) -> EngineConfig { /// Both loops are `grep`, which is the half that makes this a witness rather /// than a restatement of #1524: tool-name identity called them one loop, so /// nothing short of comparing arguments can tell "obeyed" from "ignored". -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_second_distinct_loop_earns_its_own_steer_before_the_turn_dies() { let grep_for = |call_id: &str, pattern: &str| { call_of( @@ -231,7 +231,7 @@ async fn a_second_distinct_loop_earns_its_own_steer_before_the_turn_dies() { let tools = ConstantTools { calls: tool_calls.clone(), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, exact_repeat_only(30), &sleeper, seams); let mut messages = vec![ @@ -318,7 +318,7 @@ async fn a_second_distinct_loop_earns_its_own_steer_before_the_turn_dies() { /// The abort's reason must still not claim the warned loop "persisted" — it /// did not, this is a different loop — and the whole abort must reach the bus /// as exactly one `Error` event. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_third_loop_is_not_steered_and_is_not_blamed_on_the_warned_one() { let read_of = |call_id: &str, path: &str| { call_of(call_id, "read_file", serde_json::json!({ "path": path })) @@ -341,7 +341,7 @@ async fn a_third_loop_is_not_steered_and_is_not_blamed_on_the_warned_one() { let tools = ConstantTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, exact_repeat_only(30), &sleeper, seams); let mut messages = vec![ @@ -433,7 +433,7 @@ impl Provider for MutatingGrepProvider { /// returned the same bytes. On the live turn that produced this shape the /// tool ran 38 times before an accidental exact triple finally tripped the /// old detector. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_tool_answering_identically_to_every_new_argument_is_steered_then_killed() { let provider = MutatingGrepProvider { calls: Arc::new(AtomicU32::new(0)), @@ -442,7 +442,7 @@ async fn a_tool_answering_identically_to_every_new_argument_is_steered_then_kill let tools = ConstantTools { calls: tool_calls.clone(), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let config = EngineConfig { max_steps: Some(60), ..EngineConfig::default() @@ -494,7 +494,7 @@ async fn a_tool_answering_identically_to_every_new_argument_is_steered_then_kill /// THE reported shape (issue #1477's ArenaBench trace): two read-only tool /// calls, then a short line with no terminal punctuation standing in for a /// result. This must never reach `Completed`. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_confident_zero_never_reports_as_completed() { let provider = ScriptedProvider { id: "scripted".into(), @@ -512,7 +512,7 @@ async fn a_confident_zero_never_reports_as_completed() { let tools = ConstantTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &sleeper, seams); let mut messages = vec![ @@ -555,7 +555,7 @@ async fn a_confident_zero_never_reports_as_completed() { /// Zero tool calls this turn must never trip the check, however short or /// unterminated the answer — a task answerable without investigation at all /// is an ordinary short completion, not an abstain. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_direct_zero_tool_answer_still_completes() { let provider = ScriptedProvider { id: "scripted".into(), @@ -565,7 +565,7 @@ async fn a_direct_zero_tool_answer_still_completes() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &sleeper, seams); let mut messages = vec![ @@ -589,7 +589,7 @@ async fn a_direct_zero_tool_answer_still_completes() { /// last line says. `CountingTools` (shared by other driver tests, declared /// in the parent `tests.rs`) declares its one tool, `bash`, as NOT /// read-only. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_turn_that_did_mutating_work_still_completes_despite_a_bare_closing_line() { let provider = ScriptedProvider { id: "scripted".into(), @@ -602,7 +602,7 @@ async fn a_turn_that_did_mutating_work_still_completes_despite_a_bare_closing_li let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &sleeper, seams); let mut messages = vec![ @@ -624,7 +624,7 @@ async fn a_turn_that_did_mutating_work_still_completes_despite_a_bare_closing_li /// A properly terminated short answer following read-only investigation is /// not a confident zero — only an UNTERMINATED, orientation-shaped line /// trips the check. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_terminated_short_answer_after_investigation_still_completes() { let provider = ScriptedProvider { id: "scripted".into(), @@ -641,7 +641,7 @@ async fn a_terminated_short_answer_after_investigation_still_completes() { let tools = ConstantTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &sleeper, seams); let mut messages = vec![ @@ -671,7 +671,7 @@ async fn a_terminated_short_answer_after_investigation_still_completes() { /// The paired `LoopDetected` is asserted alongside: it is what /// establishes that the steer under inspection really came from the loop rung, /// so the cause assertion is about the emitter and not about this test's setup. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_stuck_loop_steer_names_the_loop_rung_as_its_cause() { let grep_open = |call_id: &str| { call_of( @@ -692,7 +692,7 @@ async fn a_stuck_loop_steer_names_the_loop_rung_as_its_cause() { let tools = ConstantTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, exact_repeat_only(30), &sleeper, seams); let mut messages = vec![ diff --git a/crates/stella-core/src/driver/tests/model_fallback.rs b/crates/stella-core/src/driver/tests/model_fallback.rs index dd457a06f7..08be2a88f1 100644 --- a/crates/stella-core/src/driver/tests/model_fallback.rs +++ b/crates/stella-core/src/driver/tests/model_fallback.rs @@ -172,6 +172,10 @@ impl crate::retry::Sleeper for NoSleep { async fn sleep(&self, _duration_ms: u64) {} // The floor: a test that asserts on retry timing wants no spread in it. + fn now(&self) -> std::time::Instant { + std::time::Instant::now() + } + fn jitter(&self, _upper: u64) -> u64 { 0 } diff --git a/crates/stella-core/src/driver/tests/output_budget.rs b/crates/stella-core/src/driver/tests/output_budget.rs index dd949d549b..bd8da7f5bd 100644 --- a/crates/stella-core/src/driver/tests/output_budget.rs +++ b/crates/stella-core/src/driver/tests/output_budget.rs @@ -160,6 +160,10 @@ impl crate::retry::Sleeper for NoSleep { async fn sleep(&self, _duration_ms: u64) {} // The floor: a test that asserts on retry timing wants no spread in it. + fn now(&self) -> std::time::Instant { + std::time::Instant::now() + } + fn jitter(&self, _upper: u64) -> u64 { 0 } diff --git a/crates/stella-core/src/driver/tests/parked_wait.rs b/crates/stella-core/src/driver/tests/parked_wait.rs index 14039d5300..b28e9d2055 100644 --- a/crates/stella-core/src/driver/tests/parked_wait.rs +++ b/crates/stella-core/src/driver/tests/parked_wait.rs @@ -111,7 +111,7 @@ fn ci_wait_call() -> CompletionResultAlias { /// The witness: the watched state changes on the THIRD engine-side probe, /// and the model is re-invoked exactly once — not three times — with the /// wake delta on the transcript tail and zero poll debris anywhere in it. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_change_on_the_nth_probe_re_invokes_the_model_exactly_once() { let provider = ScriptedProvider { id: "scripted".into(), @@ -124,7 +124,7 @@ async fn a_change_on_the_nth_probe_re_invokes_the_model_exactly_once() { let tools = ParkingTools::depositing(ci_wait_request(600), 3); let probe_calls = tools.probe_calls.clone(); let wake_calls = tools.wake_calls.clone(); - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &sleeper, seams); let mut messages = vec![ @@ -225,7 +225,7 @@ async fn a_change_on_the_nth_probe_re_invokes_the_model_exactly_once() { /// A condition that never changes wakes the model once with the timeout /// marked — never N poll-steps, and never a silent hang past the deadline. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn an_unchanged_condition_wakes_once_at_the_deadline() { let provider = ScriptedProvider { id: "scripted".into(), @@ -239,7 +239,7 @@ async fn an_unchanged_condition_wakes_once_at_the_deadline() { let tools = ParkingTools::depositing(ci_wait_request(20), u32::MAX); let probe_calls = tools.probe_calls.clone(); let wake_calls = tools.wake_calls.clone(); - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &sleeper, seams); let mut messages = vec![ @@ -283,7 +283,7 @@ async fn an_unchanged_condition_wakes_once_at_the_deadline() { /// A request whose replayed calls are not read-only is refused outright — /// the engine must never mutate on a timer the model cannot see. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_non_read_only_probe_is_refused_not_replayed() { let provider = ScriptedProvider { id: "scripted".into(), @@ -297,7 +297,7 @@ async fn a_non_read_only_probe_is_refused_not_replayed() { request.probe.name = "bash".into(); // not in the read-only schema set let tools = ParkingTools::depositing(request, 1); let probe_calls = tools.probe_calls.clone(); - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &sleeper, seams); let mut messages = vec![ @@ -346,7 +346,7 @@ fn overloaded() -> ProviderError { /// inline ladder now parks — bounded by wall clock, narrated on the stream — /// and the step completes once the provider recovers. On the pre-#2677 /// ladder this run dies at the seventh attempt with `RetriesExhausted`. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn sustained_rate_limiting_parks_within_budget_and_recovers() { // Nine 429s: six absorbed by the inline ladder, three more that only a // park survives. The tenth call succeeds. @@ -361,7 +361,7 @@ async fn sustained_rate_limiting_parks_within_budget_and_recovers() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &sleeper, seams); let mut messages = vec![ @@ -409,7 +409,7 @@ async fn sustained_rate_limiting_parks_within_budget_and_recovers() { /// The witness for #2677's headline defect: a `Retry-After` past the 120s /// inline ceiling used to fail the call fast as `Terminal` even with hours /// of wall clock left. With headroom it is now honored as a parked wait. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_long_retry_after_hint_is_honored_by_parking_when_budget_allows() { let provider = ScriptedProvider { id: "scripted".into(), @@ -425,7 +425,7 @@ async fn a_long_retry_after_hint_is_honored_by_parking_when_budget_allows() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &sleeper, seams); let mut messages = vec![ @@ -462,7 +462,7 @@ async fn a_long_retry_after_hint_is_honored_by_parking_when_budget_allows() { /// the task deadline's remaining headroom still fails fast — waiting less /// than the server asked guarantees a re-429, so the wait would spend the /// budget and buy nothing. The park machinery must not engage. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_hint_past_the_remaining_deadline_still_fails_fast_without_parking() { let provider = ScriptedProvider { id: "scripted".into(), @@ -475,7 +475,7 @@ async fn a_hint_past_the_remaining_deadline_still_fails_fast_without_parking() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &sleeper, seams); let mut messages = vec![ @@ -519,7 +519,7 @@ async fn a_hint_past_the_remaining_deadline_still_fails_fast_without_parking() { /// /// Fails before the change with `TurnOutcome::Aborted` and a /// `RetriesExhausted` on the stream. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_sustained_529_brownout_parks_within_budget_and_recovers() { // Nine 529s: six absorbed by the inline ladder, three more that only a // park survives. The tenth call succeeds. @@ -534,7 +534,7 @@ async fn a_sustained_529_brownout_parks_within_budget_and_recovers() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &sleeper, seams); let mut messages = vec![ @@ -584,7 +584,7 @@ async fn a_sustained_529_brownout_parks_within_budget_and_recovers() { /// aborts. Without this, "529 recovers" and "every retryable failure now /// waits six hours" look identical from inside the diff — and the second is a /// far worse bug than the one #2742 fixes. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_sustained_transport_fault_still_exhausts_the_ladder_and_aborts() { let script: Vec> = (0..9) .map(|_| Err(ProviderError::transport("connection reset by peer"))) @@ -597,7 +597,7 @@ async fn a_sustained_transport_fault_still_exhausts_the_ladder_and_aborts() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &sleeper, seams); let mut messages = vec![ @@ -652,7 +652,7 @@ impl crate::ports::TurnSteering for StopOnAsk { /// `RetriesExhausted` + `Error` pair, recording a provider failure for what /// was a person's decision. The provider really was rate limiting; that is /// simply not why the turn ended. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_soft_stop_during_a_park_ends_the_turn_as_a_deliberate_stop() { let provider = ScriptedProvider { id: "scripted".into(), @@ -662,7 +662,7 @@ async fn a_soft_stop_during_a_park_ends_the_turn_as_a_deliberate_stop() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; // Ask 1 is the step boundary's, which must answer "no" or the turn ends // before a park exists; ask 2 is the park's first per-chunk tick. let steering = StopOnAsk { diff --git a/crates/stella-core/src/driver/tests/provider_outcomes.rs b/crates/stella-core/src/driver/tests/provider_outcomes.rs index 70dc4378dd..c97570d1a1 100644 --- a/crates/stella-core/src/driver/tests/provider_outcomes.rs +++ b/crates/stella-core/src/driver/tests/provider_outcomes.rs @@ -83,6 +83,10 @@ impl crate::retry::Sleeper for NoSleep { async fn sleep(&self, _duration_ms: u64) {} // The floor: a test that asserts on retry timing wants no spread in it. + fn now(&self) -> std::time::Instant { + std::time::Instant::now() + } + fn jitter(&self, _upper: u64) -> u64 { 0 } diff --git a/crates/stella-core/src/driver/tests/requery.rs b/crates/stella-core/src/driver/tests/requery.rs index 2f1dd874ce..a648d3dc97 100644 --- a/crates/stella-core/src/driver/tests/requery.rs +++ b/crates/stella-core/src/driver/tests/requery.rs @@ -30,6 +30,10 @@ impl Sleeper for NoopSleeper { async fn sleep(&self, _duration_ms: u64) {} // The floor: a test that asserts on retry timing wants no spread in it. + fn now(&self) -> std::time::Instant { + std::time::Instant::now() + } + fn jitter(&self, _upper: u64) -> u64 { 0 } diff --git a/crates/stella-core/src/driver/tests/steer_midturn.rs b/crates/stella-core/src/driver/tests/steer_midturn.rs index ff1d21b997..0f42d8ea51 100644 --- a/crates/stella-core/src/driver/tests/steer_midturn.rs +++ b/crates/stella-core/src/driver/tests/steer_midturn.rs @@ -52,7 +52,7 @@ impl crate::ports::TurnSteering for StopNow { /// closed the pairing; the soft stop lands at the same boundary with the /// same keep-the-transcript contract, so it must repair the same shape or /// the kept history hard-fails the next provider call. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_soft_stop_closes_caller_supplied_open_tool_calls() { let provider = ScriptedProvider { id: "scripted".into(), @@ -62,7 +62,7 @@ async fn a_soft_stop_closes_caller_supplied_open_tool_calls() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let steering = StopNow; let seams = TurnCapabilities { steering: Some(&steering), @@ -111,7 +111,7 @@ async fn a_soft_stop_closes_caller_supplied_open_tool_calls() { ); } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_steer_after_a_tool_round_keeps_every_call_paired_with_its_result() { // Step 0 calls a tool; step 1's boundary is where the steer lands, so the // transcript it appends to ends with the Tool message from step 0. @@ -126,7 +126,7 @@ async fn a_steer_after_a_tool_round_keeps_every_call_paired_with_its_result() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let steering = SteerAtDrain { queue: std::sync::Mutex::new(vec!["also check the tests".into()]), fire_on_drain: 2, @@ -186,7 +186,7 @@ async fn a_steer_after_a_tool_round_keeps_every_call_paired_with_its_result() { /// (consecutive same-role turns are combined), and this witness pins the /// transcript shape so a provider adapter that cannot tolerate it is caught /// here rather than at runtime. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_mid_tool_round_steer_leaves_a_tool_message_immediately_before_it() { let provider = ScriptedProvider { id: "scripted".into(), @@ -199,7 +199,7 @@ async fn a_mid_tool_round_steer_leaves_a_tool_message_immediately_before_it() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let steering = SteerAtDrain { queue: std::sync::Mutex::new(vec!["actually, stop after this".into()]), fire_on_drain: 2, diff --git a/crates/stella-core/src/driver/tests/streaming_deadline.rs b/crates/stella-core/src/driver/tests/streaming_deadline.rs index eaedb6017e..95e21e0518 100644 --- a/crates/stella-core/src/driver/tests/streaming_deadline.rs +++ b/crates/stella-core/src/driver/tests/streaming_deadline.rs @@ -51,7 +51,7 @@ impl Provider for SlowStreamingProvider { /// The test shows the turn does not stop in a span many times the deadline. /// It is still going when the test times out, so the call is dropped in /// flight. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_streaming_generation_outlives_the_deadline_because_it_is_not_stalled() { let calls = Arc::new(AtomicU32::new(0)); let provider = SlowStreamingProvider { @@ -61,7 +61,7 @@ async fn a_streaming_generation_outlives_the_deadline_because_it_is_not_stalled( let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let config = EngineConfig { model_timeout: Some(Duration::from_millis(50)), ..EngineConfig::default() @@ -143,7 +143,7 @@ impl Provider for CallOnlyStreamingProvider { /// Put the tick on the gate's event sender and only text goes through it. A /// call that writes one big file then looks silent. It dies at the deadline, /// and the whole call is paid for. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_call_only_stream_outlives_the_deadline_because_it_is_not_stalled() { let calls = Arc::new(AtomicU32::new(0)); let provider = CallOnlyStreamingProvider { @@ -153,7 +153,7 @@ async fn a_call_only_stream_outlives_the_deadline_because_it_is_not_stalled() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let config = EngineConfig { model_timeout: Some(Duration::from_millis(50)), ..EngineConfig::default() diff --git a/crates/stella-core/src/driver/tests/unbounded_by_default.rs b/crates/stella-core/src/driver/tests/unbounded_by_default.rs index 8406cde7ec..2fa0b165d8 100644 --- a/crates/stella-core/src/driver/tests/unbounded_by_default.rs +++ b/crates/stella-core/src/driver/tests/unbounded_by_default.rs @@ -65,7 +65,7 @@ async fn run_productive_turn(steps: u32, config: EngineConfig) -> (TurnOutcome, script: TokioMutex::new(productive_script(steps)), calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &DistinctTools, config, &sleeper, seams); let mut messages = vec![ @@ -80,7 +80,7 @@ async fn run_productive_turn(steps: u32, config: EngineConfig) -> (TurnOutcome, /// The witness. A thousand steps is five times the old cap. A cap that came /// back at any round number under it would fail this. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_long_productive_turn_runs_to_completion_under_the_default_config() { const STEPS: u32 = 1_000; assert_eq!( @@ -104,7 +104,7 @@ async fn a_long_productive_turn_runs_to_completion_under_the_default_config() { /// A host that sets a cap still gets it. The stop is a `DeliberateStop` and /// it names the number the host set. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_host_set_cap_still_ends_the_turn_where_it_says() { const CAP: usize = 25; let config = EngineConfig { diff --git a/crates/stella-core/src/driver/tests/usage_anchor.rs b/crates/stella-core/src/driver/tests/usage_anchor.rs index 45eb844ca0..5bed702a63 100644 --- a/crates/stella-core/src/driver/tests/usage_anchor.rs +++ b/crates/stella-core/src/driver/tests/usage_anchor.rs @@ -61,7 +61,7 @@ async fn compaction_fired_with_report(usage: CompletionUsage) -> bool { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let mut messages = compactable_history(); let raw = crate::estimator::estimate_conversation_tokens(&messages); let config = EngineConfig { @@ -91,7 +91,7 @@ async fn compaction_fired_with_report(usage: CompletionUsage) -> bool { /// budget (#2671 measured the estimator ~1.8× low; this fixture makes the /// gap decisive rather than marginal). Anchored, the next step's decision is /// `reported + estimate(tail) > budget` and must compact. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn provider_reported_usage_rebases_the_compaction_decision() { let raw = crate::estimator::estimate_conversation_tokens(&compactable_history()); assert!( @@ -113,7 +113,7 @@ async fn provider_reported_usage_rebases_the_compaction_decision() { /// scripted default — reported, all counters zero) anchors nothing, and the /// same conversation under the same budget stays estimate-governed: no /// compaction, exactly the pre-anchor behavior. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn an_absent_usage_report_leaves_the_estimator_in_charge() { assert!( !compaction_fired_with_report(CompletionUsage::reported_zero()).await, @@ -125,7 +125,7 @@ async fn an_absent_usage_report_leaves_the_estimator_in_charge() { /// toward it (they are prompt tokens the provider read — the same rule as /// the calibration feed), so a cache-writing first call anchors just as /// decisively as a plain one. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn cache_write_tokens_count_toward_the_anchor() { let raw = crate::estimator::estimate_conversation_tokens(&compactable_history()); assert!( diff --git a/crates/stella-core/src/driver/tests/usage_completeness.rs b/crates/stella-core/src/driver/tests/usage_completeness.rs index a85432102d..3bbf8f7d5b 100644 --- a/crates/stella-core/src/driver/tests/usage_completeness.rs +++ b/crates/stella-core/src/driver/tests/usage_completeness.rs @@ -2,7 +2,7 @@ use super::*; -#[tokio::test] +#[tokio::test(start_paused = true)] async fn exhausted_worker_call_emits_one_content_free_incompleteness_event() { let provider = ScriptedProvider { id: "anthropic-fallback".into(), @@ -14,7 +14,7 @@ async fn exhausted_worker_call_emits_one_content_free_incompleteness_event() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &sleeper, seams); let mut messages = vec![ @@ -84,7 +84,7 @@ impl Provider for ModelBoundProvider { /// this emit site — 435 such rows in one 16-trial panel. That is exactly the /// population mid-turn model fallback (#2769) re-resolves from, so the /// failures that trigger a swap were the ones unable to say what had failed. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_failed_call_names_the_model_that_made_it() { let provider = ModelBoundProvider { inner: ScriptedProvider { @@ -97,7 +97,7 @@ async fn a_failed_call_names_the_model_that_made_it() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &sleeper, seams); let mut messages = vec![ @@ -141,7 +141,7 @@ async fn a_failed_call_names_the_model_that_made_it() { /// per-attempt observer is the sole path by which a doomed attempt's usage can /// ever be recorded. If it drops the partial, the fix stops at the adapter /// boundary and nothing downstream is any wiser. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_failed_attempts_recovered_usage_reaches_the_event_stream() { let recovered = stella_protocol::PartialUsage { usage: stella_protocol::CompletionUsage { @@ -164,7 +164,7 @@ async fn a_failed_attempts_recovered_usage_reaches_the_event_stream() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &sleeper, seams); let mut messages = vec![ @@ -202,7 +202,7 @@ async fn a_failed_attempts_recovered_usage_reaches_the_event_stream() { assert!(wire.contains("14000"), "the numbers do cross: {wire}"); } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn exhausted_retries_emit_typed_reasons_before_the_error() { // Receipts spec §6.3 (#364 gap 3): `Retry` events only flush for steps // that COMMIT, so a terminally-failed call's doomed attempts were @@ -221,7 +221,7 @@ async fn exhausted_retries_emit_typed_reasons_before_the_error() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &sleeper, seams); let mut messages = vec![ @@ -265,7 +265,7 @@ async fn exhausted_retries_emit_typed_reasons_before_the_error() { ); } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn auth_failure_on_first_attempt_reports_not_retryable() { // #926: a terminal `ProviderError::Auth` on attempt 1 was previously // indistinguishable, at the typed level, from a genuine retry-budget @@ -281,7 +281,7 @@ async fn auth_failure_on_first_attempt_reports_not_retryable() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &sleeper, seams); let mut messages = vec![ @@ -325,7 +325,7 @@ async fn auth_failure_on_first_attempt_reports_not_retryable() { ); } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn successful_retry_keeps_the_failed_attempt_usage_incomplete() { let provider = ScriptedProvider { id: "scripted".into(), @@ -338,7 +338,7 @@ async fn successful_retry_keeps_the_failed_attempt_usage_incomplete() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let config = EngineConfig { retry_policy: RetryPolicy::new(1, 0, 0), ..EngineConfig::default() @@ -384,7 +384,7 @@ async fn successful_retry_keeps_the_failed_attempt_usage_incomplete() { /// asked for — the resolved effort and the effective output ceiling — not /// blanks the Observatory's profile card has to render as "not recorded for /// this run". Fails before #4565, when `StepUsage` had neither field. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn step_usage_carries_the_requests_effort_and_output_ceiling() { let provider = ScriptedProvider { id: "scripted".into(), @@ -394,7 +394,7 @@ async fn step_usage_carries_the_requests_effort_and_output_ceiling() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let config = EngineConfig { effort: Some(stella_protocol::completion::ReasoningEffort::High), max_output_tokens: Some(32_000), @@ -440,7 +440,7 @@ async fn step_usage_carries_the_requests_effort_and_output_ceiling() { /// failing if the emitter re-derived the values from the engine config rather /// than reading the request it dispatched, which is what /// `settlement::RequestShape::of` exists to make impossible. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn step_usage_carries_the_requests_generation_params() { let provider = ScriptedProvider { id: "scripted".into(), @@ -450,7 +450,7 @@ async fn step_usage_carries_the_requests_generation_params() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let asked = stella_protocol::completion::GenerationParams { top_p: Some(0.9), seed: Some(4_621), @@ -510,7 +510,7 @@ fn overflow_messages() -> Vec { messages } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn overflow_summarizer_emits_its_own_usage_envelope() { let provider = ScriptedProvider { id: "scripted".into(), @@ -520,7 +520,7 @@ async fn overflow_summarizer_emits_its_own_usage_envelope() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, overflow_config(), &sleeper, seams); let mut messages = overflow_messages(); @@ -542,7 +542,7 @@ async fn overflow_summarizer_emits_its_own_usage_envelope() { ))); } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn failed_overflow_summarizer_emits_content_free_incompleteness() { let provider = ScriptedProvider { id: "scripted".into(), @@ -555,7 +555,7 @@ async fn failed_overflow_summarizer_emits_content_free_incompleteness() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, overflow_config(), &sleeper, seams); let mut messages = overflow_messages(); @@ -601,7 +601,7 @@ async fn failed_overflow_summarizer_emits_content_free_incompleteness() { /// `run_accounted_call` — the path every management role takes. The two rows /// share a step and differ in `call_seq`, so the join the glossary describes /// is available on both sides. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn two_calls_at_one_step_are_separable_by_their_usage_rows() { const TURN: u32 = 4; @@ -613,7 +613,7 @@ async fn two_calls_at_one_step_are_separable_by_their_usage_rows() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &sleeper, seams) .with_turn_instance(TURN); diff --git a/crates/stella-core/src/driver/tests/user_hooks.rs b/crates/stella-core/src/driver/tests/user_hooks.rs index 88bc3c18ae..8271002bce 100644 --- a/crates/stella-core/src/driver/tests/user_hooks.rs +++ b/crates/stella-core/src/driver/tests/user_hooks.rs @@ -40,7 +40,7 @@ impl HookRunner for RecordingHookRunner { } } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn pre_tool_use_hook_nonzero_exit_blocks_the_tool_and_model_sees_it() { let provider = ScriptedProvider { id: "scripted".into(), @@ -54,7 +54,7 @@ async fn pre_tool_use_hook_nonzero_exit_blocks_the_tool_and_model_sees_it() { let tools = CountingTools { calls: tool_calls.clone(), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let payloads = Arc::new(TokioMutex::new(Vec::new())); let runner = RecordingHookRunner { exit_code: 1, @@ -114,7 +114,7 @@ async fn pre_tool_use_hook_nonzero_exit_blocks_the_tool_and_model_sees_it() { assert!(payloads[0].contains("\"event\":\"PreToolUse\"")); } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn post_tool_use_hook_runs_after_the_tool_and_never_blocks() { let provider = ScriptedProvider { id: "scripted".into(), @@ -128,7 +128,7 @@ async fn post_tool_use_hook_runs_after_the_tool_and_never_blocks() { let tools = CountingTools { calls: tool_calls.clone(), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let payloads = Arc::new(TokioMutex::new(Vec::new())); // Exit 3 (non-zero) proves a *failing* PostToolUse hook is still a // pure observation — it can neither block nor abort the turn. @@ -179,7 +179,7 @@ async fn post_tool_use_hook_runs_after_the_tool_and_never_blocks() { ); } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn non_blocking_hook_failure_surfaces_as_one_retryable_error_event() { // A PostToolUse hook exiting non-zero stays non-blocking (pinned by the // test above) but must no longer vanish: the dispatch path forwards the @@ -196,7 +196,7 @@ async fn non_blocking_hook_failure_surfaces_as_one_retryable_error_event() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let payloads = Arc::new(TokioMutex::new(Vec::new())); let runner = RecordingHookRunner { exit_code: 3, @@ -253,7 +253,7 @@ async fn non_blocking_hook_failure_surfaces_as_one_retryable_error_event() { ); } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn no_hooks_configured_leaves_the_turn_path_unchanged() { // With no hooks attached the tool executes normally and the turn // completes exactly as it did before the hooks seam existed — the @@ -270,7 +270,7 @@ async fn no_hooks_configured_leaves_the_turn_path_unchanged() { let tools = CountingTools { calls: tool_calls.clone(), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; // Built WITHOUT `with_hooks` — `hooks` stays `None`. let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &sleeper, seams); @@ -304,7 +304,7 @@ async fn no_hooks_configured_leaves_the_turn_path_unchanged() { ); } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn run_turn_never_fires_session_start_hooks() { // SessionStart is a session-level event and a host obligation (#2674): // the host fires it once, via `hooks::run_hooks`, while assembling the @@ -319,7 +319,7 @@ async fn run_turn_never_fires_session_start_hooks() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let payloads = Arc::new(TokioMutex::new(Vec::new())); let runner = RecordingHookRunner { exit_code: 0, @@ -358,7 +358,7 @@ async fn run_turn_never_fires_session_start_hooks() { ); } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_hooked_read_fires_its_hook_once_never_for_a_dropped_speculative_attempt() { // A read-only tool with a configured PreToolUse hook. The first stream // attempt announces it and then fails; the retry commits it. The hook @@ -383,7 +383,7 @@ async fn a_hooked_read_fires_its_hook_once_never_for_a_dropped_speculative_attem calls: calls.clone(), executed, }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let payloads = Arc::new(TokioMutex::new(Vec::new())); // exit 0: non-blocking, so the tool runs and the hook is a pure // observation — what matters is how many times it is invoked. @@ -478,7 +478,7 @@ fn bash_schema(read_only: bool) -> ToolSchema { /// `PostToolUse` payload reports the input the tool saw. On the base /// commit stdout JSON is ignored and the tool runs with the model's /// original input, so this fails there. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn pre_tool_use_modify_decision_rewrites_the_input_the_tool_receives() { let provider = ScriptedProvider { id: "scripted".into(), @@ -493,7 +493,7 @@ async fn pre_tool_use_modify_decision_rewrites_the_input_the_tool_receives() { schemas: vec![bash_schema(false)], inputs: inputs.clone(), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let payloads = Arc::new(TokioMutex::new(Vec::new())); let runner = RecordingHookRunner { exit_code: 0, @@ -613,7 +613,7 @@ fn require_approval_fixture(stdout: &str) -> ApprovalFixture { /// parks the dispatch on the attached route: an approving route lets the /// tool run, a denying route blocks it with the human's reason, and the /// route sees the tool's name and `read_only` bit. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_hook_require_approval_parks_on_the_route_and_the_answer_decides() { let ask = r#"{"action":"require_approval","reason":"mutating call"}"#; // Approving route: the tool runs. @@ -624,7 +624,7 @@ async fn a_hook_require_approval_parks_on_the_route_and_the_answer_decides() { runner, hooks, } = require_approval_fixture(ask); - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let route = ScriptedRoute { resolution: crate::hooks::decision::ApprovalRouteResolution::Approved, calls: Arc::new(AtomicU32::new(0)), @@ -712,7 +712,7 @@ async fn a_hook_require_approval_parks_on_the_route_and_the_answer_decides() { /// With no route attached, a `require_approval` decision refuses the call /// and names the missing surface — the same headless posture as #2676's /// broker, never a silent allow or a hang. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn require_approval_without_a_route_refuses_with_the_grant_path() { let ask = r#"{"action":"require_approval","reason":"mutating call"}"#; let ApprovalFixture { @@ -722,7 +722,7 @@ async fn require_approval_without_a_route_refuses_with_the_grant_path() { runner, hooks, } = require_approval_fixture(ask); - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities { hooks: Some((&hooks, &runner)), ..TurnCapabilities::none() @@ -759,7 +759,7 @@ async fn require_approval_without_a_route_refuses_with_the_grant_path() { /// **Deliverable 4, #2684.** The `PreToolUse` payload spells the tool's /// advertised `read_only` bit from its schema, so a "deny anything /// non-read-only" hook needs no tool-name allowlist. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn pre_tool_use_payload_carries_the_schemas_read_only_bit() { let provider = ScriptedProvider { id: "scripted".into(), @@ -773,7 +773,7 @@ async fn pre_tool_use_payload_carries_the_schemas_read_only_bit() { schemas: vec![bash_schema(true)], inputs: Arc::new(TokioMutex::new(Vec::new())), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let payloads = Arc::new(TokioMutex::new(Vec::new())); let runner = RecordingHookRunner { exit_code: 0, @@ -853,7 +853,7 @@ impl HookRunner for ScriptedHookRunner { /// itself as final — and then the next completion stands: the death-spiral /// guard, now a counter. On the once-per-turn boolean this fails: the turn /// completed on the second answer with a single Stop fire. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn an_always_denying_stop_hook_is_consulted_to_the_bound_then_the_turn_stands() { let provider = ScriptedProvider { id: "scripted".into(), @@ -869,7 +869,7 @@ async fn an_always_denying_stop_hook_is_consulted_to_the_bound_then_the_turn_sta let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let payloads = Arc::new(TokioMutex::new(Vec::new())); let runner = RecordingHookRunner { exit_code: 0, @@ -963,7 +963,7 @@ async fn an_always_denying_stop_hook_is_consulted_to_the_bound_then_the_turn_sta /// consulted BOTH times — the fail→pass observation a verification hook /// exists to make. On the once-per-turn boolean this fails: the second /// completion was never offered to the hook (one Stop fire, not two). -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_deny_then_allow_is_reconsulted_and_the_allowed_completion_stands() { let provider = ScriptedProvider { id: "scripted".into(), @@ -977,7 +977,7 @@ async fn a_deny_then_allow_is_reconsulted_and_the_allowed_completion_stands() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let payloads = Arc::new(TokioMutex::new(Vec::new())); let runner = ScriptedHookRunner { stdouts: TokioMutex::new(vec![ @@ -1040,7 +1040,7 @@ async fn a_deny_then_allow_is_reconsulted_and_the_allowed_completion_stands() { /// A failing Stop hook (non-zero exit) never blocks completion — failing /// closed at a turn boundary IS the death spiral, so the failure surfaces /// as a diagnostic and the turn ends normally. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_failing_stop_hook_never_holds_the_turn_open() { let provider = ScriptedProvider { id: "scripted".into(), @@ -1051,7 +1051,7 @@ async fn a_failing_stop_hook_never_holds_the_turn_open() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let runner = RecordingHookRunner { exit_code: 3, stdout: String::new(), @@ -1103,7 +1103,7 @@ async fn a_failing_stop_hook_never_holds_the_turn_open() { /// `ApprovalRouteRequest` was keyed on a tool name and there is no tool here. /// The route was never asked at all, so the approved half is anti-vacuity: the /// same hook, the same engine, and the answer is what changes the outcome. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_stop_hooks_require_approval_resolves_both_ways() { async fn run( resolution: crate::hooks::decision::ApprovalRouteResolution, @@ -1125,7 +1125,7 @@ async fn a_stop_hooks_require_approval_resolves_both_ways() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let runner = RecordingHookRunner { exit_code: 0, stdout: r#"{"action":"require_approval","reason":"verification budget exhausted, continue?"}"# @@ -1210,7 +1210,7 @@ async fn a_stop_hooks_require_approval_resolves_both_ways() { /// The opposite of `PreToolUse`'s posture and so — the module /// docs' § "The Stop gate" argues it: refusing to complete because nobody was /// there to answer is the compact→error→stop-hook→retry spiral, not safety. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_stop_hooks_require_approval_with_no_route_lets_the_turn_complete() { let provider = ScriptedProvider { id: "scripted".into(), @@ -1220,7 +1220,7 @@ async fn a_stop_hooks_require_approval_with_no_route_lets_the_turn_complete() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let runner = RecordingHookRunner { exit_code: 0, stdout: r#"{"action":"require_approval","reason":"ask a human"}"#.into(), @@ -1264,7 +1264,7 @@ async fn a_stop_hooks_require_approval_with_no_route_lets_the_turn_complete() { /// (proven by the summarizer never running at all) and the transcript is /// left un-spliced. On the base commit the `PreCompact` key is unknown and /// the summarizer runs, so this fails there. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn pre_compact_hook_veto_skips_the_summarization_round() { let provider = ScriptedProvider { id: "scripted".into(), @@ -1275,7 +1275,7 @@ async fn pre_compact_hook_veto_skips_the_summarization_round() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let hook_payloads = Arc::new(TokioMutex::new(Vec::new())); let runner = RecordingHookRunner { exit_code: 0, @@ -1337,7 +1337,7 @@ async fn pre_compact_hook_veto_skips_the_summarization_round() { /// A `PreCompact` `modify` decision's `instructions` reach the /// summarizer's request, visibly separated from the span content. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn pre_compact_modify_instructions_reach_the_summarizer_request() { let requests = Arc::new(TokioMutex::new(Vec::new())); let provider = RequestCapturingProvider { @@ -1346,7 +1346,7 @@ async fn pre_compact_modify_instructions_reach_the_summarizer_request() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let runner = RecordingHookRunner { exit_code: 0, stdout: r#"{"action":"modify","payload":{"instructions":"keep every file path verbatim"}}"# diff --git a/crates/stella-core/src/driver/tests/user_hooks/verdicts.rs b/crates/stella-core/src/driver/tests/user_hooks/verdicts.rs index 57feb253f5..723c7927d1 100644 --- a/crates/stella-core/src/driver/tests/user_hooks/verdicts.rs +++ b/crates/stella-core/src/driver/tests/user_hooks/verdicts.rs @@ -16,7 +16,7 @@ use super::*; /// Fails before #3380: `HookDecision::Deny` carried only a `String`, so serde /// dropped the whole `evidence` object as an unknown key and neither consumer /// could have seen a witness, a command, a flip or a digest. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_structured_stop_denial_reaches_the_model_and_the_journal_intact() { let provider = ScriptedProvider { id: "scripted".into(), @@ -29,7 +29,7 @@ async fn a_structured_stop_denial_reaches_the_model_and_the_journal_intact() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let runner = ScriptedHookRunner { stdouts: TokioMutex::new(vec![ r#"{"action":"deny","reason":"the witness is still red", @@ -110,7 +110,7 @@ async fn a_structured_stop_denial_reaches_the_model_and_the_journal_intact() { /// A prose-only denial — every pre-#3380 hook — renders exactly as it always /// did, and its journal payload states that this hook does not verify rather /// than inventing an empty evidence object. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_prose_only_stop_denial_grows_no_evidence_section() { let provider = ScriptedProvider { id: "scripted".into(), @@ -123,7 +123,7 @@ async fn a_prose_only_stop_denial_grows_no_evidence_section() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let runner = ScriptedHookRunner { stdouts: TokioMutex::new(vec![ r#"{"action":"deny","reason":"the checklist is not done"}"#.into(), @@ -169,7 +169,7 @@ async fn a_prose_only_stop_denial_grows_no_evidence_section() { /// /// Fails before #3380: nothing read a bound, so the turn completed on the /// fourth answer whatever the host asked for. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_host_supplied_hold_allowance_buys_more_rounds_than_the_default() { let held = run_always_denying_stop_gate(Some(5), 6).await; assert_eq!( @@ -183,7 +183,7 @@ async fn a_host_supplied_hold_allowance_buys_more_rounds_than_the_default() { /// [`STOP_HOLD_CEILING`](crate::driver::STOP_HOLD_CEILING) rather than /// honoured, so no manifest can buy an unbounded deny → revise → re-check /// loop. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_hold_allowance_above_the_ceiling_is_clamped_not_honoured() { let ceiling = crate::driver::STOP_HOLD_CEILING; let held = run_always_denying_stop_gate(Some(1_000), ceiling as usize + 1).await; @@ -209,7 +209,7 @@ async fn run_always_denying_stop_gate(allowance: Option, answers: usize) -> let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let runner = RecordingHookRunner { exit_code: 0, stdout: r#"{"action":"deny","reason":"not yet"}"#.into(), diff --git a/crates/stella-core/src/driver/tests/zero_copy_request.rs b/crates/stella-core/src/driver/tests/zero_copy_request.rs index 871b338073..9433d58298 100644 --- a/crates/stella-core/src/driver/tests/zero_copy_request.rs +++ b/crates/stella-core/src/driver/tests/zero_copy_request.rs @@ -56,7 +56,7 @@ impl Provider for SliceAddressProvider { /// schema's full JSON parameter document. Two attempts meant two copies. /// /// Identical addresses across both attempts prove there are now zero. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_retried_attempt_re_sends_the_same_slices_it_did_the_first_time() { let provider = SliceAddressProvider { seen: std::sync::Mutex::new(Vec::new()), @@ -65,7 +65,7 @@ async fn a_retried_attempt_re_sends_the_same_slices_it_did_the_first_time() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &sleeper, seams); let (tx, _rx) = mpsc::unbounded_channel(); @@ -113,7 +113,7 @@ async fn a_retried_attempt_re_sends_the_same_slices_it_did_the_first_time() { /// `the_system_prefix_stays_byte_stable_across_a_compacting_turn`, which pins /// the *bytes*. There is no intermediate buffer in which the system prefix /// could drift, because there is no intermediate buffer. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn the_adapter_serializes_off_the_callers_own_transcript() { let provider = SliceAddressProvider { seen: std::sync::Mutex::new(Vec::new()), @@ -122,7 +122,7 @@ async fn the_adapter_serializes_off_the_callers_own_transcript() { let tools = CountingTools { calls: Arc::new(AtomicU32::new(0)), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &sleeper, seams); let (tx, _rx) = mpsc::unbounded_channel(); diff --git a/crates/stella-core/src/driver/turn_clock.rs b/crates/stella-core/src/driver/turn_clock.rs index 1fd536aae4..1282dce4b9 100644 --- a/crates/stella-core/src/driver/turn_clock.rs +++ b/crates/stella-core/src/driver/turn_clock.rs @@ -298,6 +298,7 @@ mod tests { Vec::new(), crate::budget::BudgetGuard::new(stella_protocol::BudgetMode::Off, None, None), &config, + std::time::Instant::now(), ); let clock = TurnClock::read(&config, &state); diff --git a/crates/stella-core/src/goal.rs b/crates/stella-core/src/goal.rs index 6941f786f9..46815c5aa1 100644 --- a/crates/stella-core/src/goal.rs +++ b/crates/stella-core/src/goal.rs @@ -563,6 +563,10 @@ mod tests { async fn sleep(&self, _duration_ms: u64) {} // The floor: a test that asserts on retry timing wants no spread in it. + fn now(&self) -> std::time::Instant { + std::time::Instant::now() + } + fn jitter(&self, _upper: u64) -> u64 { 0 } diff --git a/crates/stella-core/src/retry.rs b/crates/stella-core/src/retry.rs index 20b6fbf9ef..17c9d3d418 100644 --- a/crates/stella-core/src/retry.rs +++ b/crates/stella-core/src/retry.rs @@ -24,10 +24,9 @@ //! keep-alive, budget-derived allowance — and only an unaffordable wait //! still fails fast. Which failures those are is decided at the adapter //! and asked here through [`ProviderError::is_park_eligible`], never -//! re-derived (L-M7 again). Nothing here reads a budget; the supervisor -//! is a port, like [`Sleeper`]. The one clock read left is the monotonic -//! `Instant` that times each attempt for the failure observer, and -//! `make core-no-io` counts it. +//! re-derived (L-M7 again). Nothing here reads a budget or a clock; the +//! supervisor is a port, like [`Sleeper`], and the attempt timing the +//! failure observer receives is two readings of [`Sleeper::now`]. //! //! Per-call timeouts (L-E4) are a caller concern layered on top of //! `attempt_fn`; this module owns "should we try again, and if so after how @@ -40,21 +39,30 @@ //! and `stella-core` would link an entropy source for one draw. use std::future::Future; +use std::time::{Duration, Instant}; use async_trait::async_trait; +use futures_util::future::{Either, select}; use stella_protocol::ProviderError; -/// The backoff port: how `retry_with_backoff` waits between attempts, and -/// where the entropy that spreads those waits apart comes from. +/// The engine's time port: what time it is, how a wait is waited out, and +/// where the entropy that spreads retries apart comes from. /// /// Injectable so retry-loop tests run instantly and deterministically -/// instead of paying real wall-clock delays — the same seam -/// [`crate::ports::Clock`] provides for reading time, but for the one place -/// this crate needs to actually suspend a task. Only the trait lives here — +/// instead of paying real wall-clock delays. Only the trait lives here — /// the production tokio-backed impl belongs to the binary that constructs -/// the engine (the CLI's `runtime` module). +/// the engine (the CLI's `runtime` module), so `stella-core` links neither +/// a timer nor a clock nor an entropy source. [`crate::ports::Clock`] is +/// the millisecond clock a host hands to things that stamp records; this +/// one answers in [`Instant`], which is what a deadline is. /// -/// Both methods are required, with no default. A default `jitter` of zero +/// The three live on one port because a double cannot answer them apart: a +/// sleeper that suspends virtually has moved its own `now`, and a timeout +/// is a sleep racing a call (`bounded`, one function down). tokio's paused +/// runtime is the same shape — one virtual clock behind both `sleep` and +/// `Instant::now`. +/// +/// Every method is required, with no default. A default `jitter` of zero /// would let a host forget the draw and ship a fleet whose workers all wake /// on the same millisecond after a shared 429, silently — the herd this /// jitter exists to break up. A host that wants no jitter writes that down. @@ -63,6 +71,15 @@ pub trait Sleeper: Send + Sync { /// Suspend the current task for `duration_ms` milliseconds. async fn sleep(&self, duration_ms: u64); + /// The monotonic clock's reading now. + /// + /// Every deadline the engine holds is an `Instant` from this reading, + /// and every elapsed time is the difference of two of them. Nothing in + /// `stella-core` calls `Instant::now()` itself — `make core-no-io` + /// refuses it — so a host that answers from a virtual clock replays a + /// turn to the millisecond. + fn now(&self) -> Instant; + /// A uniform draw from `0..=upper`, the spread between the backoff floor /// and its cap that this attempt actually waits. /// @@ -73,6 +90,29 @@ pub trait Sleeper: Send + Sync { fn jitter(&self, upper: u64) -> u64; } +/// Run `call` for at most `limit`, through the port's own sleep. +/// +/// `Some` is the call's answer; `None` means the limit passed first, with +/// the call dropped where it stood. This is `tokio::time::timeout` written +/// against the port: the call is polled first, so a call that is ready is +/// never lost to a sleep that is also ready, and the sleep is the port's, so +/// a paused or virtual host clock bounds the call the way it bounds a +/// backoff. A `&mut` pinned future may be passed and re-armed after a +/// `None`, which is how the idle bounds in [`crate::step`] keep waiting +/// while fragments still arrive. +pub(crate) async fn bounded( + sleeper: &dyn Sleeper, + limit: Duration, + call: F, +) -> Option { + let call = std::pin::pin!(call); + let limit_ms = u64::try_from(limit.as_millis()).unwrap_or(u64::MAX); + match select(call, sleeper.sleep(limit_ms)).await { + Either::Left((output, _)) => Some(output), + Either::Right(((), _)) => None, + } +} + /// Milliseconds per parked-wait chunk: a long rate-limit wait is slept in /// pieces of this size so the supervisor is consulted (and can narrate /// liveness or end the park) at a human cadence rather than once per @@ -454,7 +494,7 @@ where let mut parked_total_ms: u64 = 0; let mut park_streak: u32 = 0; loop { - let attempt_started = std::time::Instant::now(); + let attempt_started = sleeper.now(); match attempt_fn().await { Ok(value) => { return Ok(RetryOutcome { @@ -464,7 +504,11 @@ where }); } Err(error) => { - observe_failure(attempt + 1, &error, attempt_started.elapsed()); + observe_failure( + attempt + 1, + &error, + sleeper.now().duration_since(attempt_started), + ); if !error.is_retryable() { return Err(error); } @@ -632,6 +676,10 @@ mod tests { .push(duration_ms); } + fn now(&self) -> Instant { + Instant::now() + } + fn jitter(&self, upper: u64) -> u64 { self.rng .lock() @@ -691,11 +739,38 @@ mod tests { impl Sleeper for FixedJitter { async fn sleep(&self, _duration_ms: u64) {} + fn now(&self) -> Instant { + Instant::now() + } + fn jitter(&self, _upper: u64) -> u64 { self.0 } } + /// The timeout is the port's sleep, not tokio's timer. A sleeper whose + /// sleep returns at once times out a pending call at once, with no + /// runtime clock consulted; a call that is already ready still wins. + /// A timeout built on `tokio::time::timeout` would wait the real hour. + #[tokio::test] + async fn bounded_races_the_ports_own_sleep() { + let sleeper = NoopSleeper::default(); + let hour = std::time::Duration::from_secs(3_600); + assert_eq!( + bounded(&sleeper, hour, std::future::pending::()).await, + None + ); + assert_eq!( + bounded(&sleeper, hour, std::future::ready(5u8)).await, + Some(5) + ); + assert_eq!( + sleeper.delays_ms.lock().unwrap().as_slice(), + &[3_600_000], + "the pending call slept the whole limit through the port; the ready one slept nothing" + ); + } + /// The draw comes from the port. A sleeper answering a fixed number /// lands the delay at exactly the floor plus that number, which no RNG /// this crate seeded itself could promise. A ladder drawing from diff --git a/crates/stella-core/src/step.rs b/crates/stella-core/src/step.rs index 59131a1892..a2d21b2bbd 100644 --- a/crates/stella-core/src/step.rs +++ b/crates/stella-core/src/step.rs @@ -433,6 +433,7 @@ impl TurnState { messages: Vec, budget: BudgetGuard, config: &EngineConfig, + now: std::time::Instant, ) -> Self { age_session_output_ceilings(config); Self { @@ -451,7 +452,7 @@ impl TurnState { memos: TurnMemos::new(config.turn_instance, config.lifecycle_enabled), length_continuations: 0, stop_hook_consults: 0, - started_at: std::time::Instant::now(), + started_at: now, pace: StepPace::default(), steps_since_requery: 0, cancel: CancelToken::new(), @@ -462,7 +463,11 @@ impl TurnState { /// money, calibration model, loop-steer latch and step index — see the /// module docs for exactly what resuming costs. #[must_use] - pub fn from_checkpoint(checkpoint: Checkpoint, config: &EngineConfig) -> Self { + pub fn from_checkpoint( + checkpoint: Checkpoint, + config: &EngineConfig, + now: std::time::Instant, + ) -> Self { age_session_output_ceilings(config); Self { messages: checkpoint.messages, @@ -498,7 +503,7 @@ impl TurnState { // latch re-arms, which only re-permits one more held-open round — // the same bounded-allowance reasoning as `length_continuations`. stop_hook_consults: 0, - started_at: std::time::Instant::now(), + started_at: now, pace: StepPace::default(), steps_since_requery: 0, cancel: CancelToken::new(), @@ -1044,8 +1049,9 @@ impl<'a> BorrowedTurn<'a> { messages: &'a mut Vec, budget: &'a mut BudgetGuard, config: &EngineConfig, + now: std::time::Instant, ) -> Self { - let state = TurnState::new(std::mem::take(messages), *budget, config); + let state = TurnState::new(std::mem::take(messages), *budget, config, now); Self { messages, budget, @@ -1172,6 +1178,7 @@ impl StreamProgress { /// it, and a slow-but-progressing provider is never cut off by time another /// attempt already spent. pub(crate) async fn bounded_generation( + sleeper: &dyn crate::retry::Sleeper, limit: Option, progress: &StreamProgress, call: F, @@ -1185,9 +1192,9 @@ where let mut call = std::pin::pin!(call); let mut seen = progress.count(); loop { - match tokio::time::timeout(limit, &mut call).await { - Ok(result) => return result, - Err(_) => { + match crate::retry::bounded(sleeper, limit, &mut call).await { + Some(result) => return result, + None => { // The window elapsed. Whether that is a fault depends on what // arrived during it: any fragment at all means the provider is // answering, so re-arm and keep waiting. Only a window that @@ -1243,6 +1250,7 @@ where /// [`bounded_generation`]'s is: the task is out of time, so retrying only /// spends more of a deadline that has already run out. pub(crate) async fn deadline_bounded_generation( + sleeper: &dyn crate::retry::Sleeper, idle_limit: Option, task_deadline: Option, progress: &StreamProgress, @@ -1251,14 +1259,14 @@ pub(crate) async fn deadline_bounded_generation( where F: Future>, { - let generation = bounded_generation(idle_limit, progress, call); + let generation = bounded_generation(sleeper, idle_limit, progress, call); let Some(deadline) = task_deadline else { return generation.await; }; - let remaining = deadline.saturating_duration_since(std::time::Instant::now()); - match tokio::time::timeout(remaining, generation).await { - Ok(result) => result, - Err(_) => Err(ProviderError::Terminal(format!( + let remaining = deadline.saturating_duration_since(sleeper.now()); + match crate::retry::bounded(sleeper, remaining, generation).await { + Some(result) => result, + None => Err(ProviderError::Terminal(format!( "generation exceeded the task's remaining wall clock ({}ms): \ the task deadline ran out mid-call", remaining.as_millis() @@ -1283,7 +1291,7 @@ where /// /// Content-free by construction, same privacy rule as every other /// `UsageIncomplete` envelope: no request or response body is representable. -pub(crate) struct CancelUsageGuard { +pub(crate) struct CancelUsageGuard<'a> { pub(crate) events: EventSender, pub(crate) role: stella_protocol::ModelCallRole, pub(crate) provider: String, @@ -1293,17 +1301,20 @@ pub(crate) struct CancelUsageGuard { /// [`stella_protocol::UNKNOWN_MODEL`] when the adapter names none. pub(crate) model: String, pub(crate) started: std::time::Instant, + /// The clock `started` was read from, asked again at the drop: a `Drop` + /// has no caller to hand it `now`. + pub(crate) sleeper: &'a dyn crate::retry::Sleeper, pub(crate) armed: bool, pub(crate) attempt_in_flight: Arc, } -impl CancelUsageGuard { +impl CancelUsageGuard<'_> { pub(crate) fn disarm(&mut self) { self.armed = false; } } -impl Drop for CancelUsageGuard { +impl Drop for CancelUsageGuard<'_> { fn drop(&mut self) { if !self.armed || !self.attempt_in_flight.load(Ordering::SeqCst) { return; @@ -1313,7 +1324,7 @@ impl Drop for CancelUsageGuard { provider: self.provider.clone(), model: self.model.clone(), reason: stella_protocol::UsageIncompleteReason::Cancelled, - duration_ms: self.started.elapsed().as_millis() as u64, + duration_ms: self.sleeper.now().duration_since(self.started).as_millis() as u64, retries: None, // A hard cancel drops the call future mid-flight, so no adapter // ever returned an error to salvage from. The server-side cost of diff --git a/crates/stella-core/src/step/tests.rs b/crates/stella-core/src/step/tests.rs index 4ceb6087b3..275ac6e03a 100644 --- a/crates/stella-core/src/step/tests.rs +++ b/crates/stella-core/src/step/tests.rs @@ -1,6 +1,26 @@ use super::*; use stella_protocol::ToolCall; +/// A [`crate::retry::Sleeper`] on real tokio time, for the bounds below: they +/// race a trickling call against a sleep, and only a sleep that takes time +/// can lose that race. +struct RealTime; + +#[async_trait::async_trait] +impl crate::retry::Sleeper for RealTime { + async fn sleep(&self, duration_ms: u64) { + tokio::time::sleep(Duration::from_millis(duration_ms)).await; + } + + fn now(&self) -> std::time::Instant { + std::time::Instant::now() + } + + fn jitter(&self, _upper: u64) -> u64 { + 0 + } +} + fn checkpoint_fixture() -> Checkpoint { let mut budget = BudgetGuard::new(BudgetMode::Enforced, Some(2.5), Some(10.0)); let _ = budget.record_spend(0.375); @@ -33,7 +53,12 @@ fn checkpoint_fixture() -> Checkpoint { attachments: Vec::new(), }, ]; - let state = TurnState::new(messages, budget, &EngineConfig::default()); + let state = TurnState::new( + messages, + budget, + &EngineConfig::default(), + std::time::Instant::now(), + ); let mut state = state; state.total_cost_usd = 0.375; state.calibration_model = Some("glm-5.2".into()); @@ -73,7 +98,11 @@ fn checkpoint_round_trips_byte_identically() { #[test] fn a_restored_turn_state_carries_the_whole_checkpoint() { let checkpoint = checkpoint_fixture(); - let state = TurnState::from_checkpoint(checkpoint.clone(), &EngineConfig::default()); + let state = TurnState::from_checkpoint( + checkpoint.clone(), + &EngineConfig::default(), + std::time::Instant::now(), + ); assert_eq!(state.step(), checkpoint.step); assert_eq!(state.messages(), checkpoint.messages.as_slice()); @@ -111,7 +140,8 @@ fn a_resumed_turn_cannot_re_open_a_steer_the_checkpoint_says_it_spent() { "the count must survive the JSON, not just the struct" ); - let state = TurnState::from_checkpoint(decoded, &EngineConfig::default()); + let state = + TurnState::from_checkpoint(decoded, &EngineConfig::default(), std::time::Instant::now()); assert_eq!( state.loop_steer.remaining(), 0, @@ -127,7 +157,8 @@ fn a_resumed_turn_cannot_re_open_a_steer_the_checkpoint_says_it_spent() { .remove("loop_steers_spent"); let legacy = Checkpoint::from_json(&legacy.to_string()).expect("a v1 checkpoint still decodes"); assert_eq!(legacy.loop_steers_spent, 0); - let restored = TurnState::from_checkpoint(legacy, &EngineConfig::default()); + let restored = + TurnState::from_checkpoint(legacy, &EngineConfig::default(), std::time::Instant::now()); assert_eq!( restored.loop_steer.spent(), 1, @@ -198,6 +229,7 @@ fn a_cancel_closes_every_open_tool_use_so_the_history_stays_reusable() { ], BudgetGuard::new(BudgetMode::Off, None, None), &EngineConfig::default(), + std::time::Instant::now(), ); assert!( state.cancel_outcome(&events).is_none(), @@ -239,7 +271,11 @@ fn closing_open_calls_is_a_no_op_on_a_well_paired_transcript() { let (tx, _rx) = tokio::sync::mpsc::unbounded_channel(); let events = EventSender::new(tx); let checkpoint = checkpoint_fixture(); - let mut state = TurnState::from_checkpoint(checkpoint, &EngineConfig::default()); + let mut state = TurnState::from_checkpoint( + checkpoint, + &EngineConfig::default(), + std::time::Instant::now(), + ); let before = state.messages().len(); state.cancel_token().cancel(); let _ = state.cancel_outcome(&events); @@ -256,6 +292,7 @@ fn a_cloned_token_cancels_the_turn_it_came_from() { Vec::new(), BudgetGuard::new(BudgetMode::Off, None, None), &EngineConfig::default(), + std::time::Instant::now(), ); let handed_off = state.cancel_token(); assert!(!state.cancel.is_cancelled()); @@ -280,6 +317,7 @@ fn the_compaction_budget_is_latched_once_the_calibration_corrects() { Vec::new(), BudgetGuard::new(BudgetMode::Observed, None, None), &EngineConfig::default(), + std::time::Instant::now(), ); state.calibration_model = Some("anthropic/claude-fable-5".to_string()); @@ -310,6 +348,7 @@ fn a_warming_calibrations_identity_is_served_live_never_captured() { Vec::new(), BudgetGuard::new(BudgetMode::Observed, None, None), &EngineConfig::default(), + std::time::Instant::now(), ); state.calibration_model = Some("claude-fable-5".to_string()); @@ -346,6 +385,7 @@ fn nothing_is_latched_while_the_model_is_still_unknown() { Vec::new(), BudgetGuard::new(BudgetMode::Observed, None, None), &EngineConfig::default(), + std::time::Instant::now(), ); assert!(state.calibration_model.is_none()); @@ -373,6 +413,7 @@ fn the_usage_anchor_survives_appends_and_dies_with_a_rewrite() { vec![CompletionMessage::system("sys")], BudgetGuard::new(BudgetMode::Observed, None, None), &EngineConfig::default(), + std::time::Instant::now(), ); let latched = (150_000, 1.0); assert_eq!( @@ -428,6 +469,7 @@ fn stub_completion_result() -> CompletionResult { async fn a_call_with_no_idle_bound_is_still_cut_by_the_task_deadline() { let progress = StreamProgress::default(); let result = deadline_bounded_generation( + &RealTime, None, Some(std::time::Instant::now() + Duration::from_millis(30)), &progress, @@ -463,6 +505,7 @@ async fn a_trickling_generation_under_a_generous_idle_bound_is_still_cut_by_the_ Ok(stub_completion_result()) }; let result = deadline_bounded_generation( + &RealTime, Some(Duration::from_secs(10)), Some(std::time::Instant::now() + Duration::from_millis(30)), &progress, @@ -489,6 +532,7 @@ async fn a_trickling_generation_under_a_generous_idle_bound_is_still_cut_by_the_ async fn no_armed_deadline_leaves_the_idle_bound_as_the_only_cut() { let progress = StreamProgress::default(); let result = deadline_bounded_generation( + &RealTime, Some(Duration::from_millis(20)), None, &progress, diff --git a/crates/stella-core/src/subagent.rs b/crates/stella-core/src/subagent.rs index 9b961a6c49..f455ce1450 100644 --- a/crates/stella-core/src/subagent.rs +++ b/crates/stella-core/src/subagent.rs @@ -692,7 +692,7 @@ impl Engine<'_> { let deadline = bounded_by_ceiling( carve.task_deadline(), self.config.tool_timeout, - std::time::Instant::now(), + self.sleeper.now(), ); let outcome = match refusal(spec, &carve, &deadline) { Some(refusal) => SubAgentOutcome::Refused { @@ -950,7 +950,7 @@ impl Engine<'_> { // The parent's own post-settlement numbers, since the child's ticks // were dropped at the boundary and a HUD would otherwise sit stale // for the whole child run. - let _ = events.send(budget.tick_event(std::time::Instant::now())); + let _ = events.send(budget.tick_event(self.sleeper.now())); let absorbed_messages = messages.len().saturating_sub(seeded); let steps = tally.steps(); diff --git a/crates/stella-core/src/subagent/tests.rs b/crates/stella-core/src/subagent/tests.rs index a57ae6c408..c980411cd9 100644 --- a/crates/stella-core/src/subagent/tests.rs +++ b/crates/stella-core/src/subagent/tests.rs @@ -27,10 +27,19 @@ use crate::retry::Sleeper; // ---- fakes ----------------------------------------------------------- -pub(crate) struct NoSleep; +/// A `Sleeper` on tokio's clock, run paused by every test here: a sleep is +/// free while the runtime is idle and still lets a pending child finish +/// first, and `now` reads the same virtual timeline. +pub(crate) struct TokioSleeper; #[async_trait] -impl Sleeper for NoSleep { - async fn sleep(&self, _duration_ms: u64) {} +impl Sleeper for TokioSleeper { + async fn sleep(&self, duration_ms: u64) { + tokio::time::sleep(std::time::Duration::from_millis(duration_ms)).await; + } + + fn now(&self) -> std::time::Instant { + tokio::time::Instant::now().into_std() + } // The floor: a test that asserts on retry timing wants no spread in it. fn jitter(&self, _upper: u64) -> u64 { diff --git a/crates/stella-core/src/subagent/tests/failure_and_events.rs b/crates/stella-core/src/subagent/tests/failure_and_events.rs index 0d95d8d59a..124ed91230 100644 --- a/crates/stella-core/src/subagent/tests/failure_and_events.rs +++ b/crates/stella-core/src/subagent/tests/failure_and_events.rs @@ -2,7 +2,7 @@ use super::*; // ---- failure is data -------------------------------------------------- -#[tokio::test] +#[tokio::test(start_paused = true)] async fn an_aborted_child_salvages_the_last_answer_it_paid_for() { // The child answers, is told to keep going, then hits its step cap. Its // text is real work; throwing it away with the transcript would make an @@ -33,7 +33,7 @@ async fn an_aborted_child_salvages_the_last_answer_it_paid_for() { &parent_provider, &tools, EngineConfig::default(), - &NoSleep, + &TokioSleeper, seams, ); let mut budget = BudgetGuard::new(BudgetMode::Observed, None, None); @@ -60,7 +60,7 @@ async fn an_aborted_child_salvages_the_last_answer_it_paid_for() { } } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_failed_child_never_becomes_an_error_the_parent_has_to_handle() { // The provider errors terminally on the first call. The parent still // gets a value back, with the reason in it. @@ -72,7 +72,7 @@ async fn a_failed_child_never_becomes_an_error_the_parent_has_to_handle() { &parent_provider, &tools, EngineConfig::default(), - &NoSleep, + &TokioSleeper, seams, ); let mut budget = BudgetGuard::new(BudgetMode::Observed, None, None); @@ -96,7 +96,7 @@ async fn a_failed_child_never_becomes_an_error_the_parent_has_to_handle() { } } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn nesting_deeper_than_the_cap_is_refused_before_spending() { let parent_provider = ScriptedProvider::new(vec![]); let child_provider = ScriptedProvider::new(vec![Ok(text_result("hi", 0.01))]); @@ -106,7 +106,7 @@ async fn nesting_deeper_than_the_cap_is_refused_before_spending() { &parent_provider, &tools, EngineConfig::default(), - &NoSleep, + &TokioSleeper, seams, ); let mut budget = BudgetGuard::new(BudgetMode::Observed, None, None); @@ -136,7 +136,7 @@ async fn nesting_deeper_than_the_cap_is_refused_before_spending() { // ---- the event plane -------------------------------------------------- -#[tokio::test] +#[tokio::test(start_paused = true)] async fn the_childs_stage_and_narration_never_reach_the_parents_stream() { let parent_provider = ScriptedProvider::new(vec![]); let child_provider = ScriptedProvider::new(vec![ @@ -149,7 +149,7 @@ async fn the_childs_stage_and_narration_never_reach_the_parents_stream() { &parent_provider, &tools, EngineConfig::default(), - &NoSleep, + &TokioSleeper, seams, ); let mut budget = BudgetGuard::new(BudgetMode::Observed, None, None); @@ -200,7 +200,7 @@ async fn the_childs_stage_and_narration_never_reach_the_parents_stream() { /// The bracket cannot answer this, and that is the whole reason for the field: /// independent delegates are dispatched concurrently, so `Started`/`Finished` /// pairs interleave and enclose each other's calls. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_childs_metering_records_name_the_child_that_spent_them() { let parent_provider = ScriptedProvider::new(vec![]); let child_provider = ScriptedProvider::new(vec![Ok(text_result("done", 0.02))]); @@ -210,7 +210,7 @@ async fn a_childs_metering_records_name_the_child_that_spent_them() { &parent_provider, &tools, EngineConfig::default(), - &NoSleep, + &TokioSleeper, seams, ); let mut budget = BudgetGuard::new(BudgetMode::Observed, None, None); @@ -249,12 +249,18 @@ async fn a_childs_metering_records_name_the_child_that_spent_them() { /// A call the lead made itself is the lead's, and must not acquire an id from /// a child that ran beside it. `None` is a fact — "the lead spent this" — so a /// reader summing by spender gets the turn's real shape. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn the_leads_own_calls_name_no_sub_agent() { let provider = ScriptedProvider::new(vec![Ok(text_result("answered", 0.05))]); let tools = MixedTools::default(); let seams = TurnCapabilities::none(); - let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &NoSleep, seams); + let engine = Engine::assemble( + &provider, + &tools, + EngineConfig::default(), + &TokioSleeper, + seams, + ); let mut budget = BudgetGuard::new(BudgetMode::Observed, None, None); let (tx, mut rx) = mpsc::unbounded_channel(); let mut messages = vec![CompletionMessage::user("do it")]; @@ -272,7 +278,7 @@ async fn the_leads_own_calls_name_no_sub_agent() { assert_eq!(spenders, vec![None], "{spenders:?}"); } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn step_usage_and_tool_activity_reach_the_parent_so_cost_rolls_up() { // Dropping StepUsage is precisely how child spend would vanish from // `stella stats` and quietly falsify the $/resolved-task number. @@ -287,7 +293,7 @@ async fn step_usage_and_tool_activity_reach_the_parent_so_cost_rolls_up() { &parent_provider, &tools, EngineConfig::default(), - &NoSleep, + &TokioSleeper, seams, ); let mut budget = BudgetGuard::new(BudgetMode::Observed, None, None); @@ -339,7 +345,7 @@ async fn step_usage_and_tool_activity_reach_the_parent_so_cost_rolls_up() { /// under the parent execution id with nothing naming the child — the bracket /// cannot stand in for it, because independent delegates are dispatched /// concurrently and no `Started`/`Finished` pair encloses a particular call. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_childs_tool_calls_name_the_child_that_ran_them() { let parent_provider = ScriptedProvider::new(vec![]); let child_provider = ScriptedProvider::new(vec![ @@ -352,7 +358,7 @@ async fn a_childs_tool_calls_name_the_child_that_ran_them() { &parent_provider, &tools, EngineConfig::default(), - &NoSleep, + &TokioSleeper, seams, ); let mut budget = BudgetGuard::new(BudgetMode::Observed, None, None); @@ -388,7 +394,7 @@ async fn a_childs_tool_calls_name_the_child_that_ran_them() { /// The other half of the same fact: a call the lead made itself is the lead's, /// and `None` is that answer rather than the absence of one. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn the_leads_own_tool_calls_name_no_sub_agent() { let provider = ScriptedProvider::new(vec![ Ok(tool_call_result("read_file", "c1", 0.01)), @@ -396,7 +402,13 @@ async fn the_leads_own_tool_calls_name_no_sub_agent() { ]); let tools = MixedTools::default(); let seams = TurnCapabilities::none(); - let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &NoSleep, seams); + let engine = Engine::assemble( + &provider, + &tools, + EngineConfig::default(), + &TokioSleeper, + seams, + ); let mut budget = BudgetGuard::new(BudgetMode::Observed, None, None); let (tx, mut rx) = mpsc::unbounded_channel(); let mut messages = vec![CompletionMessage::user("do it")]; diff --git a/crates/stella-core/src/subagent/tests/fork_scope.rs b/crates/stella-core/src/subagent/tests/fork_scope.rs index 2b49cefc00..ed35a2d248 100644 --- a/crates/stella-core/src/subagent/tests/fork_scope.rs +++ b/crates/stella-core/src/subagent/tests/fork_scope.rs @@ -15,7 +15,7 @@ use async_trait::async_trait; use stella_protocol::{BudgetMode, CompletionRequestRef, CompletionResult, ProviderError}; use tokio::sync::mpsc; -use super::{MixedTools, NoSleep, ScriptedProvider, text_result, tool_call_result}; +use super::{MixedTools, ScriptedProvider, TokioSleeper, text_result, tool_call_result}; use crate::subagent::*; // ---- forked-skill scoping: allowed_tools + effort (#2682) --------------- @@ -65,7 +65,7 @@ impl Provider for RecordingProvider { /// #2682: a child scoped by `allowed_tools` sees only the granted schemas /// AND cannot execute outside them by guessing a name — the grant is /// structural (`GrantedTools`), not prompt-side. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_grant_scoped_child_cannot_see_or_call_outside_its_grant() { let parent_provider = ScriptedProvider::new(vec![]); // The child tries the un-granted read first, then the granted write, @@ -81,7 +81,7 @@ async fn a_grant_scoped_child_cannot_see_or_call_outside_its_grant() { &parent_provider, &tools, EngineConfig::default(), - &NoSleep, + &TokioSleeper, seams, ); let mut budget = BudgetGuard::new(BudgetMode::Observed, None, None); @@ -123,7 +123,7 @@ async fn a_grant_scoped_child_cannot_see_or_call_outside_its_grant() { /// #2682: a spec-pinned effort reaches the child's requests; absent, the /// child inherits the parent's. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_forked_skills_effort_overrides_the_parents_and_absent_inherits() { use stella_protocol::ReasoningEffort; @@ -134,7 +134,7 @@ async fn a_forked_skills_effort_overrides_the_parents_and_absent_inherits() { ..EngineConfig::default() }; let seams = TurnCapabilities::none(); - let parent = Engine::assemble(&parent_provider, &tools, config, &NoSleep, seams); + let parent = Engine::assemble(&parent_provider, &tools, config, &TokioSleeper, seams); let mut budget = BudgetGuard::new(BudgetMode::Observed, None, None); let (tx, _rx) = mpsc::unbounded_channel(); diff --git a/crates/stella-core/src/subagent/tests/seams.rs b/crates/stella-core/src/subagent/tests/seams.rs index dd25973b9f..65f9e6612d 100644 --- a/crates/stella-core/src/subagent/tests/seams.rs +++ b/crates/stella-core/src/subagent/tests/seams.rs @@ -19,7 +19,7 @@ impl TurnSteering for SpySteering { } } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_child_honors_the_soft_stop_but_never_eats_the_parents_steering() { let parent_provider = ScriptedProvider::new(vec![]); let child_provider = ScriptedProvider::new(vec![Ok(text_result("hi", 0.01))]); @@ -36,7 +36,7 @@ async fn a_child_honors_the_soft_stop_but_never_eats_the_parents_steering() { &parent_provider, &tools, EngineConfig::default(), - &NoSleep, + &TokioSleeper, seams, ); let mut budget = BudgetGuard::new(BudgetMode::Observed, None, None); @@ -91,7 +91,7 @@ impl TurnGate for CountingGate { } } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_child_polls_the_parents_pause_gate() { // `assess` dropped the gate when it hand-rolled a verifier engine, so a // paused session kept spending inside the verifier. Inheritance here is @@ -111,7 +111,7 @@ async fn a_child_polls_the_parents_pause_gate() { &parent_provider, &tools, EngineConfig::default(), - &NoSleep, + &TokioSleeper, seams, ); let mut budget = BudgetGuard::new(BudgetMode::Observed, None, None); @@ -138,7 +138,7 @@ async fn a_child_polls_the_parents_pause_gate() { /// sub-agent dispatcher can give a child the seams of the turn that asked for /// it. This pins the two properties that make it safe to call blindly at /// dispatch time. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn owned_turn_controls_stop_a_child_without_clobbering_an_attached_gate() { let parent_provider = ScriptedProvider::new(vec![]); let child_provider = ScriptedProvider::new(vec![Ok(text_result("hi", 0.01))]); @@ -161,7 +161,7 @@ async fn owned_turn_controls_stop_a_child_without_clobbering_an_attached_gate() &parent_provider, &tools, EngineConfig::default(), - &NoSleep, + &TokioSleeper, seams, ) .with_turn_controls(&controls); @@ -216,8 +216,14 @@ fn turn_controls_carrying_both_seams_give_a_child_both() { assert!(!both.is_empty()); let seams = TurnCapabilities::none(); - let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &NoSleep, seams) - .with_turn_controls(&both); + let engine = Engine::assemble( + &provider, + &tools, + EngineConfig::default(), + &TokioSleeper, + seams, + ) + .with_turn_controls(&both); assert!(engine.gate.is_some(), "the pause must survive the steering"); assert!( @@ -237,8 +243,14 @@ fn empty_turn_controls_leave_an_engine_exactly_as_it_was() { gate: Some(&gate), ..TurnCapabilities::none() }; - let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &NoSleep, seams) - .with_turn_controls(¬hing); + let engine = Engine::assemble( + &provider, + &tools, + EngineConfig::default(), + &TokioSleeper, + seams, + ) + .with_turn_controls(¬hing); assert!( engine.gate.is_some(), @@ -287,7 +299,7 @@ fn attribution_with_no_bus_is_a_no_op() { // ---- receipts --------------------------------------------------------- -#[tokio::test] +#[tokio::test(start_paused = true)] async fn the_child_claims_its_own_receipt_turn_slot() { // Receipts key on (execution_id, turn_instance, step, call_seq) and every // turn restarts step at 0, so a child sharing the parent's slot would @@ -301,7 +313,7 @@ async fn the_child_claims_its_own_receipt_turn_slot() { ..EngineConfig::default() }; let seams = TurnCapabilities::none(); - let parent = Engine::assemble(&parent_provider, &tools, config, &NoSleep, seams); + let parent = Engine::assemble(&parent_provider, &tools, config, &TokioSleeper, seams); let mut budget = BudgetGuard::new(BudgetMode::Observed, None, None); let (tx, mut rx) = mpsc::unbounded_channel(); @@ -358,7 +370,7 @@ impl HookRunner for RecordingHookRunner { /// on the parent fires around the child's turn. It carries the child's id /// and, for `Stop`, its outcome. It never blocks the child, even when the /// hook itself fails with a non-zero exit. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn subagent_start_and_stop_hooks_fire_around_a_child_turn() { let parent_provider = ScriptedProvider::new(vec![]); let child_provider = ScriptedProvider::new(vec![Ok(text_result("done", 0.01))]); @@ -386,7 +398,7 @@ async fn subagent_start_and_stop_hooks_fire_around_a_child_turn() { &parent_provider, &tools, EngineConfig::default(), - &NoSleep, + &TokioSleeper, seams, ); let mut budget = BudgetGuard::new(BudgetMode::Observed, None, None); @@ -449,7 +461,7 @@ async fn subagent_start_and_stop_hooks_fire_around_a_child_turn() { /// The bus is attached to the PARENT, and the child inherits it — which is /// why one turn opens on it here rather than two: this parent never drives a /// turn of its own. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_forked_child_stamps_the_subagent_fork_lane() { let bus = HookBus::new("fork-lane-test", crate::ports::FixedClock(0)); let seen: std::sync::Arc>> = @@ -472,7 +484,7 @@ async fn a_forked_child_stamps_the_subagent_fork_lane() { &parent_provider, &tools, EngineConfig::default(), - &NoSleep, + &TokioSleeper, seams, ); let mut budget = BudgetGuard::new(BudgetMode::Observed, None, None); diff --git a/crates/stella-core/src/subagent/tests/spend_and_cancellation.rs b/crates/stella-core/src/subagent/tests/spend_and_cancellation.rs index 04c39d5da3..34be303bf3 100644 --- a/crates/stella-core/src/subagent/tests/spend_and_cancellation.rs +++ b/crates/stella-core/src/subagent/tests/spend_and_cancellation.rs @@ -44,7 +44,7 @@ impl ToolExecutor for SpendingTools { /// that spend in at the next step boundary is what keeps `--spend-limit` a hard /// ceiling once turns nest; deferring to end-of-turn would let the parent and /// its children each run to the cap independently. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn tool_dispatched_child_spend_aborts_the_parent_at_the_next_step_boundary() { let provider = ScriptedProvider::new(vec![ // The parent's own calls are free; every dollar here is the child's. @@ -58,7 +58,13 @@ async fn tool_dispatched_child_spend_aborts_the_parent_at_the_next_step_boundary drains: AtomicUsize::new(0), }; let seams = TurnCapabilities::none(); - let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &NoSleep, seams); + let engine = Engine::assemble( + &provider, + &tools, + EngineConfig::default(), + &TokioSleeper, + seams, + ); let mut messages = vec![CompletionMessage::user("go")]; let mut budget = BudgetGuard::new(BudgetMode::Enforced, None, Some(1.0)); let (tx, mut rx) = mpsc::unbounded_channel(); @@ -113,7 +119,7 @@ async fn tool_dispatched_child_spend_aborts_the_parent_at_the_next_step_boundary ); } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn the_drain_is_destructive_so_child_spend_is_never_charged_twice() { let provider = ScriptedProvider::new(vec![ Ok(tool_call_result("delegate", "c1", 0.0)), @@ -125,7 +131,13 @@ async fn the_drain_is_destructive_so_child_spend_is_never_charged_twice() { drains: AtomicUsize::new(0), }; let seams = TurnCapabilities::none(); - let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &NoSleep, seams); + let engine = Engine::assemble( + &provider, + &tools, + EngineConfig::default(), + &TokioSleeper, + seams, + ); let mut messages = vec![CompletionMessage::user("go")]; let mut budget = BudgetGuard::new(BudgetMode::Observed, None, None); let (tx, _rx) = mpsc::unbounded_channel(); @@ -217,7 +229,7 @@ impl Provider for HangAfterScript { /// `CancelBracket`, the `Started` bracket stayed open forever and every /// ceiling-bearing caller had to forge a `Finished` it could only fill with /// `steps: 0`. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_cancelled_child_closes_its_bracket_with_committed_steps_and_cost() { let parent_provider = ScriptedProvider::new(vec![]); let hang_reached = std::sync::Arc::new(tokio::sync::Notify::new()); @@ -232,7 +244,7 @@ async fn a_cancelled_child_closes_its_bracket_with_committed_steps_and_cost() { &parent_provider, &tools, EngineConfig::default(), - &NoSleep, + &TokioSleeper, seams, ); let mut budget = BudgetGuard::new(BudgetMode::Observed, None, None); @@ -367,7 +379,7 @@ async fn a_child_is_bounded_by_the_ceiling_its_whole_run_sits_under() { model_timeout: None, ..EngineConfig::default() }, - &NoSleep, + &TokioSleeper, seams, ); let mut budget = BudgetGuard::new(BudgetMode::Observed, None, None); @@ -463,7 +475,7 @@ fn a_ceiling_at_or_under_the_reserve_is_refused_not_floored() { /// **Loud refusal, half two.** The refusal is not just the seam function — /// it has to reach the parent as a [`SubAgentOutcome::Refused`] with zero /// model calls and zero cost, the same contract every other refusal keeps. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_ceiling_too_short_refuses_the_whole_spawn_loudly() { use std::time::Duration; @@ -479,7 +491,7 @@ async fn a_ceiling_too_short_refuses_the_whole_spawn_loudly() { tool_timeout: Some(Duration::from_secs(30)), ..EngineConfig::default() }, - &NoSleep, + &TokioSleeper, seams, ); let mut budget = BudgetGuard::new(BudgetMode::Observed, None, None); diff --git a/crates/stella-core/src/subagent/tests/tools_and_budget.rs b/crates/stella-core/src/subagent/tests/tools_and_budget.rs index 9882c16442..daae2369a1 100644 --- a/crates/stella-core/src/subagent/tests/tools_and_budget.rs +++ b/crates/stella-core/src/subagent/tests/tools_and_budget.rs @@ -2,7 +2,7 @@ use super::*; // ---- read-only by default --------------------------------------------- -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_read_only_child_cannot_execute_a_mutating_tool_even_when_it_tries() { let parent_provider = ScriptedProvider::new(vec![]); let child_provider = ScriptedProvider::new(vec![ @@ -15,7 +15,7 @@ async fn a_read_only_child_cannot_execute_a_mutating_tool_even_when_it_tries() { &parent_provider, &tools, EngineConfig::default(), - &NoSleep, + &TokioSleeper, seams, ); let mut budget = BudgetGuard::new(BudgetMode::Observed, None, None); @@ -37,7 +37,7 @@ async fn a_read_only_child_cannot_execute_a_mutating_tool_even_when_it_tries() { ); } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn write_access_is_opt_in_per_spawn() { let parent_provider = ScriptedProvider::new(vec![]); let child_provider = ScriptedProvider::new(vec![ @@ -50,7 +50,7 @@ async fn write_access_is_opt_in_per_spawn() { &parent_provider, &tools, EngineConfig::default(), - &NoSleep, + &TokioSleeper, seams, ); let mut budget = BudgetGuard::new(BudgetMode::Observed, None, None); @@ -69,7 +69,7 @@ async fn write_access_is_opt_in_per_spawn() { // ---- budget: carved, settled, and never a hole in the accounting ------ -#[tokio::test] +#[tokio::test(start_paused = true)] async fn child_spend_settles_into_the_parent_exactly_once() { let parent_provider = ScriptedProvider::new(vec![]); let child_provider = ScriptedProvider::new(vec![ @@ -82,7 +82,7 @@ async fn child_spend_settles_into_the_parent_exactly_once() { &parent_provider, &tools, EngineConfig::default(), - &NoSleep, + &TokioSleeper, seams, ); let mut budget = BudgetGuard::new(BudgetMode::Observed, None, None); @@ -109,7 +109,7 @@ async fn child_spend_settles_into_the_parent_exactly_once() { ); } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn an_enforced_carve_stops_the_child_without_touching_the_parents_turn() { let parent_provider = ScriptedProvider::new(vec![]); // Each call costs more than the whole carve, so the child trips at the @@ -124,7 +124,7 @@ async fn an_enforced_carve_stops_the_child_without_touching_the_parents_turn() { &parent_provider, &tools, EngineConfig::default(), - &NoSleep, + &TokioSleeper, seams, ); let mut budget = BudgetGuard::new(BudgetMode::Enforced, None, Some(100.0)); @@ -154,7 +154,7 @@ async fn an_enforced_carve_stops_the_child_without_touching_the_parents_turn() { ); } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_child_can_never_be_carved_past_the_parents_remaining_headroom() { // The hard-ceiling property at the spawn boundary: the caller asks for // ten dollars, the parent has four cents left, and the child is bounded @@ -167,7 +167,7 @@ async fn a_child_can_never_be_carved_past_the_parents_remaining_headroom() { &parent_provider, &tools, EngineConfig::default(), - &NoSleep, + &TokioSleeper, seams, ); let mut budget = BudgetGuard::new(BudgetMode::Enforced, None, Some(1.0)); @@ -203,7 +203,7 @@ async fn a_child_can_never_be_carved_past_the_parents_remaining_headroom() { } } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn an_enforced_parent_with_no_headroom_refuses_before_spending_anything() { let parent_provider = ScriptedProvider::new(vec![]); // Would answer happily if it were ever asked. It must not be. @@ -214,7 +214,7 @@ async fn an_enforced_parent_with_no_headroom_refuses_before_spending_anything() &parent_provider, &tools, EngineConfig::default(), - &NoSleep, + &TokioSleeper, seams, ); let mut budget = BudgetGuard::new(BudgetMode::Enforced, None, Some(1.0)); diff --git a/crates/stella-core/src/subagent/tests/witness.rs b/crates/stella-core/src/subagent/tests/witness.rs index 1f56396b1c..7c0703a69f 100644 --- a/crates/stella-core/src/subagent/tests/witness.rs +++ b/crates/stella-core/src/subagent/tests/witness.rs @@ -13,7 +13,7 @@ use super::*; /// for the rest of the session. The report the parent *may* choose to append /// is bounded separately (see /// [`the_report_is_clamped_to_the_spec_cap_and_says_so`]). -#[tokio::test] +#[tokio::test(start_paused = true)] async fn the_parent_transcript_does_not_grow_by_the_childs_intermediate_work() { let parent_provider = ScriptedProvider::new(vec![]); // Four read steps, then an answer: 9 messages of child transcript. @@ -30,7 +30,7 @@ async fn the_parent_transcript_does_not_grow_by_the_childs_intermediate_work() { &parent_provider, &tools, EngineConfig::default(), - &NoSleep, + &TokioSleeper, seams, ); @@ -122,7 +122,7 @@ impl crate::step::CheckpointSink for RecordingSink { /// reaching a terminal outcome retracts the parent's resume point outright — so /// a crash moments later would find either nothing to resume from, or a /// conversation belonging to a different agent. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn a_child_turn_never_writes_or_clears_the_parents_resume_point() { let parent_provider = ScriptedProvider::new(vec![]); // Two tool calls then an answer: the child crosses two step boundaries, so @@ -139,7 +139,7 @@ async fn a_child_turn_never_writes_or_clears_the_parents_resume_point() { ..EngineConfig::default() }; let seams = TurnCapabilities::none(); - let parent = Engine::assemble(&parent_provider, &tools, config, &NoSleep, seams); + let parent = Engine::assemble(&parent_provider, &tools, config, &TokioSleeper, seams); let mut budget = BudgetGuard::new(BudgetMode::Observed, None, None); let (tx, mut rx) = mpsc::unbounded_channel(); @@ -178,7 +178,7 @@ async fn a_child_turn_never_writes_or_clears_the_parents_resume_point() { // ---- context economy is a mechanism, not an intention ----------------- -#[tokio::test] +#[tokio::test(start_paused = true)] async fn the_report_is_clamped_to_the_spec_cap_and_says_so() { let parent_provider = ScriptedProvider::new(vec![]); let child_provider = ScriptedProvider::new(vec![Ok(text_result(&"y".repeat(5_000), 0.001))]); @@ -188,7 +188,7 @@ async fn the_report_is_clamped_to_the_spec_cap_and_says_so() { &parent_provider, &tools, EngineConfig::default(), - &NoSleep, + &TokioSleeper, seams, ); let mut budget = BudgetGuard::new(BudgetMode::Observed, None, None); diff --git a/crates/stella-core/tests/engine_emits_no_stage.rs b/crates/stella-core/tests/engine_emits_no_stage.rs index 2e61263845..519c75f926 100644 --- a/crates/stella-core/tests/engine_emits_no_stage.rs +++ b/crates/stella-core/tests/engine_emits_no_stage.rs @@ -36,6 +36,10 @@ impl Sleeper for NoopSleeper { async fn sleep(&self, _duration_ms: u64) {} // The floor: a test that asserts on retry timing wants no spread in it. + fn now(&self) -> std::time::Instant { + std::time::Instant::now() + } + fn jitter(&self, _upper: u64) -> u64 { 0 } diff --git a/crates/stella-core/tests/hard_drop_write_back.rs b/crates/stella-core/tests/hard_drop_write_back.rs index f40c7fcee9..a545d371e3 100644 --- a/crates/stella-core/tests/hard_drop_write_back.rs +++ b/crates/stella-core/tests/hard_drop_write_back.rs @@ -29,12 +29,23 @@ use stella_protocol::{ MessageRole, Provider, ProviderError, ToolCall, ToolOutput, ToolSchema, }; -struct NoopSleeper; +/// A `Sleeper` on tokio's clock. The tool below hangs for an hour, and the +/// engine's tool timeout is that same clock racing it: a sleeper that +/// returned at once would time the tool out on its first poll and the turn +/// this test drops mid-tool would already be over. +struct TokioSleeper; #[async_trait] -impl Sleeper for NoopSleeper { - async fn sleep(&self, _duration_ms: u64) {} +impl Sleeper for TokioSleeper { + async fn sleep(&self, duration_ms: u64) { + tokio::time::sleep(Duration::from_millis(duration_ms)).await; + } + + fn now(&self) -> std::time::Instant { + std::time::Instant::now() + } // The floor: a test that asserts on retry timing wants no spread in it. + fn jitter(&self, _upper: u64) -> u64 { 0 } @@ -97,7 +108,7 @@ impl ToolExecutor for WedgedTool { async fn dropping_a_turn_mid_tool_still_leaves_the_partial_history_with_the_caller() { let provider = AlwaysCallsATool; let tools = WedgedTool; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &sleeper, seams); diff --git a/crates/stella-core/tests/parallel_dispatch.rs b/crates/stella-core/tests/parallel_dispatch.rs index 39cc06c38c..73b2052064 100644 --- a/crates/stella-core/tests/parallel_dispatch.rs +++ b/crates/stella-core/tests/parallel_dispatch.rs @@ -33,12 +33,22 @@ use stella_protocol::{ }; use tokio::sync::mpsc; -struct NoopSleeper; +/// A `Sleeper` on tokio's clock. The engine's tool timeout is this sleep racing +/// the dispatch, and the barrier test below needs a parked tool to stay +/// parked: a sleeper that returned at once would time it out on the first +/// poll and hand the test a completion it must not see. +struct TokioSleeper; #[async_trait] -impl Sleeper for NoopSleeper { - async fn sleep(&self, _duration_ms: u64) {} +impl Sleeper for TokioSleeper { + async fn sleep(&self, duration_ms: u64) { + tokio::time::sleep(std::time::Duration::from_millis(duration_ms)).await; + } // The floor: a test that asserts on retry timing wants no spread in it. + fn now(&self) -> std::time::Instant { + std::time::Instant::now() + } + fn jitter(&self, _upper: u64) -> u64 { 0 } @@ -130,7 +140,7 @@ async fn sibling_delegate_calls_in_one_step_execute_concurrently() { let tools = BarrierSpawns { barrier: tokio::sync::Barrier::new(2), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &sleeper, seams); let mut messages = vec![ @@ -241,7 +251,7 @@ async fn a_mutating_call_between_spawns_keeps_its_barrier() { let tools = BarrierSpawnsAndEdit { barrier: tokio::sync::Barrier::new(2), }; - let sleeper = NoopSleeper; + let sleeper = TokioSleeper; let seams = TurnCapabilities::none(); let engine = Engine::assemble(&provider, &tools, EngineConfig::default(), &sleeper, seams); let mut messages = vec![ diff --git a/crates/stella-core/tests/spend_gate.rs b/crates/stella-core/tests/spend_gate.rs index 5845239e73..afe3dd7178 100644 --- a/crates/stella-core/tests/spend_gate.rs +++ b/crates/stella-core/tests/spend_gate.rs @@ -65,6 +65,10 @@ impl Sleeper for NoopSleeper { async fn sleep(&self, _duration_ms: u64) {} // The floor: a test that asserts on retry timing wants no spread in it. + fn now(&self) -> std::time::Instant { + std::time::Instant::now() + } + fn jitter(&self, _upper: u64) -> u64 { 0 } diff --git a/crates/stella-core/tests/tool_wall_clock.rs b/crates/stella-core/tests/tool_wall_clock.rs index 867bd551f9..fa5a79996d 100644 --- a/crates/stella-core/tests/tool_wall_clock.rs +++ b/crates/stella-core/tests/tool_wall_clock.rs @@ -40,6 +40,10 @@ impl Sleeper for NoopSleeper { async fn sleep(&self, _duration_ms: u64) {} // The floor: a test that asserts on retry timing wants no spread in it. + fn now(&self) -> std::time::Instant { + std::time::Instant::now() + } + fn jitter(&self, _upper: u64) -> u64 { 0 } diff --git a/crates/stella-engine/src/tests.rs b/crates/stella-engine/src/tests.rs index 2be4bd1121..633fa65212 100644 --- a/crates/stella-engine/src/tests.rs +++ b/crates/stella-engine/src/tests.rs @@ -30,6 +30,10 @@ impl Sleeper for NoopSleeper { async fn sleep(&self, _duration_ms: u64) {} // The floor: a test that asserts on retry timing wants no spread in it. + fn now(&self) -> std::time::Instant { + std::time::Instant::now() + } + fn jitter(&self, _upper: u64) -> u64 { 0 } diff --git a/crates/stella-engine/tests/embedding.rs b/crates/stella-engine/tests/embedding.rs index fe3022e26b..be577efde7 100644 --- a/crates/stella-engine/tests/embedding.rs +++ b/crates/stella-engine/tests/embedding.rs @@ -311,6 +311,10 @@ impl Sleeper for NoopSleeper { async fn sleep(&self, _duration_ms: u64) {} // The floor: a test that asserts on retry timing wants no spread in it. + fn now(&self) -> std::time::Instant { + std::time::Instant::now() + } + fn jitter(&self, _upper: u64) -> u64 { 0 } diff --git a/crates/stella-serve/src/remote.rs b/crates/stella-serve/src/remote.rs index bcc6323975..085915eb08 100644 --- a/crates/stella-serve/src/remote.rs +++ b/crates/stella-serve/src/remote.rs @@ -142,6 +142,10 @@ impl Sleeper for TokioSleeper { tokio::time::sleep(std::time::Duration::from_millis(duration_ms)).await; } + fn now(&self) -> std::time::Instant { + std::time::Instant::now() + } + fn jitter(&self, upper: u64) -> u64 { rand::rng().random_range(0..=upper) } diff --git a/crates/stella-serve/tests/checkpoint.rs b/crates/stella-serve/tests/checkpoint.rs index b3f869d52d..429f9c3128 100644 --- a/crates/stella-serve/tests/checkpoint.rs +++ b/crates/stella-serve/tests/checkpoint.rs @@ -299,6 +299,7 @@ async fn a_served_resume_point_reconstitutes_the_turn_it_came_from() { let resumed = stella_engine::TurnState::from_checkpoint( stella_engine::decode_checkpoint(&body).expect("the wire body decodes as a Checkpoint"), &stella_engine::EngineConfig::default(), + std::time::Instant::now(), ); assert_eq!( diff --git a/scripts/check-core-no-io.py b/scripts/check-core-no-io.py index fc2b420128..f20ef041e1 100755 --- a/scripts/check-core-no-io.py +++ b/scripts/check-core-no-io.py @@ -20,18 +20,21 @@ stripped first — a test may read a fixture; the engine may not. **The manifest.** `[dependencies]` may not name a crate whose purpose is I/O -or entropy, and `tokio` may take only the features that schedule and time -(`sync`, `time`, `macros`, `rt`). `[dev-dependencies]` is exempt for the -same reason `#[cfg(test)]` is. +or entropy, and `tokio` may take only the features that schedule (`sync`, +`macros`, `rt`) — not `time`, because the engine waits through +`retry::Sleeper` and never owns a timer. `[dev-dependencies]` is exempt for +the same reason `#[cfg(test)]` is. **The ratchet.** Reads of the monotonic clock — `Instant::now()` — are not -I/O, and the engine's deadline arithmetic is full of them. They are ambient -state all the same: a turn that reads the clock itself cannot be replayed -from its record, which is why `ports::Clock` exists. Each file's count is -recorded in `scripts/core-no-io-baseline.txt` and may only go down. +I/O. They are ambient state all the same: a turn that reads the clock itself +cannot be replayed from its record, which is why `Sleeper::now` exists. Each +file's count is recorded in `scripts/core-no-io-baseline.txt` and may only +go down. The baseline reached empty on 2026-09-10, so every read now fails; +the ratchet stays so a read that lands reports as the count it is. `--update` refuses to add a file or raise a count, so a red run is cleared -by reading the clock once at the edge and passing `now` in, never by -recording the read. +by reading the clock once through the port and passing `now` in, never by +recording the read. `.elapsed()` is the same read in disguise and sits on +the floor. This is a fact about the repository rather than about a crate, so it is never scoped by CARGO_SCOPE (AGENTS.md § "The gate"), and a text-level walk keeps it @@ -112,9 +115,14 @@ "the engine suspends through retry::Sleeper, never a thread", ), ( - "tokio::time::sleep", - re.compile(r"\btokio\s*::\s*time\s*::\s*sleep\b"), - "the engine suspends through retry::Sleeper, never a timer of its own", + "tokio's timer", + re.compile(r"\btokio\s*::\s*time\b"), + "the engine waits through retry::Sleeper — retry::bounded is the timeout, sleep is the sleep", + ), + ( + "Instant::elapsed", + re.compile(r"\.\s*elapsed\s*\(\s*\)"), + "a hidden Instant::now(): subtract two readings of Sleeper::now instead", ), ( "an entropy source", @@ -156,7 +164,7 @@ "which", "git2", } -TOKIO_FEATURES_ALLOWED = {"sync", "time", "macros", "rt"} +TOKIO_FEATURES_ALLOWED = {"sync", "macros", "rt"} SECTION = re.compile(r"^\s*\[([^\]]+)\]\s*$") DEP_KEY = re.compile(r"^\s*([A-Za-z0-9_-]+)\s*(?:\.\s*workspace\s*)?=") @@ -273,8 +281,8 @@ def read_baseline(path: Path) -> dict[str, int]: # # A DOWN-ONLY ratchet. It records the clock reads that predate the guard. It # may never gain a file or a larger count, and `--update` refuses both. To -# clear a red run, read the clock once at the edge and pass `now` in, or take -# `ports::Clock`. Never record the read. The list is meant to reach empty. +# clear a red run, read the clock through `Sleeper::now` and pass `now` in. +# Never record the read. The list reached empty on 2026-09-10 and stays so. # # Regenerate after removing a read: ./scripts/check-core-no-io.py --update """ diff --git a/scripts/core-no-io-baseline.txt b/scripts/core-no-io-baseline.txt index 17d3d1a9ee..2e51ff711e 100644 --- a/scripts/core-no-io-baseline.txt +++ b/scripts/core-no-io-baseline.txt @@ -2,17 +2,7 @@ # # A DOWN-ONLY ratchet. It records the clock reads that predate the guard. It # may never gain a file or a larger count, and `--update` refuses both. To -# clear a red run, read the clock once at the edge and pass `now` in, or take -# `ports::Clock`. Never record the read. The list is meant to reach empty. +# clear a red run, read the clock through `Sleeper::now` and pass `now` in. +# Never record the read. The list reached empty on 2026-09-10 and stays so. # # Regenerate after removing a read: ./scripts/check-core-no-io.py --update -2 crates/stella-core/src/accounted_call.rs -1 crates/stella-core/src/bus.rs -4 crates/stella-core/src/driver.rs -1 crates/stella-core/src/driver/completion.rs -2 crates/stella-core/src/driver/dispatch.rs -2 crates/stella-core/src/driver/rate_limit.rs -1 crates/stella-core/src/driver/settlement.rs -1 crates/stella-core/src/retry.rs -3 crates/stella-core/src/step.rs -2 crates/stella-core/src/subagent.rs diff --git a/scripts/test-core-no-io.sh b/scripts/test-core-no-io.sh index 6ed815214e..c55aa25de7 100755 --- a/scripts/test-core-no-io.sh +++ b/scripts/test-core-no-io.sh @@ -38,7 +38,7 @@ fail=0 new_core() { # local dir="$TMP/$1/crates/stella-core" mkdir -p "$dir/src" "$TMP/$1/scripts" - printf '[package]\nname = "stella-core"\n\n[dependencies]\nserde = "1"\ntokio = { version = "1", features = ["sync", "time"] }\n\n[dev-dependencies]\nrand = "0.10"\n' >"$dir/Cargo.toml" + printf '[package]\nname = "stella-core"\n\n[dependencies]\nserde = "1"\ntokio = { version = "1", features = ["sync"] }\n\n[dev-dependencies]\nrand = "0.10"\n' >"$dir/Cargo.toml" printf 'pub mod driver;\n' >"$dir/src/lib.rs" printf 'pub fn drive() {}\n' >"$dir/src/driver.rs" : >"$TMP/$1/scripts/core-no-io-baseline.txt" @@ -105,6 +105,14 @@ src="$(new_core spawn)" printf 'pub fn drive() { let _ = std::process::Command::new("sh"); }\n' >"$src/driver.rs" want "std::process is reported" expect-fail spawn "std::process" +src="$(new_core elapsed)" +printf 'pub fn drive(started: std::time::Instant) -> u128 { started.elapsed().as_millis() }\n' >"$src/driver.rs" +want "Instant::elapsed is reported as the clock read it hides" expect-fail elapsed "Instant::elapsed" + +src="$(new_core timer)" +printf 'pub async fn drive() { let _ = tokio::time::timeout(std::time::Duration::from_secs(1), async {}).await; }\n' >"$src/driver.rs" +want "a tokio timer is reported" expect-fail timer "tokio's timer" + # ── Fabrication: what is not shipping code ─────────────────────────────────── src="$(new_core cfgtest)" cat >"$src/driver.rs" <<'RS' @@ -146,6 +154,10 @@ src="$(new_core tokiofs)" printf '[package]\nname = "stella-core"\n\n[dependencies]\ntokio = { version = "1", features = ["sync", "fs"] }\n' >"$TMP/tokiofs/crates/stella-core/Cargo.toml" want "a tokio I/O feature is reported" expect-fail tokiofs "tokio feature \`fs\`" +src="$(new_core tokiotime)" +printf '[package]\nname = "stella-core"\n\n[dependencies]\ntokio = { version = "1", features = ["sync", "time"] }\n' >"$TMP/tokiotime/crates/stella-core/Cargo.toml" +want "tokio's time feature is reported" expect-fail tokiotime "tokio feature \`time\`" + src="$(new_core tokiotable)" printf '[package]\nname = "stella-core"\n\n[dependencies.tokio]\nversion = "1"\nfeatures = ["process"]\n' >"$TMP/tokiotable/crates/stella-core/Cargo.toml" want "the table spelling of a tokio dependency is read too" expect-fail tokiotable "tokio feature \`process\`"