diff --git a/CHANGELOG.md b/CHANGELOG.md index 995648431..b98c98f6f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,11 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). on the Responses wire, `reasoning_effort` on Chat Completions), so a strong tier can run at `max` behind a client that sends `high`. `extra_body` only fills absent keys and could not do this. Rejected on Anthropic clients. +- **Escalation handoff note** — an optional `handoff_note` in the escalation + block is handed to the strong tier on the latching turn and every strong-tier + turn after it, the same mechanism as the stage router's handoff notes, so the + strong model knows it is taking over a session and re-checks the work rather + than trusting it. - **Raw Responses stream trace** — an opt-in trace of every upstream Responses event as received, under `RUST_LOG=switchyard_translation::responses::raw=trace`, for diagnosing provider-specific event shapes. (#646) diff --git a/crates/libsy/src/algorithms/escalation.rs b/crates/libsy/src/algorithms/escalation.rs index 47523dc13..7f24ee550 100644 --- a/crates/libsy/src/algorithms/escalation.rs +++ b/crates/libsy/src/algorithms/escalation.rs @@ -14,6 +14,7 @@ use super::util::classifier_contract::ClassifierContractConfig; use super::util::decisive; use super::util::escalation::{self, EscalationJudge, EscalationJudgeConfig, EscalationPolicy}; use super::util::llm_judge::JudgeClassifier; +use super::util::prompts; use crate::core::algorithm::Driver; use crate::core::classifier::{Classification, Classifier}; use crate::core::state::{State, StateValue}; @@ -48,6 +49,18 @@ struct EscalationClassifier { efficient: ModelId, /// Consecutive escalate verdicts required to latch. confirmations: u32, + /// Note spliced into every turn the judge has sent to the capable tier. + handoff_note: Option, +} + +impl EscalationClassifier { + /// Hands the capable tier the configured note. Only judge-driven turns call this; a + /// fallback to capable is not a verdict and must not tell the model the other tier failed. + fn apply_handoff_note(&self, request: &mut Request) { + if let Some(note) = &self.handoff_note { + prompts::append_note(request, note); + } + } } /// Builds the escalation classifier used by the shared LLM classifier route shell. @@ -60,6 +73,7 @@ pub(super) fn build_classifier( max_output_tokens: u64, ) -> Result>> { let confirmations = config.confirmations; + let handoff_note = config.handoff_note.clone(); let classifier: Arc> = Arc::new(EscalationClassifier { judge: escalation::build_judge( judge_target, @@ -72,6 +86,7 @@ pub(super) fn build_classifier( capable: capable_target.clone(), efficient: efficient_target.clone(), confirmations, + handoff_note, }); Ok(classifier) } @@ -96,6 +111,7 @@ impl Classifier for EscalationClassifier { "source": "escalation", "verdict": "latched", })); + self.apply_handoff_note(request); return Ok((decisive(&self.capable), None)); } @@ -176,6 +192,7 @@ impl Classifier for EscalationClassifier { "source": "escalation", "verdict": "escalate", })); + self.apply_handoff_note(request); return Ok((decisive(&self.capable), None)); } @@ -276,21 +293,75 @@ mod tests { /// Builds a router with escalation enabled (`confirmations=1` latches immediately). fn escalation_router() -> Result> { + escalation_router_with(EscalationJudgeConfig { + confirmations: 1, + ..EscalationJudgeConfig::default() + }) + } + + fn escalation_router_with(config: EscalationJudgeConfig) -> Result> { Ok(Arc::new(LlmTaskClassifier::new( LlmClassifierConfig::Escalation { judge_target: ModelId::from("judge"), efficient_target: ModelId::from("efficient"), capable_target: ModelId::from("capable"), contract: ClassifierContractConfig::default(), - config: EscalationJudgeConfig { - confirmations: 1, - ..EscalationJudgeConfig::default() - }, + config, max_output_tokens: DEFAULT_JUDGE_MAX_OUTPUT_TOKENS, }, )?)) } + /// The handoff note reaches the capable tier on the latching turn and on every confirmed + /// turn after it, and never reaches the efficient tier or the judge. + #[tokio::test] + async fn handoff_note_reaches_only_the_capable_tier() -> Result<()> { + const NOTE: &str = "You are taking over mid-task; verify before continuing."; + let seen = Arc::new(Mutex::new(Vec::new())); + let serve = { + let seen = Arc::clone(&seen); + move |model: ModelId, request: Request| { + let seen = Arc::clone(&seen); + async move { + let noted = request.llm_request.messages.iter().any(|message| { + message.role == switchyard_protocol::Role::User + && message.content.iter().any(|block| { + matches!(block, ContentBlock::Text { text } if text.contains(NOTE)) + }) + }); + let model = model.to_string(); + seen.lock().push((model.clone(), noted)); + Ok(match model.as_str() { + "judge" => reply(r#"{"escalate":true,"reason":"stuck"}"#), + "efficient" => reply("efficient draft"), + _ => reply("capable answer"), + }) + } + } + }; + let router = escalation_router_with(EscalationJudgeConfig { + confirmations: 1, + handoff_note: Some(NOTE.to_string()), + ..EscalationJudgeConfig::default() + })?; + let request = classify_session_request(); + + test_drive(router.clone(), request.clone(), serve.clone()).await?; + let (selected_model, _) = test_drive(router, request, serve).await?; + + assert_eq!(selected_model, "capable"); + assert_eq!( + &*seen.lock(), + &[ + ("efficient".to_string(), false), + ("judge".to_string(), false), + ("capable".to_string(), true), + ("capable".to_string(), true), + ] + ); + Ok(()) + } + #[tokio::test] async fn serves_efficient_when_judge_declines() -> Result<()> { let judge = Queue::new([r#"{"escalate":false,"reason":"progressing"}"#]); diff --git a/crates/libsy/src/algorithms/util/escalation.rs b/crates/libsy/src/algorithms/util/escalation.rs index fb093f52a..94e92ed63 100644 --- a/crates/libsy/src/algorithms/util/escalation.rs +++ b/crates/libsy/src/algorithms/util/escalation.rs @@ -60,6 +60,12 @@ pub struct EscalationJudgeConfig { pub recent_turn_window: usize, /// Per-message cap inside the trailing window. pub window_message_chars: usize, + /// Note appended to the request on every turn the judge has sent to the capable tier: the + /// latching turn and each confirmed turn after it. Turns that reach the capable tier by + /// fallback (context overflow, transport failure) carry no note, since the judge did not + /// speak. The note rides in the forwarded request only, never in the caller's conversation, + /// so it cannot accumulate across turns. `None` sends nothing. + pub handoff_note: Option, } impl EscalationJudgeConfig { @@ -78,6 +84,13 @@ impl EscalationJudgeConfig { self.window_message_chars )); } + if self + .handoff_note + .as_deref() + .is_some_and(|note| note.trim().is_empty()) + { + return reject("handoff_note must not be blank".to_string()); + } Ok(()) } } @@ -88,6 +101,7 @@ impl Default for EscalationJudgeConfig { confirmations: 2, recent_turn_window: 28, window_message_chars: 500, + handoff_note: None, } } } diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rs index 867a8e53a..4b5f2c30e 100644 --- a/crates/switchyard-runner/src/config.rs +++ b/crates/switchyard-runner/src/config.rs @@ -1035,6 +1035,24 @@ new = ["send_message"] Ok(()) } + /// A configured `handoff_note` loads with the deployment, and a blank note is rejected at + /// load time rather than being sent to the strong tier as empty text. + #[test] + fn an_escalation_handoff_note_parses_and_must_not_be_blank() -> RunnerResult<()> { + let noted = VALID_CONFIG.replace( + "base_threshold = 0.5", + "base_threshold = 0.5\nescalation = { confirmations = 2, handoff_note = \"You are taking over this task mid-session; verify before continuing.\" }", + ); + runner_from_toml(¬ed)?; + + let blank = VALID_CONFIG.replace( + "base_threshold = 0.5", + "base_threshold = 0.5\nescalation = { confirmations = 2, handoff_note = \" \" }", + ); + assert!(error_message(&blank).contains("handoff_note must not be blank")); + Ok(()) + } + #[test] fn classifier_judge_completion_caps_are_configurable() -> RunnerResult<()> { let capability = VALID_CONFIG.replace( diff --git a/docs/routing_algorithms/escalation_router_routing.md b/docs/routing_algorithms/escalation_router_routing.md index 1ed9e8085..4f5929cd4 100644 --- a/docs/routing_algorithms/escalation_router_routing.md +++ b/docs/routing_algorithms/escalation_router_routing.md @@ -98,7 +98,7 @@ compatibility guidance as the LLM classifier judge. See ## Tuning options -The judge exposes three settings. Their defaults are the benchmarked +The judge exposes four settings. Their defaults are the benchmarked configuration, so a bare `escalation = {}` is a valid, tuned route: | Key | Default | Meaning | @@ -106,6 +106,7 @@ configuration, so a bare `escalation = {}` is a valid, tuned route: | `confirmations` | `2` | Consecutive escalate verdicts required before the session latches to strong. Must be at least `1`. | | `recent_turn_window` | `28` | Trailing messages shown to the judge on top of the anchors. Must be at least `1`. | | `window_message_chars` | `500` | Per-message truncation cap inside that trailing window. Must be at least `50`. | +| `handoff_note` | none | Text handed to the strong tier on every turn the judge sent there: the latching turn and each turn after it. Must not be blank when set. | `confirmations` is the main cost dial. `1` latches sooner and spends more on the strong tier. `2` or higher requires a session identity, because the streak is @@ -116,6 +117,20 @@ Anchor and transcript caps remain fixed. Set the route-level `max_output_tokens` key to change the judge's reply budget. Any decline still resets the streak to zero. +`handoff_note` tells the strong model that it is taking over a session another +model started, so it can re-check the task and the work so far instead of +trusting it. The note is appended to the last user message of the forwarded +request (or added as a user message when the turn ends on a tool result), on the +latching turn and on every strong-tier turn after it. It travels in the forwarded +request only, never in the caller's conversation, so it cannot accumulate. Turns +that reach the strong tier without a verdict, such as a context-window overflow +on the weak tier, carry no note. Keep the wording general: it describes the +handoff, not the task. + +```toml +escalation = { confirmations = 2, handoff_note = "You are taking over this task from another model mid-session. Re-read the task, restate its acceptance criteria, and verify the current state against them end to end before continuing." } +``` + ## Run the route After installing the Rust server, as described in