From d4d4dd24c5c0be7ee96bf7545d815a1093461655 Mon Sep 17 00:00:00 2001 From: Lin Jia Date: Wed, 9 Sep 2026 13:57:45 -0700 Subject: [PATCH 1/2] feat(libsy): hand the strong tier a note when the escalation judge latches The escalation route already gives the strong model the full conversation when it takes over, but nothing tells it that a handoff happened. In practice the strong model then continues the weak model's plan and trusts its tests, and the session ends with the same confident near miss the weak model would have produced. The stage router solved the same problem with handoff notes. This adds an optional handoff_note to the escalation block. When set, the note is appended to the forwarded request on the latching turn and on every strong-tier turn after it, through the same append_note helper the stage router uses, so it rides in the forwarded request only and never accumulates in the caller's conversation. Turns that reach the strong tier by fallback rather than by verdict carry no note. A blank note is rejected at load time. Tests cover the note reaching only the capable tier across the latching turn and a confirmed turn, TOML parsing of the new key, and the blank rejection. The escalation routing page documents the key. Signed-off-by: Lin Jia --- CHANGELOG.md | 5 ++ crates/libsy/src/algorithms/escalation.rs | 79 ++++++++++++++++++- .../libsy/src/algorithms/util/escalation.rs | 14 ++++ crates/switchyard-runner/src/config.rs | 16 ++++ .../escalation_router_routing.md | 17 +++- 5 files changed, 126 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 995648431..b98c98f6f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,11 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). on the Responses wire, `reasoning_effort` on Chat Completions), so a strong tier can run at `max` behind a client that sends `high`. `extra_body` only fills absent keys and could not do this. Rejected on Anthropic clients. +- **Escalation handoff note** — an optional `handoff_note` in the escalation + block is handed to the strong tier on the latching turn and every strong-tier + turn after it, the same mechanism as the stage router's handoff notes, so the + strong model knows it is taking over a session and re-checks the work rather + than trusting it. - **Raw Responses stream trace** — an opt-in trace of every upstream Responses event as received, under `RUST_LOG=switchyard_translation::responses::raw=trace`, for diagnosing provider-specific event shapes. (#646) diff --git a/crates/libsy/src/algorithms/escalation.rs b/crates/libsy/src/algorithms/escalation.rs index 47523dc13..7f24ee550 100644 --- a/crates/libsy/src/algorithms/escalation.rs +++ b/crates/libsy/src/algorithms/escalation.rs @@ -14,6 +14,7 @@ use super::util::classifier_contract::ClassifierContractConfig; use super::util::decisive; use super::util::escalation::{self, EscalationJudge, EscalationJudgeConfig, EscalationPolicy}; use super::util::llm_judge::JudgeClassifier; +use super::util::prompts; use crate::core::algorithm::Driver; use crate::core::classifier::{Classification, Classifier}; use crate::core::state::{State, StateValue}; @@ -48,6 +49,18 @@ struct EscalationClassifier { efficient: ModelId, /// Consecutive escalate verdicts required to latch. confirmations: u32, + /// Note spliced into every turn the judge has sent to the capable tier. + handoff_note: Option, +} + +impl EscalationClassifier { + /// Hands the capable tier the configured note. Only judge-driven turns call this; a + /// fallback to capable is not a verdict and must not tell the model the other tier failed. + fn apply_handoff_note(&self, request: &mut Request) { + if let Some(note) = &self.handoff_note { + prompts::append_note(request, note); + } + } } /// Builds the escalation classifier used by the shared LLM classifier route shell. @@ -60,6 +73,7 @@ pub(super) fn build_classifier( max_output_tokens: u64, ) -> Result>> { let confirmations = config.confirmations; + let handoff_note = config.handoff_note.clone(); let classifier: Arc> = Arc::new(EscalationClassifier { judge: escalation::build_judge( judge_target, @@ -72,6 +86,7 @@ pub(super) fn build_classifier( capable: capable_target.clone(), efficient: efficient_target.clone(), confirmations, + handoff_note, }); Ok(classifier) } @@ -96,6 +111,7 @@ impl Classifier for EscalationClassifier { "source": "escalation", "verdict": "latched", })); + self.apply_handoff_note(request); return Ok((decisive(&self.capable), None)); } @@ -176,6 +192,7 @@ impl Classifier for EscalationClassifier { "source": "escalation", "verdict": "escalate", })); + self.apply_handoff_note(request); return Ok((decisive(&self.capable), None)); } @@ -276,21 +293,75 @@ mod tests { /// Builds a router with escalation enabled (`confirmations=1` latches immediately). fn escalation_router() -> Result> { + escalation_router_with(EscalationJudgeConfig { + confirmations: 1, + ..EscalationJudgeConfig::default() + }) + } + + fn escalation_router_with(config: EscalationJudgeConfig) -> Result> { Ok(Arc::new(LlmTaskClassifier::new( LlmClassifierConfig::Escalation { judge_target: ModelId::from("judge"), efficient_target: ModelId::from("efficient"), capable_target: ModelId::from("capable"), contract: ClassifierContractConfig::default(), - config: EscalationJudgeConfig { - confirmations: 1, - ..EscalationJudgeConfig::default() - }, + config, max_output_tokens: DEFAULT_JUDGE_MAX_OUTPUT_TOKENS, }, )?)) } + /// The handoff note reaches the capable tier on the latching turn and on every confirmed + /// turn after it, and never reaches the efficient tier or the judge. + #[tokio::test] + async fn handoff_note_reaches_only_the_capable_tier() -> Result<()> { + const NOTE: &str = "You are taking over mid-task; verify before continuing."; + let seen = Arc::new(Mutex::new(Vec::new())); + let serve = { + let seen = Arc::clone(&seen); + move |model: ModelId, request: Request| { + let seen = Arc::clone(&seen); + async move { + let noted = request.llm_request.messages.iter().any(|message| { + message.role == switchyard_protocol::Role::User + && message.content.iter().any(|block| { + matches!(block, ContentBlock::Text { text } if text.contains(NOTE)) + }) + }); + let model = model.to_string(); + seen.lock().push((model.clone(), noted)); + Ok(match model.as_str() { + "judge" => reply(r#"{"escalate":true,"reason":"stuck"}"#), + "efficient" => reply("efficient draft"), + _ => reply("capable answer"), + }) + } + } + }; + let router = escalation_router_with(EscalationJudgeConfig { + confirmations: 1, + handoff_note: Some(NOTE.to_string()), + ..EscalationJudgeConfig::default() + })?; + let request = classify_session_request(); + + test_drive(router.clone(), request.clone(), serve.clone()).await?; + let (selected_model, _) = test_drive(router, request, serve).await?; + + assert_eq!(selected_model, "capable"); + assert_eq!( + &*seen.lock(), + &[ + ("efficient".to_string(), false), + ("judge".to_string(), false), + ("capable".to_string(), true), + ("capable".to_string(), true), + ] + ); + Ok(()) + } + #[tokio::test] async fn serves_efficient_when_judge_declines() -> Result<()> { let judge = Queue::new([r#"{"escalate":false,"reason":"progressing"}"#]); diff --git a/crates/libsy/src/algorithms/util/escalation.rs b/crates/libsy/src/algorithms/util/escalation.rs index fb093f52a..94e92ed63 100644 --- a/crates/libsy/src/algorithms/util/escalation.rs +++ b/crates/libsy/src/algorithms/util/escalation.rs @@ -60,6 +60,12 @@ pub struct EscalationJudgeConfig { pub recent_turn_window: usize, /// Per-message cap inside the trailing window. pub window_message_chars: usize, + /// Note appended to the request on every turn the judge has sent to the capable tier: the + /// latching turn and each confirmed turn after it. Turns that reach the capable tier by + /// fallback (context overflow, transport failure) carry no note, since the judge did not + /// speak. The note rides in the forwarded request only, never in the caller's conversation, + /// so it cannot accumulate across turns. `None` sends nothing. + pub handoff_note: Option, } impl EscalationJudgeConfig { @@ -78,6 +84,13 @@ impl EscalationJudgeConfig { self.window_message_chars )); } + if self + .handoff_note + .as_deref() + .is_some_and(|note| note.trim().is_empty()) + { + return reject("handoff_note must not be blank".to_string()); + } Ok(()) } } @@ -88,6 +101,7 @@ impl Default for EscalationJudgeConfig { confirmations: 2, recent_turn_window: 28, window_message_chars: 500, + handoff_note: None, } } } diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rs index 867a8e53a..859b982f7 100644 --- a/crates/switchyard-runner/src/config.rs +++ b/crates/switchyard-runner/src/config.rs @@ -1035,6 +1035,22 @@ new = ["send_message"] Ok(()) } + #[test] + fn an_escalation_handoff_note_parses_and_must_not_be_blank() -> RunnerResult<()> { + let noted = VALID_CONFIG.replace( + "base_threshold = 0.5", + "base_threshold = 0.5\nescalation = { confirmations = 2, handoff_note = \"You are taking over this task mid-session; verify before continuing.\" }", + ); + runner_from_toml(¬ed)?; + + let blank = VALID_CONFIG.replace( + "base_threshold = 0.5", + "base_threshold = 0.5\nescalation = { confirmations = 2, handoff_note = \" \" }", + ); + assert!(error_message(&blank).contains("handoff_note must not be blank")); + Ok(()) + } + #[test] fn classifier_judge_completion_caps_are_configurable() -> RunnerResult<()> { let capability = VALID_CONFIG.replace( diff --git a/docs/routing_algorithms/escalation_router_routing.md b/docs/routing_algorithms/escalation_router_routing.md index 1ed9e8085..4f5929cd4 100644 --- a/docs/routing_algorithms/escalation_router_routing.md +++ b/docs/routing_algorithms/escalation_router_routing.md @@ -98,7 +98,7 @@ compatibility guidance as the LLM classifier judge. See ## Tuning options -The judge exposes three settings. Their defaults are the benchmarked +The judge exposes four settings. Their defaults are the benchmarked configuration, so a bare `escalation = {}` is a valid, tuned route: | Key | Default | Meaning | @@ -106,6 +106,7 @@ configuration, so a bare `escalation = {}` is a valid, tuned route: | `confirmations` | `2` | Consecutive escalate verdicts required before the session latches to strong. Must be at least `1`. | | `recent_turn_window` | `28` | Trailing messages shown to the judge on top of the anchors. Must be at least `1`. | | `window_message_chars` | `500` | Per-message truncation cap inside that trailing window. Must be at least `50`. | +| `handoff_note` | none | Text handed to the strong tier on every turn the judge sent there: the latching turn and each turn after it. Must not be blank when set. | `confirmations` is the main cost dial. `1` latches sooner and spends more on the strong tier. `2` or higher requires a session identity, because the streak is @@ -116,6 +117,20 @@ Anchor and transcript caps remain fixed. Set the route-level `max_output_tokens` key to change the judge's reply budget. Any decline still resets the streak to zero. +`handoff_note` tells the strong model that it is taking over a session another +model started, so it can re-check the task and the work so far instead of +trusting it. The note is appended to the last user message of the forwarded +request (or added as a user message when the turn ends on a tool result), on the +latching turn and on every strong-tier turn after it. It travels in the forwarded +request only, never in the caller's conversation, so it cannot accumulate. Turns +that reach the strong tier without a verdict, such as a context-window overflow +on the weak tier, carry no note. Keep the wording general: it describes the +handoff, not the task. + +```toml +escalation = { confirmations = 2, handoff_note = "You are taking over this task from another model mid-session. Re-read the task, restate its acceptance criteria, and verify the current state against them end to end before continuing." } +``` + ## Run the route After installing the Rust server, as described in From f17da5663d16f7ded1613396f7df192ec9765978 Mon Sep 17 00:00:00 2001 From: Lin Jia Date: Thu, 10 Sep 2026 12:48:58 -0700 Subject: [PATCH 2/2] docs(runner): state what the handoff note config test checks Signed-off-by: Lin Jia --- crates/switchyard-runner/src/config.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rs index 859b982f7..4b5f2c30e 100644 --- a/crates/switchyard-runner/src/config.rs +++ b/crates/switchyard-runner/src/config.rs @@ -1035,6 +1035,8 @@ new = ["send_message"] Ok(()) } + /// A configured `handoff_note` loads with the deployment, and a blank note is rejected at + /// load time rather than being sent to the strong tier as empty text. #[test] fn an_escalation_handoff_note_parses_and_must_not_be_blank() -> RunnerResult<()> { let noted = VALID_CONFIG.replace(