From 6a8949db61689c5dc0dab102793734c563377182 Mon Sep 17 00:00:00 2001 From: Arnav Dadarya Date: Sat, 12 Sep 2026 21:50:58 -0700 Subject: [PATCH 1/3] feat(libsy): cap the windowed classifier judge payload Signed-off-by: Arnav Dadarya --- crates/libsy/src/algorithms/llm_class.rs | 368 +++++++++++++++++- crates/libsy/src/algorithms/util.rs | 34 ++ .../libsy/src/algorithms/util/escalation.rs | 42 +- crates/switchyard-py/src/libsy_bindings.rs | 6 + crates/switchyard-runner/src/algorithm.rs | 21 + docs/reference/toml_schema.md | 2 + .../llm_classifier_routing.md | 1 + switchyard_rust/libsy.py | 2 + 8 files changed, 447 insertions(+), 29 deletions(-) diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs index 4bb5e7ff0..c7428913c 100644 --- a/crates/libsy/src/algorithms/llm_class.rs +++ b/crates/libsy/src/algorithms/llm_class.rs @@ -13,7 +13,6 @@ use switchyard_protocol::{Category, ContentBlock, Message, Role}; use super::escalation; use super::fall_through::FallThrough; -use super::util::DEFAULT_JUDGE_MAX_OUTPUT_TOKENS; use super::util::affinity::{AffinityRouter, ClassifyTrigger}; use super::util::classifier_contract::{ ClassifierContract, ClassifierContractConfig, ClassifierResponseFormat, @@ -24,6 +23,7 @@ use super::util::llm_judge::{ SerdeDecoder, StructuredJudge, }; use super::util::target_selector::TargetSelectorPolicy; +use super::util::{DEFAULT_JUDGE_CHAR_BUDGET, DEFAULT_JUDGE_MAX_OUTPUT_TOKENS, truncate_middle}; use crate::core::algorithm::{Algorithm, Driver}; use crate::core::classifier::{Classification, Classifier, Score}; use crate::core::state::State; @@ -160,9 +160,104 @@ fn task_messages(messages: &[Message]) -> Vec { } } +/// Characters one message costs a judge. +/// +/// Counts tool traffic as well as visible text: in a coding-agent conversation a single +/// tool result is routinely larger than every text block around it, so measuring text +/// alone would report a payload as small while the judge is billed for all of it. +fn message_chars(message: &Message) -> usize { + message.content.iter().map(block_chars).sum() +} + +/// Characters one content block costs a judge. +fn block_chars(block: &ContentBlock) -> usize { + match block { + ContentBlock::Text { text } | ContentBlock::Refusal { text } => text.chars().count(), + ContentBlock::ToolCall(call) => { + call.name.chars().count() + call.arguments.to_string().chars().count() + } + ContentBlock::ToolResult(result) => result.content.iter().map(block_chars).sum(), + // Reasoning is stripped before the judge is called, so it costs nothing. Media and + // unknown blocks are opaque here; their wire cost is not text. + _ => 0, + } +} + +/// Whether clipping can shrink this block. +/// +/// Tool arguments and results are JSON the judge may need to read as a unit, so they cannot +/// be cut down to fit. +fn is_clippable(block: &ContentBlock) -> bool { + matches!( + block, + ContentBlock::Text { .. } | ContentBlock::Refusal { .. } + ) +} + +/// Total judge payload size for a selected message list. +fn payload_chars(messages: &[Message]) -> usize { + messages.iter().map(message_chars).sum() +} + +/// Selects the trailing window, narrowing it until the payload fits `budget` characters. +/// +/// A window is counted in turns, and turn size varies by orders of magnitude: four turns is +/// a few hundred characters of conversation, or tens of thousands when one turn carries a +/// large tool result. Judge cost and latency would otherwise be decided by the request +/// rather than by configuration, and a single large result can crowd out the task being +/// judged. +/// +/// Whole turns are dropped from the oldest end rather than clipping individual messages, +/// because [`trim_messages`] is what keeps tool calls paired with their results; removing +/// messages by hand would hand the judge a result whose call was never introduced. +fn window_within_budget(messages: &[Message], window: usize, budget: usize) -> Vec { + let mut window = window; + let mut kept = trim_messages(messages, window); + while window > 0 && payload_chars(&kept) > budget { + window -= 1; + kept = trim_messages(messages, window); + } + // The anchors — client instructions and the opening task — survive an empty window, so + // a task statement larger than the whole budget still has to be clipped. + if payload_chars(&kept) > budget { + clip_to_budget(&mut kept, budget); + } + kept +} + +/// Clips text blocks so an unwindowable payload still fits `budget`. +/// +/// The share is per block rather than per message: one message can carry several text +/// blocks, and a per-message allowance would let each of them spend it in full. +/// +/// Blocks that cannot be clipped are charged first, so what remains is what the text is +/// allowed to spend. When they alone exceed the budget every text block collapses to +/// nothing and the payload still overruns; cutting tool JSON to fit would hand the judge +/// malformed values, which is worse than an oversized prompt. +fn clip_to_budget(messages: &mut [Message], budget: usize) { + let blocks = || messages.iter().flat_map(|message| &message.content); + let clippable = blocks().filter(|block| is_clippable(block)).count(); + if clippable == 0 { + return; + } + let fixed: usize = blocks() + .filter(|block| !is_clippable(block)) + .map(block_chars) + .sum(); + let per_block = budget.saturating_sub(fixed) / clippable; + for block in messages.iter_mut().flat_map(|message| &mut message.content) { + if let ContentBlock::Text { text } | ContentBlock::Refusal { text } = block { + *text = truncate_middle(text, per_block); + } + } +} + /// Selects the task messages shown to capability and custom-schema classifiers. struct TaskInput { recent_turn_window: Option, + /// Character budget for the windowed payload. Unused without a window, where the + /// selection is the opening task and latest follow-up rather than conversation. + judge_char_budget: usize, } impl ClassifierInput for TaskInput { @@ -170,7 +265,14 @@ impl ClassifierInput for TaskInput { // The default preserves the whole-task anchor and latest user update. A // configured window widens that to the surrounding conversation. let mut messages = match self.recent_turn_window { - Some(window) => trim_messages(&request.llm_request.messages, window), + // The routing instruction appended below is part of what the judge is sent, so + // its cost comes out of the budget rather than on top of it. + Some(window) => window_within_budget( + &request.llm_request.messages, + window, + self.judge_char_budget + .saturating_sub(TRAILING_ROUTING_INSTRUCTION.chars().count()), + ), None => task_messages(&request.llm_request.messages), }; // Reasoning is provider-private and not required to classify the task. Some @@ -297,6 +399,11 @@ pub struct TaskClassifierConfig { /// `Some(n)` widens that to the client instructions, the opening task, and /// the last `n` turns after it. pub recent_turn_window: Option, + /// Character budget for a windowed judge payload. + /// + /// Bounds what one request can spend on a judge call when `recent_turn_window` is set: + /// the window narrows until the selection fits. Ignored without a window. + pub judge_char_budget: usize, /// Prompt and verdict contract settings for the classifier judge. pub contract: ClassifierContractConfig, /// Maximum completion tokens available to the classifier verdict. @@ -316,6 +423,8 @@ struct TaskClassifierConfigWire { message_hash_fallback: bool, #[serde(default)] recent_turn_window: Option, + #[serde(default = "default_judge_char_budget")] + judge_char_budget: usize, #[serde(default)] prompt: Option, #[serde(default)] @@ -341,6 +450,7 @@ impl<'de> Deserialize<'de> for TaskClassifierConfig { classify_trigger: wire.classify_trigger, message_hash_fallback: wire.message_hash_fallback, recent_turn_window: wire.recent_turn_window, + judge_char_budget: wire.judge_char_budget, contract, max_output_tokens: wire.max_output_tokens, }) @@ -351,6 +461,21 @@ const fn default_judge_max_output_tokens() -> u64 { DEFAULT_JUDGE_MAX_OUTPUT_TOKENS } +const fn default_judge_char_budget() -> usize { + DEFAULT_JUDGE_CHAR_BUDGET +} + +/// A zero budget would clip every message to the trim marker, leaving the judge a payload +/// it cannot route. Rejecting it at construction beats serving empty verdicts. +fn validate_judge_char_budget(budget: usize) -> Result<()> { + if budget == 0 { + return Err(LibsyError::AlgorithmError { + message: "judge_char_budget must be at least 1".to_string(), + }); + } + Ok(()) +} + impl Default for TaskClassifierConfig { fn default() -> Self { Self { @@ -359,6 +484,7 @@ impl Default for TaskClassifierConfig { classify_trigger: ClassifyTrigger::default(), message_hash_fallback: false, recent_turn_window: None, + judge_char_budget: DEFAULT_JUDGE_CHAR_BUDGET, contract: ClassifierContractConfig::default(), max_output_tokens: DEFAULT_JUDGE_MAX_OUTPUT_TOKENS, } @@ -368,6 +494,7 @@ impl Default for TaskClassifierConfig { impl TaskClassifierConfig { /// Validates routing thresholds before the classifier is constructed. fn validate(&self) -> Result<()> { + validate_judge_char_budget(self.judge_char_budget)?; if !(0.0..=1.0).contains(&self.base_threshold) { return Err(LibsyError::AlgorithmError { message: format!( @@ -441,6 +568,8 @@ pub struct CustomClassifierConfig { pub message_hash_fallback: bool, /// Trailing conversation turns shown to the classifier judge. pub recent_turn_window: Option, + /// Character budget for a windowed judge payload. Ignored without a window. + pub judge_char_budget: usize, /// Maximum completion tokens available to the classifier verdict. pub max_output_tokens: u64, } @@ -459,11 +588,13 @@ impl CustomClassifierConfig { classify_trigger: ClassifyTrigger::default(), message_hash_fallback: false, recent_turn_window: None, + judge_char_budget: DEFAULT_JUDGE_CHAR_BUDGET, max_output_tokens: DEFAULT_JUDGE_MAX_OUTPUT_TOKENS, } } fn validate(&self) -> Result<()> { + validate_judge_char_budget(self.judge_char_budget)?; if self.max_output_tokens == 0 { return Err(LibsyError::AlgorithmError { message: "max_output_tokens must be at least 1".to_string(), @@ -612,6 +743,7 @@ impl LlmTaskClassifier { StructuredJudge::new( TaskInput { recent_turn_window: config.recent_turn_window, + judge_char_budget: config.judge_char_budget, }, contract, SerdeDecoder::new(), @@ -640,6 +772,7 @@ impl LlmTaskClassifier { classify_trigger, message_hash_fallback, recent_turn_window, + judge_char_budget, max_output_tokens, } = config; let contract = ClassifierContract::from_inner_schema(&prompt, response_schema)?; @@ -650,7 +783,10 @@ impl LlmTaskClassifier { }; let classifier: Arc> = Arc::new(JudgeClassifier::new( StructuredJudge::new( - TaskInput { recent_turn_window }, + TaskInput { + recent_turn_window, + judge_char_budget, + }, contract, JsonSchemaDecoder::new(), JudgeRuntimeConfig::new(max_output_tokens)?, @@ -747,7 +883,7 @@ impl Algorithm for LlmTaskClassifier { #[cfg(test)] mod tests { - use std::collections::HashMap; + use std::collections::{BTreeSet, HashMap}; use std::sync::Arc; use parking_lot::Mutex; @@ -1316,8 +1452,19 @@ mod tests { /// The text of each message a judge with `recent_turn_window` would be sent. /// The no-window case is covered by `capability_judge_builds_a_structured_request`. fn capability_judge(recent_turn_window: Option) -> Result { + capability_judge_with_budget(recent_turn_window, DEFAULT_JUDGE_CHAR_BUDGET) + } + + /// A judge whose windowed payload is capped at `judge_char_budget` characters. + fn capability_judge_with_budget( + recent_turn_window: Option, + judge_char_budget: usize, + ) -> Result { Ok(StructuredJudge::new( - TaskInput { recent_turn_window }, + TaskInput { + recent_turn_window, + judge_char_budget, + }, LlmTaskClassifier::load_capability_contract(&ClassifierContractConfig::default())?, SerdeDecoder::new(), JudgeRuntimeConfig::new(DEFAULT_JUDGE_MAX_OUTPUT_TOKENS)?, @@ -1371,6 +1518,215 @@ mod tests { Ok(()) } + /// Builds a request whose windowed selection the judge will be handed. + fn windowed_request(messages: Vec) -> Request { + Request { + llm_request: LlmRequest { + messages, + ..LlmRequest::default() + }, + raw_request: None, + metadata: None, + } + } + + /// The messages a judge with this window and budget is actually sent. + fn budgeted_messages( + messages: Vec, + window: usize, + budget: usize, + ) -> Result> { + let judge = capability_judge_with_budget(Some(window), budget)?; + Ok(judge + .build_request(&State::default(), &windowed_request(messages)) + .llm_request + .messages) + } + + /// A window is counted in turns, so one turn carrying a large tool result would + /// otherwise decide the judge's cost for a fixed configuration. The window narrows + /// from the oldest end until the payload fits, so the newest evidence survives. + #[test] + fn an_oversized_turn_narrows_the_window_to_fit_the_budget() -> Result<()> { + let messages = vec![ + Message::text(Role::System, "client instructions"), + Message::text(Role::User, "initial task"), + Message::text(Role::Assistant, "x".repeat(20_000)), + Message::text(Role::User, "recent 1"), + Message::text(Role::Assistant, "recent 2"), + ]; + let built = budgeted_messages(messages, 4, 5_000)?; + let texts: Vec = built + .iter() + .filter_map(|message| message.text_content("\n")) + .collect(); + + assert!(payload_chars(&built) <= 5_000, "{}", payload_chars(&built)); + // The anchors and the newest turns stay; only the oversized older turn goes. + assert!(texts.contains(&"client instructions".to_string())); + assert!(texts.contains(&"initial task".to_string())); + assert!(texts.contains(&"recent 2".to_string())); + assert!(!texts.iter().any(|text| text.len() > 10_000), "{texts:?}"); + Ok(()) + } + + /// Reasoning is stripped before the judge is called, so a large reasoning block must not + /// narrow the window and cost the judge visible turns it could have kept. + #[test] + fn reasoning_does_not_count_against_the_budget() -> Result<()> { + let messages = vec![ + Message::text(Role::User, "initial task"), + Message { + role: Role::Assistant, + content: vec![ + ContentBlock::Reasoning { + text: "r".repeat(20_000), + signature: None, + details: Vec::new(), + }, + ContentBlock::Text { + text: "older answer".to_string(), + }, + ], + }, + Message::text(Role::User, "recent 1"), + ]; + let built = budgeted_messages(messages, 4, 5_000)?; + let texts: Vec = built + .iter() + .filter_map(|message| message.text_content("\n")) + .collect(); + + assert!(texts.contains(&"older answer".to_string()), "{texts:?}"); + Ok(()) + } + + /// Narrowing drops whole turns through `trim_messages`, so a surviving tool result + /// still has the call that introduced its id. Removing messages directly would not. + #[test] + fn narrowing_for_the_budget_keeps_tool_pairs_whole() -> Result<()> { + let mut bulky = tool_result("call-1"); + bulky.content = vec![ContentBlock::ToolResult(ToolResult { + tool_call_id: "call-1".to_string(), + content: vec![ContentBlock::Text { + text: "y".repeat(20_000), + }], + is_error: None, + })]; + let messages = vec![ + Message::text(Role::System, "client instructions"), + Message::text(Role::User, "initial task"), + tool_call("call-1"), + bulky, + tool_call("call-2"), + tool_result("call-2"), + ]; + let built = budgeted_messages(messages, 4, 5_000)?; + + assert!(payload_chars(&built) <= 5_000, "{}", payload_chars(&built)); + let calls: BTreeSet = built + .iter() + .flat_map(|message| &message.content) + .filter_map(|block| match block { + ContentBlock::ToolCall(call) => Some(call.id.clone()), + _ => None, + }) + .collect(); + for block in built.iter().flat_map(|message| &message.content) { + if let ContentBlock::ToolResult(result) = block { + assert!( + calls.contains(&result.tool_call_id), + "orphaned result {:?} in {calls:?}", + result.tool_call_id + ); + } + } + Ok(()) + } + + /// The anchors survive an empty window, so a task statement larger than the whole + /// budget cannot be dropped and has to be clipped instead. + #[test] + fn an_oversized_task_is_clipped_once_the_window_cannot_shrink() -> Result<()> { + let messages = vec![ + Message::text(Role::System, "client instructions"), + Message::text(Role::User, "z".repeat(40_000)), + Message::text(Role::Assistant, "recent"), + ]; + let built = budgeted_messages(messages, 2, 1_000)?; + let task = built + .iter() + .filter_map(|message| message.text_content("\n")) + .find(|text| text.starts_with('z')) + .ok_or_else(|| LibsyError::AlgorithmError { + message: "clipped task missing".to_string(), + })?; + + assert!(payload_chars(&built) <= 1_000, "{}", payload_chars(&built)); + // Clipping is marked, so the judge can tell a trimmed task from a short one. + assert!(task.contains("[trimmed]"), "{task}"); + Ok(()) + } + + /// The routing instruction is appended after selection, so the budget has to cover it: + /// a selection sized to the limit would otherwise ship a payload above the limit. + #[test] + fn the_budget_covers_the_appended_routing_instruction() -> Result<()> { + let messages = vec![ + Message::text(Role::System, "client instructions"), + Message::text(Role::User, "w".repeat(4_000)), + Message::text(Role::Assistant, "recent"), + ]; + let built = budgeted_messages(messages, 2, 600)?; + + // `built` includes the trailing instruction, so this measures the whole payload. + assert!( + built + .iter() + .any(|message| message.text_content("\n").as_deref() + == Some(TRAILING_ROUTING_INSTRUCTION)) + ); + assert!(payload_chars(&built) <= 600, "{}", payload_chars(&built)); + Ok(()) + } + + /// One message can carry several text blocks. A per-message allowance would let each + /// block spend it in full, so the share is per block. + #[test] + fn several_text_blocks_in_one_message_share_the_budget() -> Result<()> { + let crowded = Message { + role: Role::User, + content: vec![ + ContentBlock::Text { + text: "a".repeat(9_000), + }, + ContentBlock::Text { + text: "b".repeat(9_000), + }, + ], + }; + let messages = vec![ + Message::text(Role::System, "client instructions"), + crowded, + Message::text(Role::Assistant, "recent"), + ]; + let built = budgeted_messages(messages, 2, 900)?; + + assert!(payload_chars(&built) <= 900, "{}", payload_chars(&built)); + Ok(()) + } + + /// A zero budget would clip every message to the trim marker, so it is rejected at + /// construction rather than serving the judge an unroutable payload. + #[test] + fn a_zero_judge_char_budget_is_rejected() { + let config = TaskClassifierConfig { + judge_char_budget: 0, + ..TaskClassifierConfig::default() + }; + assert!(config.validate().is_err()); + } + fn tool_call(id: &str) -> Message { Message { role: Role::Assistant, @@ -1546,6 +1902,7 @@ mod tests { let built = TaskInput { recent_turn_window: Some(10), + judge_char_budget: DEFAULT_JUDGE_CHAR_BUDGET, } .build_messages(&State::default(), &request); @@ -1700,6 +2057,7 @@ mod tests { let judge: CapabilityJudge = StructuredJudge::new( TaskInput { recent_turn_window: None, + judge_char_budget: DEFAULT_JUDGE_CHAR_BUDGET, }, contract, SerdeDecoder::new(), diff --git a/crates/libsy/src/algorithms/util.rs b/crates/libsy/src/algorithms/util.rs index 9a290920f..4bf7479b1 100644 --- a/crates/libsy/src/algorithms/util.rs +++ b/crates/libsy/src/algorithms/util.rs @@ -29,3 +29,37 @@ pub(crate) fn decisive(target: &ModelId) -> Classification { /// Default completion budget for internal classifier and escalation judge calls. pub(crate) const DEFAULT_JUDGE_MAX_OUTPUT_TOKENS: u64 = 4_096; + +/// Default character budget for a judge payload, shared by the escalation judge and the +/// windowed classifier judges so one turn carrying a large tool result cannot decide how +/// much a judge call costs. +pub(crate) const DEFAULT_JUDGE_CHAR_BUDGET: usize = 18_000; + +/// Separator marking where [`truncate_middle`] dropped a message's interior. +pub(crate) const TRIM_MARKER: &str = " ...[trimmed] "; + +/// Keeps the head and tail of `text` within `limit` characters. +/// +/// The head gets two thirds of the surviving budget: for a judge reading agent activity the +/// command or error signature that opens a message carries more signal than its trailing +/// output. Clipping is marked so the judge can tell a trimmed message from a short one. +pub(crate) fn truncate_middle(text: &str, limit: usize) -> String { + let chars: Vec = text.chars().collect(); + if chars.len() <= limit { + return text.to_string(); + } + // Below the marker's own width there is no room to say the text was clipped, so keep + // what fits and drop the marker. Marking anyway would push the result past `limit`, + // which callers budgeting a payload rely on it never doing. + let marker = TRIM_MARKER.chars().count(); + if limit <= marker { + return chars[..limit].iter().collect(); + } + let keep = limit - marker; + let head = keep * 2 / 3; + let tail = keep - head; + let mut out: String = chars[..head].iter().collect(); + out.push_str(TRIM_MARKER); + out.extend(chars[chars.len() - tail..].iter()); + out +} diff --git a/crates/libsy/src/algorithms/util/escalation.rs b/crates/libsy/src/algorithms/util/escalation.rs index b0979e286..7fe25900b 100644 --- a/crates/libsy/src/algorithms/util/escalation.rs +++ b/crates/libsy/src/algorithms/util/escalation.rs @@ -16,6 +16,7 @@ use super::llm_judge::{ ClassifierInput, JudgeClassifier, JudgePolicy, JudgeRuntimeConfig, SerdeDecoder, StructuredJudge, }; +use super::truncate_middle; use crate::core::algorithm::Driver; use crate::core::classifier::{Classification, Score}; use crate::core::state::State; @@ -25,9 +26,6 @@ use switchyard_protocol::Request; const PROMPT_TEMPLATE: &str = include_str!("../../prompts/escalation/prompt.md"); const SCHEMA_TEMPLATE: &str = include_str!("../../prompts/escalation/schema.json"); -/// Separator marking where [`truncate_middle`] dropped a message's interior. -const TRIM_MARKER: &str = " ...[trimmed] "; - /// Suffix marking a transcript cut off by [`MAX_REQUEST_CHARS`]. const TRUNCATION_SUFFIX: &str = "..."; @@ -239,27 +237,6 @@ fn collect_text(content: &[ContentBlock], parts: &mut Vec) { } } -/// Keeps the head and tail of `text` within `limit` characters. -/// -/// The head gets two thirds of the surviving budget: for a trajectory judge the command or -/// error signature that opens a message carries more signal than its trailing output. -fn truncate_middle(text: &str, limit: usize) -> String { - let chars: Vec = text.chars().collect(); - if chars.len() <= limit { - return text.to_string(); - } - let keep = limit - .saturating_sub(TRIM_MARKER.chars().count()) - .max(20) - .min(chars.len()); - let head = keep * 2 / 3; - let tail = keep - head; - let mut out: String = chars[..head].iter().collect(); - out.push_str(TRIM_MARKER); - out.extend(chars[chars.len() - tail..].iter()); - out -} - /// Renders a compact role-labelled transcript for the judge. /// /// The framing anchors — system/developer messages and the first user message, where agent @@ -496,6 +473,23 @@ mod tests { assert_eq!(truncate_middle("short", 50), "short"); } + /// Callers budget a payload on the promise that clipping never exceeds the limit. Below + /// the marker's own width there is no room to mark the cut, so the marker is dropped + /// rather than pushing the result over. + #[test] + fn truncate_middle_never_exceeds_a_limit_narrower_than_the_marker() { + use crate::algorithms::util::TRIM_MARKER; + + let text = "a".repeat(100); + for limit in 0..=TRIM_MARKER.chars().count() + 2 { + let trimmed = truncate_middle(&text, limit); + assert!( + trimmed.chars().count() <= limit, + "limit {limit}: {trimmed:?}" + ); + } + } + #[test] fn summary_keeps_anchors_and_the_recent_window() { let mut messages = vec![ diff --git a/crates/switchyard-py/src/libsy_bindings.rs b/crates/switchyard-py/src/libsy_bindings.rs index eea067f41..b8ac06846 100644 --- a/crates/switchyard-py/src/libsy_bindings.rs +++ b/crates/switchyard-py/src/libsy_bindings.rs @@ -168,6 +168,7 @@ impl PyCustomClassifierConfig { session_affinity=false, message_hash_fallback=false, recent_turn_window=None, + judge_char_budget=18_000, max_output_tokens=4096 ))] #[allow(clippy::too_many_arguments)] @@ -178,6 +179,7 @@ impl PyCustomClassifierConfig { session_affinity: bool, message_hash_fallback: bool, recent_turn_window: Option, + judge_char_budget: usize, max_output_tokens: u64, ) -> PyResult { // Convert the Python schema into serde JSON and pair it with the target-selector policy; @@ -190,6 +192,7 @@ impl PyCustomClassifierConfig { inner.classify_trigger = classify_trigger(session_affinity); inner.message_hash_fallback = message_hash_fallback; inner.recent_turn_window = recent_turn_window; + inner.judge_char_budget = judge_char_budget; inner.max_output_tokens = max_output_tokens; Ok(Self { inner }) } @@ -261,6 +264,7 @@ impl PyTaskClassifierConfig { session_affinity=false, message_hash_fallback=false, recent_turn_window=None, + judge_char_budget=18_000, max_output_tokens=4096, prompt=None, response_format_type="json_schema" @@ -272,6 +276,7 @@ impl PyTaskClassifierConfig { session_affinity: bool, message_hash_fallback: bool, recent_turn_window: Option, + judge_char_budget: usize, max_output_tokens: u64, prompt: Option, response_format_type: &str, @@ -283,6 +288,7 @@ impl PyTaskClassifierConfig { classify_trigger: classify_trigger(session_affinity), message_hash_fallback, recent_turn_window, + judge_char_budget, contract: classifier_contract(prompt, response_format_type)?, max_output_tokens, }, diff --git a/crates/switchyard-runner/src/algorithm.rs b/crates/switchyard-runner/src/algorithm.rs index 626c150af..a698ea2a8 100644 --- a/crates/switchyard-runner/src/algorithm.rs +++ b/crates/switchyard-runner/src/algorithm.rs @@ -106,6 +106,7 @@ struct CapabilityClassifierRouteConfig { classify_trigger: ClassifyTrigger, message_hash_fallback: bool, recent_turn_window: Option, + judge_char_budget: usize, prompt: Option, response_format_type: ClassifierResponseFormat, max_output_tokens: u64, @@ -132,6 +133,7 @@ struct CustomClassifierRouteConfig { classify_trigger: ClassifyTrigger, message_hash_fallback: bool, recent_turn_window: Option, + judge_char_budget: usize, max_output_tokens: u64, } @@ -245,6 +247,11 @@ pub struct LlmClassifierRouteConfig { /// How many trailing turns the judge sees. Unset shows it the opening task /// and the latest user follow-up only. pub recent_turn_window: Option, + /// Most characters a windowed judge payload may use. The window narrows from the + /// oldest turn until it fits, so one large tool result cannot decide the judge's + /// cost. Ignored without `recent_turn_window`. + #[serde(default = "default_judge_char_budget")] + pub judge_char_budget: usize, /// Replaces the packaged judge prompt. Required in custom mode. pub prompt: Option, /// How the judge is asked for structured output. Use `json_object` when the @@ -464,6 +471,9 @@ pub struct StageClassifierConfig { /// and the latest user follow-up only. #[serde(default)] pub recent_turn_window: Option, + /// Most characters a windowed judge payload may use. Ignored without a window. + #[serde(default = "default_judge_char_budget")] + pub judge_char_budget: usize, /// Replaces the packaged judge prompt. #[serde(default)] pub prompt: Option, @@ -508,6 +518,7 @@ impl StageClassifierConfig { classify_trigger: self.classify_trigger, message_hash_fallback: self.message_hash_fallback, recent_turn_window: self.recent_turn_window, + judge_char_budget: self.judge_char_budget, contract: classifier_contract(self.prompt.as_deref()) .with_response_format_type(self.response_format_type), max_output_tokens: self.max_output_tokens, @@ -844,6 +855,7 @@ impl LlmClassifierRouteConfig { classify_trigger, message_hash_fallback, recent_turn_window, + judge_char_budget, prompt, response_format_type, max_output_tokens, @@ -899,6 +911,7 @@ impl LlmClassifierRouteConfig { classify_trigger: *classify_trigger, message_hash_fallback: *message_hash_fallback, recent_turn_window: *recent_turn_window, + judge_char_budget: *judge_char_budget, prompt: prompt.clone(), response_format_type: *response_format_type, max_output_tokens: *max_output_tokens, @@ -994,6 +1007,7 @@ impl LlmClassifierRouteConfig { classify_trigger: *classify_trigger, message_hash_fallback: *message_hash_fallback, recent_turn_window: *recent_turn_window, + judge_char_budget: *judge_char_budget, max_output_tokens: *max_output_tokens, }, )) @@ -1079,6 +1093,7 @@ fn build_subagent_router_config( config.policy.into_libsy(), ); classifier_config.recent_turn_window = config.recent_turn_window; + classifier_config.judge_char_budget = config.judge_char_budget; classifier_config.max_output_tokens = config.max_output_tokens; let classifier = Arc::new( LlmTaskClassifier::new(LlmClassifierConfig::Custom { @@ -1176,6 +1191,7 @@ fn build_algorithm( classify_trigger: config.classify_trigger, message_hash_fallback: config.message_hash_fallback, recent_turn_window: config.recent_turn_window, + judge_char_budget: config.judge_char_budget, contract: classifier_contract(config.prompt.as_deref()) .with_response_format_type(config.response_format_type), max_output_tokens: config.max_output_tokens, @@ -1211,6 +1227,7 @@ fn build_algorithm( classifier_config.classify_trigger = config.classify_trigger; classifier_config.message_hash_fallback = config.message_hash_fallback; classifier_config.recent_turn_window = config.recent_turn_window; + classifier_config.judge_char_budget = config.judge_char_budget; classifier_config.max_output_tokens = config.max_output_tokens; LlmTaskClassifier::new(LlmClassifierConfig::Custom { default_target: config.default_target, @@ -1417,6 +1434,10 @@ fn default_classifier_max_output_tokens() -> u64 { TaskClassifierConfig::default().max_output_tokens } +fn default_judge_char_budget() -> usize { + TaskClassifierConfig::default().judge_char_budget +} + fn resolve_target_model_id( route_name: &str, name: &str, diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md index 7d4b8b914..ce1975465 100644 --- a/docs/reference/toml_schema.md +++ b/docs/reference/toml_schema.md @@ -202,6 +202,7 @@ Capability mode classifies before serving. See | `classify_trigger` | No | `every_request` | When the judge runs. `every_request` judges every request, tool continuations included. `user_turn` judges each new user message and retains that target across intervening tool calls only when requests carry a session ID; without a session ID, it behaves like `every_request`. `new_session` judges once and reuses that target for the session. | | `message_hash_fallback` | No | `false` | Keys affinity on the first user message. Requires `classify_trigger = "new_session"`. | | `recent_turn_window` | No | unset | When unset, the judge sees the opening task and latest user follow-up, when present. When set, it also sees trailing turns. | +| `judge_char_budget` | No | `18000` | Most characters a windowed judge payload may use. The window narrows from the oldest turn until it fits, so one large tool result cannot decide judge cost and latency. Ignored without `recent_turn_window`. | | `prompt` | No | packaged prompt | Replaces the capability prompt. The packaged schema is sent separately as structured-output configuration. | Escalation mode serves the weak target first and judges the completed turn. See @@ -242,6 +243,7 @@ how one route chooses between more than two models. | `classify_trigger` | No | `every_request` | When the judge runs. `every_request` judges every request, tool continuations included. `user_turn` judges each new user message and retains that target across intervening tool calls only when requests carry a session ID; without a session ID, it behaves like `every_request`. `new_session` judges once and reuses that target for the session. | | `message_hash_fallback` | No | `false` | Keys affinity on the first user message. Requires `classify_trigger = "new_session"`. | | `recent_turn_window` | No | unset | When unset, the judge sees the opening task and latest user follow-up, when present. When set, it also sees trailing turns. | +| `judge_char_budget` | No | `18000` | Most characters a windowed judge payload may use. The window narrows from the oldest turn until it fits, so one large tool result cannot decide judge cost and latency. Ignored without `recent_turn_window`. | The selected JSON label must name a configured group. A label naming a target rather than a group, or a group you did not configure, falls back to diff --git a/docs/routing_algorithms/llm_classifier_routing.md b/docs/routing_algorithms/llm_classifier_routing.md index cbc893563..40f5ee8ed 100644 --- a/docs/routing_algorithms/llm_classifier_routing.md +++ b/docs/routing_algorithms/llm_classifier_routing.md @@ -106,6 +106,7 @@ for the server merge behavior. | `base_threshold` | required | Lowest `p_solve` that routes a supported task to `weak_target`. Must be between `0` and `1`. | | `threshold_step` | `0.0` | Amount added for each boundary step. Must be finite and non-negative, and `base_threshold + 2 * threshold_step` must not exceed `1`. | | `recent_turn_window` | unset | When unset, the judge sees the opening user task and the latest user message when they differ. When set to `N`, it sees the opening user task and the last `N` conversation messages after that task. `0` keeps only the opening task. Client system and developer instructions are not shown to the judge. | +| `judge_char_budget` | `18000` | Most characters the windowed selection may use. A window is counted in turns, and one turn carrying a large tool result can be worth tens of thousands of characters, so the window narrows from the oldest turn until the payload fits. A task statement larger than the whole budget is clipped instead, marked with `...[trimmed]`. Ignored when `recent_turn_window` is unset. | | `classify_trigger` | `every_request` | When the judge runs. `every_request` judges every request, tool continuations included. `user_turn` judges each new user message and holds that target across the tool calls between. `new_session` judges once and reuses that target for the session. | | `message_hash_fallback` | `false` | When session metadata is absent, keys affinity from the first user-message text. Requires `classify_trigger = "new_session"`. | | `prompt` | packaged capability prompt | Replaces the classifier's system prompt. The packaged verdict schema and routing policy remain active. | diff --git a/switchyard_rust/libsy.py b/switchyard_rust/libsy.py index fd768ba71..2a31efafb 100644 --- a/switchyard_rust/libsy.py +++ b/switchyard_rust/libsy.py @@ -75,6 +75,7 @@ def __init__( session_affinity: bool = False, message_hash_fallback: bool = False, recent_turn_window: int | None = None, + judge_char_budget: int = 18_000, max_output_tokens: int = 4096, ) -> None: ... @@ -166,6 +167,7 @@ def __init__( session_affinity: bool = False, message_hash_fallback: bool = False, recent_turn_window: int | None = None, + judge_char_budget: int = 18_000, max_output_tokens: int = 4096, prompt: str | None = None, response_format_type: Literal["json_schema", "json_object"] = "json_schema", From cc8fbe9dd03e6d0ae844c39c484c182ea37b32a0 Mon Sep 17 00:00:00 2001 From: Arnav Dadarya Date: Sat, 12 Sep 2026 22:01:51 -0700 Subject: [PATCH 2/3] fix(libsy): require a judge_char_budget that fits the routing instruction Signed-off-by: Arnav Dadarya --- crates/libsy/src/algorithms/llm_class.rs | 24 +++++++++++-------- docs/reference/toml_schema.md | 4 ++-- .../llm_classifier_routing.md | 2 +- switchyard_rust/libsy.py | 5 +++- 4 files changed, 21 insertions(+), 14 deletions(-) diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs index c7428913c..e8723f86c 100644 --- a/crates/libsy/src/algorithms/llm_class.rs +++ b/crates/libsy/src/algorithms/llm_class.rs @@ -465,12 +465,14 @@ const fn default_judge_char_budget() -> usize { DEFAULT_JUDGE_CHAR_BUDGET } -/// A zero budget would clip every message to the trim marker, leaving the judge a payload -/// it cannot route. Rejecting it at construction beats serving empty verdicts. +/// Smallest accepted judge payload budget. It must leave room for the routing instruction +/// appended to every windowed payload, or that instruction alone would exceed the budget. +const MIN_JUDGE_CHAR_BUDGET: usize = 256; + fn validate_judge_char_budget(budget: usize) -> Result<()> { - if budget == 0 { + if budget < MIN_JUDGE_CHAR_BUDGET { return Err(LibsyError::AlgorithmError { - message: "judge_char_budget must be at least 1".to_string(), + message: format!("judge_char_budget must be at least {MIN_JUDGE_CHAR_BUDGET}"), }); } Ok(()) @@ -1716,15 +1718,17 @@ mod tests { Ok(()) } - /// A zero budget would clip every message to the trim marker, so it is rejected at - /// construction rather than serving the judge an unroutable payload. + /// A budget below the minimum could not fit the routing instruction that every windowed + /// payload ends with, so it is rejected at construction. #[test] - fn a_zero_judge_char_budget_is_rejected() { - let config = TaskClassifierConfig { - judge_char_budget: 0, + fn a_judge_char_budget_below_the_minimum_is_rejected() { + assert!(TRAILING_ROUTING_INSTRUCTION.chars().count() < MIN_JUDGE_CHAR_BUDGET); + let with_budget = |judge_char_budget| TaskClassifierConfig { + judge_char_budget, ..TaskClassifierConfig::default() }; - assert!(config.validate().is_err()); + assert!(with_budget(MIN_JUDGE_CHAR_BUDGET - 1).validate().is_err()); + assert!(with_budget(MIN_JUDGE_CHAR_BUDGET).validate().is_ok()); } fn tool_call(id: &str) -> Message { diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md index ce1975465..6834f50db 100644 --- a/docs/reference/toml_schema.md +++ b/docs/reference/toml_schema.md @@ -202,7 +202,7 @@ Capability mode classifies before serving. See | `classify_trigger` | No | `every_request` | When the judge runs. `every_request` judges every request, tool continuations included. `user_turn` judges each new user message and retains that target across intervening tool calls only when requests carry a session ID; without a session ID, it behaves like `every_request`. `new_session` judges once and reuses that target for the session. | | `message_hash_fallback` | No | `false` | Keys affinity on the first user message. Requires `classify_trigger = "new_session"`. | | `recent_turn_window` | No | unset | When unset, the judge sees the opening task and latest user follow-up, when present. When set, it also sees trailing turns. | -| `judge_char_budget` | No | `18000` | Most characters a windowed judge payload may use. The window narrows from the oldest turn until it fits, so one large tool result cannot decide judge cost and latency. Ignored without `recent_turn_window`. | +| `judge_char_budget` | No | `18000` | Most characters a windowed judge payload may use. The window narrows from the oldest turn until it fits, so one large tool result cannot decide judge cost and latency. Must be at least `256`. Ignored without `recent_turn_window`. | | `prompt` | No | packaged prompt | Replaces the capability prompt. The packaged schema is sent separately as structured-output configuration. | Escalation mode serves the weak target first and judges the completed turn. See @@ -243,7 +243,7 @@ how one route chooses between more than two models. | `classify_trigger` | No | `every_request` | When the judge runs. `every_request` judges every request, tool continuations included. `user_turn` judges each new user message and retains that target across intervening tool calls only when requests carry a session ID; without a session ID, it behaves like `every_request`. `new_session` judges once and reuses that target for the session. | | `message_hash_fallback` | No | `false` | Keys affinity on the first user message. Requires `classify_trigger = "new_session"`. | | `recent_turn_window` | No | unset | When unset, the judge sees the opening task and latest user follow-up, when present. When set, it also sees trailing turns. | -| `judge_char_budget` | No | `18000` | Most characters a windowed judge payload may use. The window narrows from the oldest turn until it fits, so one large tool result cannot decide judge cost and latency. Ignored without `recent_turn_window`. | +| `judge_char_budget` | No | `18000` | Most characters a windowed judge payload may use. The window narrows from the oldest turn until it fits, so one large tool result cannot decide judge cost and latency. Must be at least `256`. Ignored without `recent_turn_window`. | The selected JSON label must name a configured group. A label naming a target rather than a group, or a group you did not configure, falls back to diff --git a/docs/routing_algorithms/llm_classifier_routing.md b/docs/routing_algorithms/llm_classifier_routing.md index 40f5ee8ed..289546f2e 100644 --- a/docs/routing_algorithms/llm_classifier_routing.md +++ b/docs/routing_algorithms/llm_classifier_routing.md @@ -106,7 +106,7 @@ for the server merge behavior. | `base_threshold` | required | Lowest `p_solve` that routes a supported task to `weak_target`. Must be between `0` and `1`. | | `threshold_step` | `0.0` | Amount added for each boundary step. Must be finite and non-negative, and `base_threshold + 2 * threshold_step` must not exceed `1`. | | `recent_turn_window` | unset | When unset, the judge sees the opening user task and the latest user message when they differ. When set to `N`, it sees the opening user task and the last `N` conversation messages after that task. `0` keeps only the opening task. Client system and developer instructions are not shown to the judge. | -| `judge_char_budget` | `18000` | Most characters the windowed selection may use. A window is counted in turns, and one turn carrying a large tool result can be worth tens of thousands of characters, so the window narrows from the oldest turn until the payload fits. A task statement larger than the whole budget is clipped instead, marked with `...[trimmed]`. Ignored when `recent_turn_window` is unset. | +| `judge_char_budget` | `18000` | Most characters the windowed selection may use. A window is counted in turns, and one turn carrying a large tool result can be worth tens of thousands of characters, so the window narrows from the oldest turn until the payload fits. A task statement larger than the whole budget is clipped instead, marked with `...[trimmed]`. Must be at least `256`. Ignored when `recent_turn_window` is unset. | | `classify_trigger` | `every_request` | When the judge runs. `every_request` judges every request, tool continuations included. `user_turn` judges each new user message and holds that target across the tool calls between. `new_session` judges once and reuses that target for the session. | | `message_hash_fallback` | `false` | When session metadata is absent, keys affinity from the first user-message text. Requires `classify_trigger = "new_session"`. | | `prompt` | packaged capability prompt | Replaces the classifier's system prompt. The packaged verdict schema and routing policy remain active. | diff --git a/switchyard_rust/libsy.py b/switchyard_rust/libsy.py index 2a31efafb..45f378682 100644 --- a/switchyard_rust/libsy.py +++ b/switchyard_rust/libsy.py @@ -63,7 +63,8 @@ class CustomClassifierConfig: """Configure schema-validated routing across runtime model groups. ``max_output_tokens`` must be positive. Enabling ``message_hash_fallback`` - requires ``session_affinity``. + requires ``session_affinity``. ``judge_char_budget`` caps the windowed judge + payload, must be at least 256, and is ignored without ``recent_turn_window``. """ def __init__( @@ -157,6 +158,8 @@ class TaskClassifierConfig: Thresholds must remain within ``[0, 1]``, ``max_output_tokens`` must be positive, and ``message_hash_fallback`` requires ``session_affinity``. + ``judge_char_budget`` caps the windowed judge payload, must be at least 256, + and is ignored without ``recent_turn_window``. """ def __init__( From 92052f4f8cbd9080a2820eb427e1a3298aa6a711 Mon Sep 17 00:00:00 2001 From: Arnav Dadarya Date: Fri, 18 Sep 2026 17:46:01 -0700 Subject: [PATCH 3/3] fix(libsy): size the judge window in one pass and apply the budget to every judge Signed-off-by: Arnav Dadarya --- crates/libsy/src/algorithms/llm_class.rs | 243 ++++++++++++------ crates/libsy/src/algorithms/util.rs | 13 + .../libsy/src/algorithms/util/escalation.rs | 75 +++++- crates/switchyard-py/src/libsy_bindings.rs | 3 + crates/switchyard-runner/src/algorithm.rs | 13 +- crates/switchyard-runner/src/config.rs | 34 +++ docs/reference/toml_schema.md | 5 +- .../escalation_router_routing.md | 4 + .../llm_classifier_routing.md | 2 +- switchyard_rust/libsy.py | 14 +- 10 files changed, 298 insertions(+), 108 deletions(-) diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs index e8723f86c..31aa2a90f 100644 --- a/crates/libsy/src/algorithms/llm_class.rs +++ b/crates/libsy/src/algorithms/llm_class.rs @@ -23,7 +23,10 @@ use super::util::llm_judge::{ SerdeDecoder, StructuredJudge, }; use super::util::target_selector::TargetSelectorPolicy; -use super::util::{DEFAULT_JUDGE_CHAR_BUDGET, DEFAULT_JUDGE_MAX_OUTPUT_TOKENS, truncate_middle}; +use super::util::{ + DEFAULT_JUDGE_CHAR_BUDGET, DEFAULT_JUDGE_MAX_OUTPUT_TOKENS, truncate_middle, + validate_judge_char_budget, +}; use crate::core::algorithm::{Algorithm, Driver}; use crate::core::classifier::{Classification, Classifier, Score}; use crate::core::state::State; @@ -83,51 +86,77 @@ impl TaskClassifierVerdict { } } -/// Keeps the opening task and the last `recent_turn_window` turns after it. A -/// window of `0` keeps the task alone. +/// Keeps the opening task and the last `recent_turn_window` turns after it, within +/// `budget` characters. A window of `0` keeps the task alone. /// /// Inbound decoders normalize client system and developer content into /// `LlmRequest::instructions`, so it never reaches this list. /// +/// The anchors are kept whatever the budget, since the judge cannot route without the +/// task; the window gets what they leave. A task statement larger than the whole budget +/// is clipped rather than dropped. +/// /// Selects by reference and clones only what survives — a coding-agent /// conversation carries every tool result, so cloning it whole to keep a window /// would copy the transcript on each judged turn. -fn trim_messages(messages: &[Message], recent_turn_window: usize) -> Vec { +fn trim_messages(messages: &[Message], recent_turn_window: usize, budget: usize) -> Vec { let is_instruction = |message: &Message| matches!(message.role, Role::System | Role::Developer); let mut kept: Vec<&Message> = messages.iter().filter(|m| is_instruction(m)).collect(); - let Some(task) = messages.iter().position(|m| m.role == Role::User) else { - return kept.into_iter().cloned().collect(); - }; - kept.push(&messages[task]); - - let tail: Vec<&Message> = messages[task + 1..] - .iter() - .filter(|m| !is_instruction(m)) - .collect(); - kept.extend(&tail[window_start(&tail, recent_turn_window)..]); - kept.into_iter().cloned().collect() + if let Some(task) = messages.iter().position(|m| m.role == Role::User) { + kept.push(&messages[task]); + let anchor_chars: usize = kept.iter().map(|m| message_chars(m)).sum(); + let tail: Vec<&Message> = messages[task + 1..] + .iter() + .filter(|m| !is_instruction(m)) + .collect(); + let window_budget = budget.saturating_sub(anchor_chars); + kept.extend(&tail[window_start(&tail, recent_turn_window, window_budget)..]); + } + let mut kept: Vec = kept.into_iter().cloned().collect(); + // Only the anchors can still overrun: the window was sized to fit what they left. + if payload_chars(&kept) > budget { + clip_to_budget(&mut kept, budget); + } + kept } -/// The first index of the trailing window. +/// The first index of the trailing window: the widest suffix inside both limits. +/// +/// The turn count is the configured window. The character `budget` bounds what the judge +/// is sent: a window is counted in turns, and turn size varies by orders of magnitude, so +/// without it one large tool result would decide the judge's cost for a fixed +/// configuration. A message that does not fit on its own drops with everything older; +/// the newest turns, the evidence a routing judge needs most, are what survive. /// /// Counting messages alone can start the window between an assistant tool call and the /// result answering it, leaving the judge a result whose call id was never introduced. The -/// start therefore moves back to the nearest one that keeps every tool pair whole. +/// start therefore moves to the nearest one that keeps every tool pair whole: back to the +/// call when the turn count splits a pair, forward past the result when the budget does, +/// since widening past the budget is not an option. /// -/// One newest-to-oldest pass carries the ids still waiting for a call. Direction is what -/// makes it correct: ids repeat across a conversation, and in this order a call is only -/// ever seen after the results it could answer, so a later call — already passed — clears -/// nothing. A result whose call sits before the opening task, which trimming never reaches, -/// keeps the set non-empty to the end and falls back to the counted start, so an unpairable -/// result costs one pass and cannot widen the window to the whole conversation. -fn window_start(tail: &[&Message], recent_turn_window: usize) -> usize { +/// One newest-to-oldest pass carries the ids still waiting for a call and the running +/// size, so the cost is one visit per message however the limits fall. Direction is what +/// makes the pairing correct: ids repeat across a conversation, and in this order a call is +/// only ever seen after the results it could answer, so a later call — already passed — +/// clears nothing. A result whose call sits before the opening task, which trimming never +/// reaches, keeps the set non-empty to the end and falls back to the counted start, so an +/// unpairable result cannot widen the window to the whole conversation. +fn window_start(tail: &[&Message], recent_turn_window: usize, budget: usize) -> usize { let counted = tail.len().saturating_sub(recent_turn_window); // An empty window holds no result to pair, and the loop below never visits its start. if counted == tail.len() { return counted; } let mut unpaired: HashSet<&str> = HashSet::new(); + let mut chars = 0; + // The widest start seen with every pair whole, held in case the budget binds before + // the counted start is reached. + let mut whole = None; for (start, message) in tail.iter().enumerate().rev() { + chars += message_chars(message); + if chars > budget { + return whole.unwrap_or(tail.len()); + } // Blocks reverse too, so a call answers a result only when it precedes it inside // one message as well as across messages. for block in message.content.iter().rev() { @@ -141,8 +170,11 @@ fn window_start(tail: &[&Message], recent_turn_window: usize) -> usize { _ => {} } } - if start <= counted && unpaired.is_empty() { - return start; + if unpaired.is_empty() { + if start <= counted { + return start; + } + whole = Some(start); } } counted @@ -199,32 +231,6 @@ fn payload_chars(messages: &[Message]) -> usize { messages.iter().map(message_chars).sum() } -/// Selects the trailing window, narrowing it until the payload fits `budget` characters. -/// -/// A window is counted in turns, and turn size varies by orders of magnitude: four turns is -/// a few hundred characters of conversation, or tens of thousands when one turn carries a -/// large tool result. Judge cost and latency would otherwise be decided by the request -/// rather than by configuration, and a single large result can crowd out the task being -/// judged. -/// -/// Whole turns are dropped from the oldest end rather than clipping individual messages, -/// because [`trim_messages`] is what keeps tool calls paired with their results; removing -/// messages by hand would hand the judge a result whose call was never introduced. -fn window_within_budget(messages: &[Message], window: usize, budget: usize) -> Vec { - let mut window = window; - let mut kept = trim_messages(messages, window); - while window > 0 && payload_chars(&kept) > budget { - window -= 1; - kept = trim_messages(messages, window); - } - // The anchors — client instructions and the opening task — survive an empty window, so - // a task statement larger than the whole budget still has to be clipped. - if payload_chars(&kept) > budget { - clip_to_budget(&mut kept, budget); - } - kept -} - /// Clips text blocks so an unwindowable payload still fits `budget`. /// /// The share is per block rather than per message: one message can carry several text @@ -255,8 +261,7 @@ fn clip_to_budget(messages: &mut [Message], budget: usize) { /// Selects the task messages shown to capability and custom-schema classifiers. struct TaskInput { recent_turn_window: Option, - /// Character budget for the windowed payload. Unused without a window, where the - /// selection is the opening task and latest follow-up rather than conversation. + /// Character budget for the judge payload. judge_char_budget: usize, } @@ -267,13 +272,19 @@ impl ClassifierInput for TaskInput { let mut messages = match self.recent_turn_window { // The routing instruction appended below is part of what the judge is sent, so // its cost comes out of the budget rather than on top of it. - Some(window) => window_within_budget( + Some(window) => trim_messages( &request.llm_request.messages, window, self.judge_char_budget .saturating_sub(TRAILING_ROUTING_INSTRUCTION.chars().count()), ), - None => task_messages(&request.llm_request.messages), + None => { + let mut messages = task_messages(&request.llm_request.messages); + if payload_chars(&messages) > self.judge_char_budget { + clip_to_budget(&mut messages, self.judge_char_budget); + } + messages + } }; // Reasoning is provider-private and not required to classify the task. Some // upstreams also reject an unsigned reasoning item replayed without the @@ -399,10 +410,11 @@ pub struct TaskClassifierConfig { /// `Some(n)` widens that to the client instructions, the opening task, and /// the last `n` turns after it. pub recent_turn_window: Option, - /// Character budget for a windowed judge payload. + /// Character budget for the judge payload. /// - /// Bounds what one request can spend on a judge call when `recent_turn_window` is set: - /// the window narrows until the selection fits. Ignored without a window. + /// Bounds what one request can spend on a judge call. With `recent_turn_window` set, + /// the window narrows until the selection fits; without it, an oversized task + /// statement is clipped. pub judge_char_budget: usize, /// Prompt and verdict contract settings for the classifier judge. pub contract: ClassifierContractConfig, @@ -465,19 +477,6 @@ const fn default_judge_char_budget() -> usize { DEFAULT_JUDGE_CHAR_BUDGET } -/// Smallest accepted judge payload budget. It must leave room for the routing instruction -/// appended to every windowed payload, or that instruction alone would exceed the budget. -const MIN_JUDGE_CHAR_BUDGET: usize = 256; - -fn validate_judge_char_budget(budget: usize) -> Result<()> { - if budget < MIN_JUDGE_CHAR_BUDGET { - return Err(LibsyError::AlgorithmError { - message: format!("judge_char_budget must be at least {MIN_JUDGE_CHAR_BUDGET}"), - }); - } - Ok(()) -} - impl Default for TaskClassifierConfig { fn default() -> Self { Self { @@ -570,7 +569,7 @@ pub struct CustomClassifierConfig { pub message_hash_fallback: bool, /// Trailing conversation turns shown to the classifier judge. pub recent_turn_window: Option, - /// Character budget for a windowed judge payload. Ignored without a window. + /// Character budget for the judge payload. pub judge_char_budget: usize, /// Maximum completion tokens available to the classifier verdict. pub max_output_tokens: u64, @@ -1546,8 +1545,8 @@ mod tests { } /// A window is counted in turns, so one turn carrying a large tool result would - /// otherwise decide the judge's cost for a fixed configuration. The window narrows - /// from the oldest end until the payload fits, so the newest evidence survives. + /// otherwise decide the judge's cost for a fixed configuration. The window ends at + /// the first message that does not fit, so the newest evidence survives. #[test] fn an_oversized_turn_narrows_the_window_to_fit_the_budget() -> Result<()> { let messages = vec![ @@ -1603,8 +1602,90 @@ mod tests { Ok(()) } - /// Narrowing drops whole turns through `trim_messages`, so a surviving tool result - /// still has the call that introduced its id. Removing messages directly would not. + /// A message larger than the budget on its own cannot be kept. It drops with the + /// older turns, not with the newer ones that fit. + #[test] + fn a_message_larger_than_the_budget_drops_only_itself_and_older_turns() -> Result<()> { + let messages = vec![ + Message::text(Role::User, "initial task"), + Message::text(Role::User, "recent 1"), + Message::text(Role::Assistant, "x".repeat(20_000)), + Message::text(Role::User, "recent 3"), + ]; + let built = budgeted_messages(messages, 4, 5_000)?; + let texts: Vec = built + .iter() + .filter_map(|message| message.text_content("\n")) + .collect(); + + assert!(texts.contains(&"initial task".to_string()), "{texts:?}"); + assert!(texts.contains(&"recent 3".to_string()), "{texts:?}"); + assert!(!texts.contains(&"recent 1".to_string()), "{texts:?}"); + assert!(!texts.iter().any(|text| text.len() > 10_000), "{texts:?}"); + Ok(()) + } + + /// The budget applies without a window too: the task messages are clipped rather + /// than sent whole. + #[test] + fn the_default_path_is_clipped_to_the_budget() -> Result<()> { + let judge = capability_judge_with_budget(None, 1_000)?; + let messages = vec![ + Message::text(Role::User, "z".repeat(40_000)), + Message::text(Role::Assistant, "reply"), + Message::text(Role::User, "follow-up"), + ]; + let built = judge + .build_request(&State::default(), &windowed_request(messages)) + .llm_request + .messages; + + assert!(payload_chars(&built) <= 1_000, "{}", payload_chars(&built)); + assert!( + built + .iter() + .any(|message| message.text_content("\n").as_deref() == Some("follow-up")), + "{built:?}" + ); + Ok(()) + } + + /// The budget can end the window between a call and its result. Widening back to the + /// call is not an option, so the result is dropped rather than sent orphaned. + #[test] + fn a_result_whose_call_does_not_fit_is_dropped_with_it() -> Result<()> { + let mut bulky_call = tool_call("call-1"); + bulky_call.content = vec![ContentBlock::ToolCall(ToolCall { + id: "call-1".to_string(), + name: "search".to_string(), + arguments: Value::String("y".repeat(20_000)), + })]; + let messages = vec![ + Message::text(Role::User, "initial task"), + bulky_call, + tool_result("call-1"), + Message::text(Role::User, "recent"), + ]; + let built = budgeted_messages(messages, 4, 5_000)?; + + assert!( + !built + .iter() + .flat_map(|message| &message.content) + .any(|block| matches!(block, ContentBlock::ToolResult(_))), + "{built:?}" + ); + assert!( + built + .iter() + .any(|message| message.text_content("\n").as_deref() == Some("recent")), + "{built:?}" + ); + Ok(()) + } + + /// The window is cut at whole tool pairs, so a surviving tool result still has the + /// call that introduced its id. #[test] fn narrowing_for_the_budget_keeps_tool_pairs_whole() -> Result<()> { let mut bulky = tool_result("call-1"); @@ -1722,6 +1803,8 @@ mod tests { /// payload ends with, so it is rejected at construction. #[test] fn a_judge_char_budget_below_the_minimum_is_rejected() { + use crate::algorithms::util::MIN_JUDGE_CHAR_BUDGET; + assert!(TRAILING_ROUTING_INSTRUCTION.chars().count() < MIN_JUDGE_CHAR_BUDGET); let with_budget = |judge_char_budget| TaskClassifierConfig { judge_char_budget, @@ -1772,7 +1855,7 @@ mod tests { ]; // The five-message tail begins exactly on the tool result. - let kept = trim_messages(&messages, 5); + let kept = trim_messages(&messages, 5, DEFAULT_JUDGE_CHAR_BUDGET); assert_eq!( kept, @@ -1804,7 +1887,7 @@ mod tests { ]; // The four-message tail begins on the first result, whose own call sits one earlier. - let kept = trim_messages(&messages, 4); + let kept = trim_messages(&messages, 4, DEFAULT_JUDGE_CHAR_BUDGET); assert_eq!( kept, @@ -1834,7 +1917,7 @@ mod tests { Message::text(Role::User, "recent 2"), ]; - let kept = trim_messages(&messages, 3); + let kept = trim_messages(&messages, 3, DEFAULT_JUDGE_CHAR_BUDGET); assert_eq!( kept, diff --git a/crates/libsy/src/algorithms/util.rs b/crates/libsy/src/algorithms/util.rs index 4bf7479b1..900c0eef5 100644 --- a/crates/libsy/src/algorithms/util.rs +++ b/crates/libsy/src/algorithms/util.rs @@ -35,6 +35,19 @@ pub(crate) const DEFAULT_JUDGE_MAX_OUTPUT_TOKENS: u64 = 4_096; /// much a judge call costs. pub(crate) const DEFAULT_JUDGE_CHAR_BUDGET: usize = 18_000; +/// Smallest accepted judge payload budget. It must leave room for the framing every judge +/// adds around the conversation, or that framing alone would exceed the budget. +pub(crate) const MIN_JUDGE_CHAR_BUDGET: usize = 256; + +pub(crate) fn validate_judge_char_budget(budget: usize) -> crate::Result<()> { + if budget < MIN_JUDGE_CHAR_BUDGET { + return Err(crate::LibsyError::AlgorithmError { + message: format!("judge_char_budget must be at least {MIN_JUDGE_CHAR_BUDGET}"), + }); + } + Ok(()) +} + /// Separator marking where [`truncate_middle`] dropped a message's interior. pub(crate) const TRIM_MARKER: &str = " ...[trimmed] "; diff --git a/crates/libsy/src/algorithms/util/escalation.rs b/crates/libsy/src/algorithms/util/escalation.rs index 7fe25900b..229684f5b 100644 --- a/crates/libsy/src/algorithms/util/escalation.rs +++ b/crates/libsy/src/algorithms/util/escalation.rs @@ -16,7 +16,7 @@ use super::llm_judge::{ ClassifierInput, JudgeClassifier, JudgePolicy, JudgeRuntimeConfig, SerdeDecoder, StructuredJudge, }; -use super::truncate_middle; +use super::{DEFAULT_JUDGE_CHAR_BUDGET, truncate_middle, validate_judge_char_budget}; use crate::core::algorithm::Driver; use crate::core::classifier::{Classification, Score}; use crate::core::state::State; @@ -26,7 +26,7 @@ use switchyard_protocol::Request; const PROMPT_TEMPLATE: &str = include_str!("../../prompts/escalation/prompt.md"); const SCHEMA_TEMPLATE: &str = include_str!("../../prompts/escalation/schema.json"); -/// Suffix marking a transcript cut off by [`MAX_REQUEST_CHARS`]. +/// Suffix marking a transcript cut off by `judge_char_budget`. const TRUNCATION_SUFFIX: &str = "..."; /// Per-message cap for system and developer anchors, which carry no trajectory signal but @@ -40,9 +40,6 @@ const SYSTEM_CHARS: usize = 1_000; /// thousand characters, so this gets the widest anchor budget. const TASK_CHARS: usize = 4_000; -/// Backstop on the assembled transcript; the per-message caps normally bind first. -const MAX_REQUEST_CHARS: usize = 18_000; - /// The tuning surface for the trajectory judge. /// /// The routing settings retain their benchmarked defaults. Everything else is a fixed invariant @@ -59,6 +56,17 @@ pub struct EscalationJudgeConfig { pub recent_turn_window: usize, /// Per-message cap inside the trailing window. pub window_message_chars: usize, + /// Most characters the assembled transcript may use. The per-message caps normally + /// bind first; this backstop drops the oldest window lines when they do not. + /// + /// Not read from the `escalation` table: hosts set it from the route-level + /// `judge_char_budget`, which applies to every classifier mode. + #[serde(skip, default = "default_judge_char_budget")] + pub judge_char_budget: usize, +} + +const fn default_judge_char_budget() -> usize { + DEFAULT_JUDGE_CHAR_BUDGET } impl EscalationJudgeConfig { @@ -77,7 +85,7 @@ impl EscalationJudgeConfig { self.window_message_chars )); } - Ok(()) + validate_judge_char_budget(self.judge_char_budget) } } @@ -87,6 +95,7 @@ impl Default for EscalationJudgeConfig { confirmations: 2, recent_turn_window: 28, window_message_chars: 500, + judge_char_budget: DEFAULT_JUDGE_CHAR_BUDGET, } } } @@ -244,7 +253,7 @@ fn collect_text(content: &[ContentBlock], parts: &mut Vec) { /// trailing window carries recent activity. A coverage header states how much history is not /// shown, so the judge can reason about pace rather than assuming it sees everything. /// -/// When the assembled text still exceeds `max_request_chars`, the oldest window lines go +/// When the assembled text still exceeds `judge_char_budget`, the oldest window lines go /// first: for a trajectory judge the newest evidence is strictly the most valuable. fn summarize_for_judge( messages: &[Message], @@ -300,13 +309,14 @@ fn summarize_for_judge( .join("\n") }; + let budget = config.judge_char_budget; let mut text = assemble(&window); - while text.chars().count() > MAX_REQUEST_CHARS && !window.is_empty() { + while text.chars().count() > budget && !window.is_empty() { window.remove(0); text = assemble(&window); } - if text.chars().count() > MAX_REQUEST_CHARS { - let keep = MAX_REQUEST_CHARS.saturating_sub(TRUNCATION_SUFFIX.chars().count() + 1); + if text.chars().count() > budget { + let keep = budget.saturating_sub(TRUNCATION_SUFFIX.chars().count() + 1); text = text.chars().take(keep).collect::() + TRUNCATION_SUFFIX; } text @@ -566,8 +576,8 @@ mod tests { #[test] fn summary_drops_oldest_window_lines_under_the_char_cap() { - // MAX_REQUEST_CHARS is a backstop, not a dial: at default settings the window caps - // bind first (28 x 500 plus anchors sits under it), so reaching it takes an unusually + // The budget is a backstop, not a dial: at default settings the window caps bind + // first (28 x 500 plus anchors sits under it), so reaching it takes an unusually // wide per-message cap. That is the point — it only fires on pathological input. let mut messages = vec![ Message::text(Role::System, "framing"), @@ -587,7 +597,7 @@ mod tests { let summary = summarize_for_judge(&messages, 21, &config); assert!( - summary.chars().count() <= MAX_REQUEST_CHARS, + summary.chars().count() <= config.judge_char_budget, "{}", summary.chars().count() ); @@ -597,4 +607,43 @@ mod tests { assert!(summary.contains("19 xxx"), "{summary}"); assert!(!summary.contains("0 xxx"), "{summary}"); } + + /// The cap is the route's `judge_char_budget`, not a constant: a smaller budget drops + /// more of the window. + #[test] + fn the_configured_budget_caps_the_summary() { + let mut messages = vec![Message::text(Role::User, "task")]; + for i in 0..20 { + messages.push(Message::text( + Role::Assistant, + format!("{i} {}", "x".repeat(400)), + )); + } + let config = EscalationJudgeConfig { + judge_char_budget: 2_000, + ..EscalationJudgeConfig::default() + }; + + let summary = summarize_for_judge(&messages, 21, &config); + + assert!( + summary.chars().count() <= 2_000, + "{}", + summary.chars().count() + ); + assert!(summary.contains("19 xxx"), "{summary}"); + assert!(!summary.contains("10 xxx"), "{summary}"); + } + + /// The budget is set by the host from the route-level key, so the `escalation` table + /// neither reads it nor loses the default. + #[test] + fn the_budget_is_not_read_from_the_escalation_table() { + let config: EscalationJudgeConfig = serde_json::from_str("{}").unwrap(); + assert_eq!(config.judge_char_budget, DEFAULT_JUDGE_CHAR_BUDGET); + assert!( + serde_json::from_str::(r#"{"judge_char_budget": 1000}"#) + .is_err() + ); + } } diff --git a/crates/switchyard-py/src/libsy_bindings.rs b/crates/switchyard-py/src/libsy_bindings.rs index b8ac06846..34a0cee36 100644 --- a/crates/switchyard-py/src/libsy_bindings.rs +++ b/crates/switchyard-py/src/libsy_bindings.rs @@ -114,6 +114,7 @@ impl PyEscalationClassifierConfig { confirmations=2, recent_turn_window=28, window_message_chars=500, + judge_char_budget=18_000, max_output_tokens=4096, prompt=None, response_format_type="json_schema" @@ -123,6 +124,7 @@ impl PyEscalationClassifierConfig { confirmations: u32, recent_turn_window: usize, window_message_chars: usize, + judge_char_budget: usize, max_output_tokens: u64, prompt: Option, response_format_type: &str, @@ -133,6 +135,7 @@ impl PyEscalationClassifierConfig { confirmations, recent_turn_window, window_message_chars, + judge_char_budget, }, max_output_tokens, }) diff --git a/crates/switchyard-runner/src/algorithm.rs b/crates/switchyard-runner/src/algorithm.rs index a698ea2a8..c48b26a6f 100644 --- a/crates/switchyard-runner/src/algorithm.rs +++ b/crates/switchyard-runner/src/algorithm.rs @@ -247,9 +247,9 @@ pub struct LlmClassifierRouteConfig { /// How many trailing turns the judge sees. Unset shows it the opening task /// and the latest user follow-up only. pub recent_turn_window: Option, - /// Most characters a windowed judge payload may use. The window narrows from the - /// oldest turn until it fits, so one large tool result cannot decide the judge's - /// cost. Ignored without `recent_turn_window`. + /// Most characters the judge payload may use, in every mode. A window narrows from + /// the oldest turn until it fits, so one large tool result cannot decide the judge's + /// cost; without a window the task messages are clipped instead. #[serde(default = "default_judge_char_budget")] pub judge_char_budget: usize, /// Replaces the packaged judge prompt. Required in custom mode. @@ -471,7 +471,7 @@ pub struct StageClassifierConfig { /// and the latest user follow-up only. #[serde(default)] pub recent_turn_window: Option, - /// Most characters a windowed judge payload may use. Ignored without a window. + /// Most characters the judge payload may use. #[serde(default = "default_judge_char_budget")] pub judge_char_budget: usize, /// Replaces the packaged judge prompt. @@ -958,7 +958,10 @@ impl LlmClassifierRouteConfig { prompt: prompt.clone(), response_format_type: *response_format_type, max_output_tokens: *max_output_tokens, - judge: required_classifier_field(route_name, "escalation", escalation)?, + judge: EscalationJudgeConfig { + judge_char_budget: *judge_char_budget, + ..required_classifier_field(route_name, "escalation", escalation)? + }, }, )) } diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rs index 316754723..a2e12bc78 100644 --- a/crates/switchyard-runner/src/config.rs +++ b/crates/switchyard-runner/src/config.rs @@ -1125,6 +1125,34 @@ new = ["send_message"] Ok(()) } + /// The route-level budget reaches every judge: a value the escalation judge rejects + /// fails the build, so it is not silently left at the default there. + #[test] + fn classifier_judge_char_budget_applies_in_every_mode() -> RunnerResult<()> { + let capability = VALID_CONFIG.replace( + "base_threshold = 0.5", + "base_threshold = 0.5\njudge_char_budget = 1000", + ); + runner_from_toml(&capability)?; + + let escalation = VALID_CONFIG.replace( + "base_threshold = 0.5", + "base_threshold = 0.5\njudge_char_budget = 1000\nescalation = { confirmations = 2 }", + ); + runner_from_toml(&escalation)?; + + let too_small = VALID_CONFIG.replace( + "base_threshold = 0.5", + "base_threshold = 0.5\njudge_char_budget = 100\nescalation = { confirmations = 2 }", + ); + assert!( + error_message(&too_small).contains("judge_char_budget must be at least 256"), + "{}", + error_message(&too_small) + ); + Ok(()) + } + #[test] fn classifier_prompts_are_configurable_in_both_modes() -> RunnerResult<()> { let capability = VALID_CONFIG.replace( @@ -1215,6 +1243,12 @@ efficient_target = "weak" ); assert!(error_message(&nested_completion_cap).contains("unknown field")); + let nested_char_budget = VALID_CONFIG.replace( + "base_threshold = 0.5", + "base_threshold = 0.5\nescalation = { judge_char_budget = 1000 }", + ); + assert!(error_message(&nested_char_budget).contains("unknown field")); + let unknown_classifier_field = VALID_CONFIG.replace( "base_threshold = 0.5", "base_threshold = 0.5\nclassifier_magic = true", diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md index 6834f50db..b3acb35fa 100644 --- a/docs/reference/toml_schema.md +++ b/docs/reference/toml_schema.md @@ -181,13 +181,14 @@ checkpoint = "/models/router.pt" ### `llm_classifier` Runs one of three judge-backed modes: `capability`, `escalation`, or `custom`. -`max_output_tokens` applies to all three. +`max_output_tokens` and `judge_char_budget` apply to all three. | Key | Required | Default | Meaning | |---|:---:|---|---| | `mode` | No | `capability` | Classifier behavior. Set it explicitly for new configurations. | | `classifier_target` | Capability, escalation | — | Target the judge is called through. Not a routing destination. Custom mode uses `models.judge`. | | `max_output_tokens` | No | `4096` | Maximum completion tokens for the judge verdict. Must be at least `1`. | +| `judge_char_budget` | No | `18000` | Most characters sent to the judge per call, so one large tool result cannot decide judge cost and latency. A window narrows from the oldest turn until it fits; without a window the task messages are clipped, marked with `...[trimmed]`. Roughly four characters per token. Must be at least `256`. | | `response_format_type` | No | `json_schema` | Structured-output mode for capability and escalation judges. Use `json_object` when the provider does not support JSON Schema; Switchyard adds the schema to the prompt and validates the verdict locally. Custom mode always uses its configured JSON Schema. | Capability mode classifies before serving. See @@ -202,7 +203,6 @@ Capability mode classifies before serving. See | `classify_trigger` | No | `every_request` | When the judge runs. `every_request` judges every request, tool continuations included. `user_turn` judges each new user message and retains that target across intervening tool calls only when requests carry a session ID; without a session ID, it behaves like `every_request`. `new_session` judges once and reuses that target for the session. | | `message_hash_fallback` | No | `false` | Keys affinity on the first user message. Requires `classify_trigger = "new_session"`. | | `recent_turn_window` | No | unset | When unset, the judge sees the opening task and latest user follow-up, when present. When set, it also sees trailing turns. | -| `judge_char_budget` | No | `18000` | Most characters a windowed judge payload may use. The window narrows from the oldest turn until it fits, so one large tool result cannot decide judge cost and latency. Must be at least `256`. Ignored without `recent_turn_window`. | | `prompt` | No | packaged prompt | Replaces the capability prompt. The packaged schema is sent separately as structured-output configuration. | Escalation mode serves the weak target first and judges the completed turn. See @@ -243,7 +243,6 @@ how one route chooses between more than two models. | `classify_trigger` | No | `every_request` | When the judge runs. `every_request` judges every request, tool continuations included. `user_turn` judges each new user message and retains that target across intervening tool calls only when requests carry a session ID; without a session ID, it behaves like `every_request`. `new_session` judges once and reuses that target for the session. | | `message_hash_fallback` | No | `false` | Keys affinity on the first user message. Requires `classify_trigger = "new_session"`. | | `recent_turn_window` | No | unset | When unset, the judge sees the opening task and latest user follow-up, when present. When set, it also sees trailing turns. | -| `judge_char_budget` | No | `18000` | Most characters a windowed judge payload may use. The window narrows from the oldest turn until it fits, so one large tool result cannot decide judge cost and latency. Must be at least `256`. Ignored without `recent_turn_window`. | The selected JSON label must name a configured group. A label naming a target rather than a group, or a group you did not configure, falls back to diff --git a/docs/routing_algorithms/escalation_router_routing.md b/docs/routing_algorithms/escalation_router_routing.md index 1ed9e8085..6592a8392 100644 --- a/docs/routing_algorithms/escalation_router_routing.md +++ b/docs/routing_algorithms/escalation_router_routing.md @@ -107,6 +107,10 @@ configuration, so a bare `escalation = {}` is a valid, tuned route: | `recent_turn_window` | `28` | Trailing messages shown to the judge on top of the anchors. Must be at least `1`. | | `window_message_chars` | `500` | Per-message truncation cap inside that trailing window. Must be at least `50`. | +The route-level `judge_char_budget` (default `18000`) caps the whole transcript +sent to the judge. The per-message caps normally keep it well under that; when +they do not, the oldest window lines are dropped first. + `confirmations` is the main cost dial. `1` latches sooner and spends more on the strong tier. `2` or higher requires a session identity, because the streak is retained per session — without one, every turn starts from zero and the route diff --git a/docs/routing_algorithms/llm_classifier_routing.md b/docs/routing_algorithms/llm_classifier_routing.md index 289546f2e..34ca21013 100644 --- a/docs/routing_algorithms/llm_classifier_routing.md +++ b/docs/routing_algorithms/llm_classifier_routing.md @@ -106,7 +106,7 @@ for the server merge behavior. | `base_threshold` | required | Lowest `p_solve` that routes a supported task to `weak_target`. Must be between `0` and `1`. | | `threshold_step` | `0.0` | Amount added for each boundary step. Must be finite and non-negative, and `base_threshold + 2 * threshold_step` must not exceed `1`. | | `recent_turn_window` | unset | When unset, the judge sees the opening user task and the latest user message when they differ. When set to `N`, it sees the opening user task and the last `N` conversation messages after that task. `0` keeps only the opening task. Client system and developer instructions are not shown to the judge. | -| `judge_char_budget` | `18000` | Most characters the windowed selection may use. A window is counted in turns, and one turn carrying a large tool result can be worth tens of thousands of characters, so the window narrows from the oldest turn until the payload fits. A task statement larger than the whole budget is clipped instead, marked with `...[trimmed]`. Must be at least `256`. Ignored when `recent_turn_window` is unset. | +| `judge_char_budget` | `18000` | Most characters sent to the judge per call, roughly four per token. A window is counted in turns, and one turn carrying a large tool result can be worth tens of thousands of characters, so the window ends at the first message that does not fit. A task statement larger than the whole budget is clipped instead, marked with `...[trimmed]`. Must be at least `256`. | | `classify_trigger` | `every_request` | When the judge runs. `every_request` judges every request, tool continuations included. `user_turn` judges each new user message and holds that target across the tool calls between. `new_session` judges once and reuses that target for the session. | | `message_hash_fallback` | `false` | When session metadata is absent, keys affinity from the first user-message text. Requires `classify_trigger = "new_session"`. | | `prompt` | packaged capability prompt | Replaces the classifier's system prompt. The packaged verdict schema and routing policy remain active. | diff --git a/switchyard_rust/libsy.py b/switchyard_rust/libsy.py index 45f378682..6065882c1 100644 --- a/switchyard_rust/libsy.py +++ b/switchyard_rust/libsy.py @@ -63,8 +63,8 @@ class CustomClassifierConfig: """Configure schema-validated routing across runtime model groups. ``max_output_tokens`` must be positive. Enabling ``message_hash_fallback`` - requires ``session_affinity``. ``judge_char_budget`` caps the windowed judge - payload, must be at least 256, and is ignored without ``recent_turn_window``. + requires ``session_affinity``. ``judge_char_budget`` caps the characters sent + to the judge and must be at least 256. """ def __init__( @@ -84,8 +84,9 @@ def __init__( class EscalationClassifierConfig: """Configure response-based escalation between two targets. - Counts and token limits must be positive, and ``window_message_chars`` - must be at least 50. + Counts and token limits must be positive, ``window_message_chars`` must be + at least 50, and ``judge_char_budget`` caps the characters sent to the judge + and must be at least 256. """ def __init__( @@ -94,6 +95,7 @@ def __init__( confirmations: int = 2, recent_turn_window: int = 28, window_message_chars: int = 500, + judge_char_budget: int = 18_000, max_output_tokens: int = 4096, prompt: str | None = None, response_format_type: Literal["json_schema", "json_object"] = "json_schema", @@ -158,8 +160,8 @@ class TaskClassifierConfig: Thresholds must remain within ``[0, 1]``, ``max_output_tokens`` must be positive, and ``message_hash_fallback`` requires ``session_affinity``. - ``judge_char_budget`` caps the windowed judge payload, must be at least 256, - and is ignored without ``recent_turn_window``. + ``judge_char_budget`` caps the characters sent to the judge and must be at + least 256. """ def __init__(