diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs index 4bb5e7ff0..e8723f86c 100644 --- a/crates/libsy/src/algorithms/llm_class.rs +++ b/crates/libsy/src/algorithms/llm_class.rs @@ -13,7 +13,6 @@ use switchyard_protocol::{Category, ContentBlock, Message, Role}; use super::escalation; use super::fall_through::FallThrough; -use super::util::DEFAULT_JUDGE_MAX_OUTPUT_TOKENS; use super::util::affinity::{AffinityRouter, ClassifyTrigger}; use super::util::classifier_contract::{ ClassifierContract, ClassifierContractConfig, ClassifierResponseFormat, @@ -24,6 +23,7 @@ use super::util::llm_judge::{ SerdeDecoder, StructuredJudge, }; use super::util::target_selector::TargetSelectorPolicy; +use super::util::{DEFAULT_JUDGE_CHAR_BUDGET, DEFAULT_JUDGE_MAX_OUTPUT_TOKENS, truncate_middle}; use crate::core::algorithm::{Algorithm, Driver}; use crate::core::classifier::{Classification, Classifier, Score}; use crate::core::state::State; @@ -160,9 +160,104 @@ fn task_messages(messages: &[Message]) -> Vec { } } +/// Characters one message costs a judge. +/// +/// Counts tool traffic as well as visible text: in a coding-agent conversation a single +/// tool result is routinely larger than every text block around it, so measuring text +/// alone would report a payload as small while the judge is billed for all of it. +fn message_chars(message: &Message) -> usize { + message.content.iter().map(block_chars).sum() +} + +/// Characters one content block costs a judge. +fn block_chars(block: &ContentBlock) -> usize { + match block { + ContentBlock::Text { text } | ContentBlock::Refusal { text } => text.chars().count(), + ContentBlock::ToolCall(call) => { + call.name.chars().count() + call.arguments.to_string().chars().count() + } + ContentBlock::ToolResult(result) => result.content.iter().map(block_chars).sum(), + // Reasoning is stripped before the judge is called, so it costs nothing. Media and + // unknown blocks are opaque here; their wire cost is not text. + _ => 0, + } +} + +/// Whether clipping can shrink this block. +/// +/// Tool arguments and results are JSON the judge may need to read as a unit, so they cannot +/// be cut down to fit. +fn is_clippable(block: &ContentBlock) -> bool { + matches!( + block, + ContentBlock::Text { .. } | ContentBlock::Refusal { .. } + ) +} + +/// Total judge payload size for a selected message list. +fn payload_chars(messages: &[Message]) -> usize { + messages.iter().map(message_chars).sum() +} + +/// Selects the trailing window, narrowing it until the payload fits `budget` characters. +/// +/// A window is counted in turns, and turn size varies by orders of magnitude: four turns is +/// a few hundred characters of conversation, or tens of thousands when one turn carries a +/// large tool result. Judge cost and latency would otherwise be decided by the request +/// rather than by configuration, and a single large result can crowd out the task being +/// judged. +/// +/// Whole turns are dropped from the oldest end rather than clipping individual messages, +/// because [`trim_messages`] is what keeps tool calls paired with their results; removing +/// messages by hand would hand the judge a result whose call was never introduced. +fn window_within_budget(messages: &[Message], window: usize, budget: usize) -> Vec { + let mut window = window; + let mut kept = trim_messages(messages, window); + while window > 0 && payload_chars(&kept) > budget { + window -= 1; + kept = trim_messages(messages, window); + } + // The anchors — client instructions and the opening task — survive an empty window, so + // a task statement larger than the whole budget still has to be clipped. + if payload_chars(&kept) > budget { + clip_to_budget(&mut kept, budget); + } + kept +} + +/// Clips text blocks so an unwindowable payload still fits `budget`. +/// +/// The share is per block rather than per message: one message can carry several text +/// blocks, and a per-message allowance would let each of them spend it in full. +/// +/// Blocks that cannot be clipped are charged first, so what remains is what the text is +/// allowed to spend. When they alone exceed the budget every text block collapses to +/// nothing and the payload still overruns; cutting tool JSON to fit would hand the judge +/// malformed values, which is worse than an oversized prompt. +fn clip_to_budget(messages: &mut [Message], budget: usize) { + let blocks = || messages.iter().flat_map(|message| &message.content); + let clippable = blocks().filter(|block| is_clippable(block)).count(); + if clippable == 0 { + return; + } + let fixed: usize = blocks() + .filter(|block| !is_clippable(block)) + .map(block_chars) + .sum(); + let per_block = budget.saturating_sub(fixed) / clippable; + for block in messages.iter_mut().flat_map(|message| &mut message.content) { + if let ContentBlock::Text { text } | ContentBlock::Refusal { text } = block { + *text = truncate_middle(text, per_block); + } + } +} + /// Selects the task messages shown to capability and custom-schema classifiers. struct TaskInput { recent_turn_window: Option, + /// Character budget for the windowed payload. Unused without a window, where the + /// selection is the opening task and latest follow-up rather than conversation. + judge_char_budget: usize, } impl ClassifierInput for TaskInput { @@ -170,7 +265,14 @@ impl ClassifierInput for TaskInput { // The default preserves the whole-task anchor and latest user update. A // configured window widens that to the surrounding conversation. let mut messages = match self.recent_turn_window { - Some(window) => trim_messages(&request.llm_request.messages, window), + // The routing instruction appended below is part of what the judge is sent, so + // its cost comes out of the budget rather than on top of it. + Some(window) => window_within_budget( + &request.llm_request.messages, + window, + self.judge_char_budget + .saturating_sub(TRAILING_ROUTING_INSTRUCTION.chars().count()), + ), None => task_messages(&request.llm_request.messages), }; // Reasoning is provider-private and not required to classify the task. Some @@ -297,6 +399,11 @@ pub struct TaskClassifierConfig { /// `Some(n)` widens that to the client instructions, the opening task, and /// the last `n` turns after it. pub recent_turn_window: Option, + /// Character budget for a windowed judge payload. + /// + /// Bounds what one request can spend on a judge call when `recent_turn_window` is set: + /// the window narrows until the selection fits. Ignored without a window. + pub judge_char_budget: usize, /// Prompt and verdict contract settings for the classifier judge. pub contract: ClassifierContractConfig, /// Maximum completion tokens available to the classifier verdict. @@ -316,6 +423,8 @@ struct TaskClassifierConfigWire { message_hash_fallback: bool, #[serde(default)] recent_turn_window: Option, + #[serde(default = "default_judge_char_budget")] + judge_char_budget: usize, #[serde(default)] prompt: Option, #[serde(default)] @@ -341,6 +450,7 @@ impl<'de> Deserialize<'de> for TaskClassifierConfig { classify_trigger: wire.classify_trigger, message_hash_fallback: wire.message_hash_fallback, recent_turn_window: wire.recent_turn_window, + judge_char_budget: wire.judge_char_budget, contract, max_output_tokens: wire.max_output_tokens, }) @@ -351,6 +461,23 @@ const fn default_judge_max_output_tokens() -> u64 { DEFAULT_JUDGE_MAX_OUTPUT_TOKENS } +const fn default_judge_char_budget() -> usize { + DEFAULT_JUDGE_CHAR_BUDGET +} + +/// Smallest accepted judge payload budget. It must leave room for the routing instruction +/// appended to every windowed payload, or that instruction alone would exceed the budget. +const MIN_JUDGE_CHAR_BUDGET: usize = 256; + +fn validate_judge_char_budget(budget: usize) -> Result<()> { + if budget < MIN_JUDGE_CHAR_BUDGET { + return Err(LibsyError::AlgorithmError { + message: format!("judge_char_budget must be at least {MIN_JUDGE_CHAR_BUDGET}"), + }); + } + Ok(()) +} + impl Default for TaskClassifierConfig { fn default() -> Self { Self { @@ -359,6 +486,7 @@ impl Default for TaskClassifierConfig { classify_trigger: ClassifyTrigger::default(), message_hash_fallback: false, recent_turn_window: None, + judge_char_budget: DEFAULT_JUDGE_CHAR_BUDGET, contract: ClassifierContractConfig::default(), max_output_tokens: DEFAULT_JUDGE_MAX_OUTPUT_TOKENS, } @@ -368,6 +496,7 @@ impl Default for TaskClassifierConfig { impl TaskClassifierConfig { /// Validates routing thresholds before the classifier is constructed. fn validate(&self) -> Result<()> { + validate_judge_char_budget(self.judge_char_budget)?; if !(0.0..=1.0).contains(&self.base_threshold) { return Err(LibsyError::AlgorithmError { message: format!( @@ -441,6 +570,8 @@ pub struct CustomClassifierConfig { pub message_hash_fallback: bool, /// Trailing conversation turns shown to the classifier judge. pub recent_turn_window: Option, + /// Character budget for a windowed judge payload. Ignored without a window. + pub judge_char_budget: usize, /// Maximum completion tokens available to the classifier verdict. pub max_output_tokens: u64, } @@ -459,11 +590,13 @@ impl CustomClassifierConfig { classify_trigger: ClassifyTrigger::default(), message_hash_fallback: false, recent_turn_window: None, + judge_char_budget: DEFAULT_JUDGE_CHAR_BUDGET, max_output_tokens: DEFAULT_JUDGE_MAX_OUTPUT_TOKENS, } } fn validate(&self) -> Result<()> { + validate_judge_char_budget(self.judge_char_budget)?; if self.max_output_tokens == 0 { return Err(LibsyError::AlgorithmError { message: "max_output_tokens must be at least 1".to_string(), @@ -612,6 +745,7 @@ impl LlmTaskClassifier { StructuredJudge::new( TaskInput { recent_turn_window: config.recent_turn_window, + judge_char_budget: config.judge_char_budget, }, contract, SerdeDecoder::new(), @@ -640,6 +774,7 @@ impl LlmTaskClassifier { classify_trigger, message_hash_fallback, recent_turn_window, + judge_char_budget, max_output_tokens, } = config; let contract = ClassifierContract::from_inner_schema(&prompt, response_schema)?; @@ -650,7 +785,10 @@ impl LlmTaskClassifier { }; let classifier: Arc> = Arc::new(JudgeClassifier::new( StructuredJudge::new( - TaskInput { recent_turn_window }, + TaskInput { + recent_turn_window, + judge_char_budget, + }, contract, JsonSchemaDecoder::new(), JudgeRuntimeConfig::new(max_output_tokens)?, @@ -747,7 +885,7 @@ impl Algorithm for LlmTaskClassifier { #[cfg(test)] mod tests { - use std::collections::HashMap; + use std::collections::{BTreeSet, HashMap}; use std::sync::Arc; use parking_lot::Mutex; @@ -1316,8 +1454,19 @@ mod tests { /// The text of each message a judge with `recent_turn_window` would be sent. /// The no-window case is covered by `capability_judge_builds_a_structured_request`. fn capability_judge(recent_turn_window: Option) -> Result { + capability_judge_with_budget(recent_turn_window, DEFAULT_JUDGE_CHAR_BUDGET) + } + + /// A judge whose windowed payload is capped at `judge_char_budget` characters. + fn capability_judge_with_budget( + recent_turn_window: Option, + judge_char_budget: usize, + ) -> Result { Ok(StructuredJudge::new( - TaskInput { recent_turn_window }, + TaskInput { + recent_turn_window, + judge_char_budget, + }, LlmTaskClassifier::load_capability_contract(&ClassifierContractConfig::default())?, SerdeDecoder::new(), JudgeRuntimeConfig::new(DEFAULT_JUDGE_MAX_OUTPUT_TOKENS)?, @@ -1371,6 +1520,217 @@ mod tests { Ok(()) } + /// Builds a request whose windowed selection the judge will be handed. + fn windowed_request(messages: Vec) -> Request { + Request { + llm_request: LlmRequest { + messages, + ..LlmRequest::default() + }, + raw_request: None, + metadata: None, + } + } + + /// The messages a judge with this window and budget is actually sent. + fn budgeted_messages( + messages: Vec, + window: usize, + budget: usize, + ) -> Result> { + let judge = capability_judge_with_budget(Some(window), budget)?; + Ok(judge + .build_request(&State::default(), &windowed_request(messages)) + .llm_request + .messages) + } + + /// A window is counted in turns, so one turn carrying a large tool result would + /// otherwise decide the judge's cost for a fixed configuration. The window narrows + /// from the oldest end until the payload fits, so the newest evidence survives. + #[test] + fn an_oversized_turn_narrows_the_window_to_fit_the_budget() -> Result<()> { + let messages = vec![ + Message::text(Role::System, "client instructions"), + Message::text(Role::User, "initial task"), + Message::text(Role::Assistant, "x".repeat(20_000)), + Message::text(Role::User, "recent 1"), + Message::text(Role::Assistant, "recent 2"), + ]; + let built = budgeted_messages(messages, 4, 5_000)?; + let texts: Vec = built + .iter() + .filter_map(|message| message.text_content("\n")) + .collect(); + + assert!(payload_chars(&built) <= 5_000, "{}", payload_chars(&built)); + // The anchors and the newest turns stay; only the oversized older turn goes. + assert!(texts.contains(&"client instructions".to_string())); + assert!(texts.contains(&"initial task".to_string())); + assert!(texts.contains(&"recent 2".to_string())); + assert!(!texts.iter().any(|text| text.len() > 10_000), "{texts:?}"); + Ok(()) + } + + /// Reasoning is stripped before the judge is called, so a large reasoning block must not + /// narrow the window and cost the judge visible turns it could have kept. + #[test] + fn reasoning_does_not_count_against_the_budget() -> Result<()> { + let messages = vec![ + Message::text(Role::User, "initial task"), + Message { + role: Role::Assistant, + content: vec![ + ContentBlock::Reasoning { + text: "r".repeat(20_000), + signature: None, + details: Vec::new(), + }, + ContentBlock::Text { + text: "older answer".to_string(), + }, + ], + }, + Message::text(Role::User, "recent 1"), + ]; + let built = budgeted_messages(messages, 4, 5_000)?; + let texts: Vec = built + .iter() + .filter_map(|message| message.text_content("\n")) + .collect(); + + assert!(texts.contains(&"older answer".to_string()), "{texts:?}"); + Ok(()) + } + + /// Narrowing drops whole turns through `trim_messages`, so a surviving tool result + /// still has the call that introduced its id. Removing messages directly would not. + #[test] + fn narrowing_for_the_budget_keeps_tool_pairs_whole() -> Result<()> { + let mut bulky = tool_result("call-1"); + bulky.content = vec![ContentBlock::ToolResult(ToolResult { + tool_call_id: "call-1".to_string(), + content: vec![ContentBlock::Text { + text: "y".repeat(20_000), + }], + is_error: None, + })]; + let messages = vec![ + Message::text(Role::System, "client instructions"), + Message::text(Role::User, "initial task"), + tool_call("call-1"), + bulky, + tool_call("call-2"), + tool_result("call-2"), + ]; + let built = budgeted_messages(messages, 4, 5_000)?; + + assert!(payload_chars(&built) <= 5_000, "{}", payload_chars(&built)); + let calls: BTreeSet = built + .iter() + .flat_map(|message| &message.content) + .filter_map(|block| match block { + ContentBlock::ToolCall(call) => Some(call.id.clone()), + _ => None, + }) + .collect(); + for block in built.iter().flat_map(|message| &message.content) { + if let ContentBlock::ToolResult(result) = block { + assert!( + calls.contains(&result.tool_call_id), + "orphaned result {:?} in {calls:?}", + result.tool_call_id + ); + } + } + Ok(()) + } + + /// The anchors survive an empty window, so a task statement larger than the whole + /// budget cannot be dropped and has to be clipped instead. + #[test] + fn an_oversized_task_is_clipped_once_the_window_cannot_shrink() -> Result<()> { + let messages = vec![ + Message::text(Role::System, "client instructions"), + Message::text(Role::User, "z".repeat(40_000)), + Message::text(Role::Assistant, "recent"), + ]; + let built = budgeted_messages(messages, 2, 1_000)?; + let task = built + .iter() + .filter_map(|message| message.text_content("\n")) + .find(|text| text.starts_with('z')) + .ok_or_else(|| LibsyError::AlgorithmError { + message: "clipped task missing".to_string(), + })?; + + assert!(payload_chars(&built) <= 1_000, "{}", payload_chars(&built)); + // Clipping is marked, so the judge can tell a trimmed task from a short one. + assert!(task.contains("[trimmed]"), "{task}"); + Ok(()) + } + + /// The routing instruction is appended after selection, so the budget has to cover it: + /// a selection sized to the limit would otherwise ship a payload above the limit. + #[test] + fn the_budget_covers_the_appended_routing_instruction() -> Result<()> { + let messages = vec![ + Message::text(Role::System, "client instructions"), + Message::text(Role::User, "w".repeat(4_000)), + Message::text(Role::Assistant, "recent"), + ]; + let built = budgeted_messages(messages, 2, 600)?; + + // `built` includes the trailing instruction, so this measures the whole payload. + assert!( + built + .iter() + .any(|message| message.text_content("\n").as_deref() + == Some(TRAILING_ROUTING_INSTRUCTION)) + ); + assert!(payload_chars(&built) <= 600, "{}", payload_chars(&built)); + Ok(()) + } + + /// One message can carry several text blocks. A per-message allowance would let each + /// block spend it in full, so the share is per block. + #[test] + fn several_text_blocks_in_one_message_share_the_budget() -> Result<()> { + let crowded = Message { + role: Role::User, + content: vec![ + ContentBlock::Text { + text: "a".repeat(9_000), + }, + ContentBlock::Text { + text: "b".repeat(9_000), + }, + ], + }; + let messages = vec![ + Message::text(Role::System, "client instructions"), + crowded, + Message::text(Role::Assistant, "recent"), + ]; + let built = budgeted_messages(messages, 2, 900)?; + + assert!(payload_chars(&built) <= 900, "{}", payload_chars(&built)); + Ok(()) + } + + /// A budget below the minimum could not fit the routing instruction that every windowed + /// payload ends with, so it is rejected at construction. + #[test] + fn a_judge_char_budget_below_the_minimum_is_rejected() { + assert!(TRAILING_ROUTING_INSTRUCTION.chars().count() < MIN_JUDGE_CHAR_BUDGET); + let with_budget = |judge_char_budget| TaskClassifierConfig { + judge_char_budget, + ..TaskClassifierConfig::default() + }; + assert!(with_budget(MIN_JUDGE_CHAR_BUDGET - 1).validate().is_err()); + assert!(with_budget(MIN_JUDGE_CHAR_BUDGET).validate().is_ok()); + } + fn tool_call(id: &str) -> Message { Message { role: Role::Assistant, @@ -1546,6 +1906,7 @@ mod tests { let built = TaskInput { recent_turn_window: Some(10), + judge_char_budget: DEFAULT_JUDGE_CHAR_BUDGET, } .build_messages(&State::default(), &request); @@ -1700,6 +2061,7 @@ mod tests { let judge: CapabilityJudge = StructuredJudge::new( TaskInput { recent_turn_window: None, + judge_char_budget: DEFAULT_JUDGE_CHAR_BUDGET, }, contract, SerdeDecoder::new(), diff --git a/crates/libsy/src/algorithms/util.rs b/crates/libsy/src/algorithms/util.rs index 9a290920f..4bf7479b1 100644 --- a/crates/libsy/src/algorithms/util.rs +++ b/crates/libsy/src/algorithms/util.rs @@ -29,3 +29,37 @@ pub(crate) fn decisive(target: &ModelId) -> Classification { /// Default completion budget for internal classifier and escalation judge calls. pub(crate) const DEFAULT_JUDGE_MAX_OUTPUT_TOKENS: u64 = 4_096; + +/// Default character budget for a judge payload, shared by the escalation judge and the +/// windowed classifier judges so one turn carrying a large tool result cannot decide how +/// much a judge call costs. +pub(crate) const DEFAULT_JUDGE_CHAR_BUDGET: usize = 18_000; + +/// Separator marking where [`truncate_middle`] dropped a message's interior. +pub(crate) const TRIM_MARKER: &str = " ...[trimmed] "; + +/// Keeps the head and tail of `text` within `limit` characters. +/// +/// The head gets two thirds of the surviving budget: for a judge reading agent activity the +/// command or error signature that opens a message carries more signal than its trailing +/// output. Clipping is marked so the judge can tell a trimmed message from a short one. +pub(crate) fn truncate_middle(text: &str, limit: usize) -> String { + let chars: Vec = text.chars().collect(); + if chars.len() <= limit { + return text.to_string(); + } + // Below the marker's own width there is no room to say the text was clipped, so keep + // what fits and drop the marker. Marking anyway would push the result past `limit`, + // which callers budgeting a payload rely on it never doing. + let marker = TRIM_MARKER.chars().count(); + if limit <= marker { + return chars[..limit].iter().collect(); + } + let keep = limit - marker; + let head = keep * 2 / 3; + let tail = keep - head; + let mut out: String = chars[..head].iter().collect(); + out.push_str(TRIM_MARKER); + out.extend(chars[chars.len() - tail..].iter()); + out +} diff --git a/crates/libsy/src/algorithms/util/escalation.rs b/crates/libsy/src/algorithms/util/escalation.rs index b0979e286..7fe25900b 100644 --- a/crates/libsy/src/algorithms/util/escalation.rs +++ b/crates/libsy/src/algorithms/util/escalation.rs @@ -16,6 +16,7 @@ use super::llm_judge::{ ClassifierInput, JudgeClassifier, JudgePolicy, JudgeRuntimeConfig, SerdeDecoder, StructuredJudge, }; +use super::truncate_middle; use crate::core::algorithm::Driver; use crate::core::classifier::{Classification, Score}; use crate::core::state::State; @@ -25,9 +26,6 @@ use switchyard_protocol::Request; const PROMPT_TEMPLATE: &str = include_str!("../../prompts/escalation/prompt.md"); const SCHEMA_TEMPLATE: &str = include_str!("../../prompts/escalation/schema.json"); -/// Separator marking where [`truncate_middle`] dropped a message's interior. -const TRIM_MARKER: &str = " ...[trimmed] "; - /// Suffix marking a transcript cut off by [`MAX_REQUEST_CHARS`]. const TRUNCATION_SUFFIX: &str = "..."; @@ -239,27 +237,6 @@ fn collect_text(content: &[ContentBlock], parts: &mut Vec) { } } -/// Keeps the head and tail of `text` within `limit` characters. -/// -/// The head gets two thirds of the surviving budget: for a trajectory judge the command or -/// error signature that opens a message carries more signal than its trailing output. -fn truncate_middle(text: &str, limit: usize) -> String { - let chars: Vec = text.chars().collect(); - if chars.len() <= limit { - return text.to_string(); - } - let keep = limit - .saturating_sub(TRIM_MARKER.chars().count()) - .max(20) - .min(chars.len()); - let head = keep * 2 / 3; - let tail = keep - head; - let mut out: String = chars[..head].iter().collect(); - out.push_str(TRIM_MARKER); - out.extend(chars[chars.len() - tail..].iter()); - out -} - /// Renders a compact role-labelled transcript for the judge. /// /// The framing anchors — system/developer messages and the first user message, where agent @@ -496,6 +473,23 @@ mod tests { assert_eq!(truncate_middle("short", 50), "short"); } + /// Callers budget a payload on the promise that clipping never exceeds the limit. Below + /// the marker's own width there is no room to mark the cut, so the marker is dropped + /// rather than pushing the result over. + #[test] + fn truncate_middle_never_exceeds_a_limit_narrower_than_the_marker() { + use crate::algorithms::util::TRIM_MARKER; + + let text = "a".repeat(100); + for limit in 0..=TRIM_MARKER.chars().count() + 2 { + let trimmed = truncate_middle(&text, limit); + assert!( + trimmed.chars().count() <= limit, + "limit {limit}: {trimmed:?}" + ); + } + } + #[test] fn summary_keeps_anchors_and_the_recent_window() { let mut messages = vec![ diff --git a/crates/switchyard-py/src/libsy_bindings.rs b/crates/switchyard-py/src/libsy_bindings.rs index eea067f41..b8ac06846 100644 --- a/crates/switchyard-py/src/libsy_bindings.rs +++ b/crates/switchyard-py/src/libsy_bindings.rs @@ -168,6 +168,7 @@ impl PyCustomClassifierConfig { session_affinity=false, message_hash_fallback=false, recent_turn_window=None, + judge_char_budget=18_000, max_output_tokens=4096 ))] #[allow(clippy::too_many_arguments)] @@ -178,6 +179,7 @@ impl PyCustomClassifierConfig { session_affinity: bool, message_hash_fallback: bool, recent_turn_window: Option, + judge_char_budget: usize, max_output_tokens: u64, ) -> PyResult { // Convert the Python schema into serde JSON and pair it with the target-selector policy; @@ -190,6 +192,7 @@ impl PyCustomClassifierConfig { inner.classify_trigger = classify_trigger(session_affinity); inner.message_hash_fallback = message_hash_fallback; inner.recent_turn_window = recent_turn_window; + inner.judge_char_budget = judge_char_budget; inner.max_output_tokens = max_output_tokens; Ok(Self { inner }) } @@ -261,6 +264,7 @@ impl PyTaskClassifierConfig { session_affinity=false, message_hash_fallback=false, recent_turn_window=None, + judge_char_budget=18_000, max_output_tokens=4096, prompt=None, response_format_type="json_schema" @@ -272,6 +276,7 @@ impl PyTaskClassifierConfig { session_affinity: bool, message_hash_fallback: bool, recent_turn_window: Option, + judge_char_budget: usize, max_output_tokens: u64, prompt: Option, response_format_type: &str, @@ -283,6 +288,7 @@ impl PyTaskClassifierConfig { classify_trigger: classify_trigger(session_affinity), message_hash_fallback, recent_turn_window, + judge_char_budget, contract: classifier_contract(prompt, response_format_type)?, max_output_tokens, }, diff --git a/crates/switchyard-runner/src/algorithm.rs b/crates/switchyard-runner/src/algorithm.rs index 626c150af..a698ea2a8 100644 --- a/crates/switchyard-runner/src/algorithm.rs +++ b/crates/switchyard-runner/src/algorithm.rs @@ -106,6 +106,7 @@ struct CapabilityClassifierRouteConfig { classify_trigger: ClassifyTrigger, message_hash_fallback: bool, recent_turn_window: Option, + judge_char_budget: usize, prompt: Option, response_format_type: ClassifierResponseFormat, max_output_tokens: u64, @@ -132,6 +133,7 @@ struct CustomClassifierRouteConfig { classify_trigger: ClassifyTrigger, message_hash_fallback: bool, recent_turn_window: Option, + judge_char_budget: usize, max_output_tokens: u64, } @@ -245,6 +247,11 @@ pub struct LlmClassifierRouteConfig { /// How many trailing turns the judge sees. Unset shows it the opening task /// and the latest user follow-up only. pub recent_turn_window: Option, + /// Most characters a windowed judge payload may use. The window narrows from the + /// oldest turn until it fits, so one large tool result cannot decide the judge's + /// cost. Ignored without `recent_turn_window`. + #[serde(default = "default_judge_char_budget")] + pub judge_char_budget: usize, /// Replaces the packaged judge prompt. Required in custom mode. pub prompt: Option, /// How the judge is asked for structured output. Use `json_object` when the @@ -464,6 +471,9 @@ pub struct StageClassifierConfig { /// and the latest user follow-up only. #[serde(default)] pub recent_turn_window: Option, + /// Most characters a windowed judge payload may use. Ignored without a window. + #[serde(default = "default_judge_char_budget")] + pub judge_char_budget: usize, /// Replaces the packaged judge prompt. #[serde(default)] pub prompt: Option, @@ -508,6 +518,7 @@ impl StageClassifierConfig { classify_trigger: self.classify_trigger, message_hash_fallback: self.message_hash_fallback, recent_turn_window: self.recent_turn_window, + judge_char_budget: self.judge_char_budget, contract: classifier_contract(self.prompt.as_deref()) .with_response_format_type(self.response_format_type), max_output_tokens: self.max_output_tokens, @@ -844,6 +855,7 @@ impl LlmClassifierRouteConfig { classify_trigger, message_hash_fallback, recent_turn_window, + judge_char_budget, prompt, response_format_type, max_output_tokens, @@ -899,6 +911,7 @@ impl LlmClassifierRouteConfig { classify_trigger: *classify_trigger, message_hash_fallback: *message_hash_fallback, recent_turn_window: *recent_turn_window, + judge_char_budget: *judge_char_budget, prompt: prompt.clone(), response_format_type: *response_format_type, max_output_tokens: *max_output_tokens, @@ -994,6 +1007,7 @@ impl LlmClassifierRouteConfig { classify_trigger: *classify_trigger, message_hash_fallback: *message_hash_fallback, recent_turn_window: *recent_turn_window, + judge_char_budget: *judge_char_budget, max_output_tokens: *max_output_tokens, }, )) @@ -1079,6 +1093,7 @@ fn build_subagent_router_config( config.policy.into_libsy(), ); classifier_config.recent_turn_window = config.recent_turn_window; + classifier_config.judge_char_budget = config.judge_char_budget; classifier_config.max_output_tokens = config.max_output_tokens; let classifier = Arc::new( LlmTaskClassifier::new(LlmClassifierConfig::Custom { @@ -1176,6 +1191,7 @@ fn build_algorithm( classify_trigger: config.classify_trigger, message_hash_fallback: config.message_hash_fallback, recent_turn_window: config.recent_turn_window, + judge_char_budget: config.judge_char_budget, contract: classifier_contract(config.prompt.as_deref()) .with_response_format_type(config.response_format_type), max_output_tokens: config.max_output_tokens, @@ -1211,6 +1227,7 @@ fn build_algorithm( classifier_config.classify_trigger = config.classify_trigger; classifier_config.message_hash_fallback = config.message_hash_fallback; classifier_config.recent_turn_window = config.recent_turn_window; + classifier_config.judge_char_budget = config.judge_char_budget; classifier_config.max_output_tokens = config.max_output_tokens; LlmTaskClassifier::new(LlmClassifierConfig::Custom { default_target: config.default_target, @@ -1417,6 +1434,10 @@ fn default_classifier_max_output_tokens() -> u64 { TaskClassifierConfig::default().max_output_tokens } +fn default_judge_char_budget() -> usize { + TaskClassifierConfig::default().judge_char_budget +} + fn resolve_target_model_id( route_name: &str, name: &str, diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md index 7d4b8b914..6834f50db 100644 --- a/docs/reference/toml_schema.md +++ b/docs/reference/toml_schema.md @@ -202,6 +202,7 @@ Capability mode classifies before serving. See | `classify_trigger` | No | `every_request` | When the judge runs. `every_request` judges every request, tool continuations included. `user_turn` judges each new user message and retains that target across intervening tool calls only when requests carry a session ID; without a session ID, it behaves like `every_request`. `new_session` judges once and reuses that target for the session. | | `message_hash_fallback` | No | `false` | Keys affinity on the first user message. Requires `classify_trigger = "new_session"`. | | `recent_turn_window` | No | unset | When unset, the judge sees the opening task and latest user follow-up, when present. When set, it also sees trailing turns. | +| `judge_char_budget` | No | `18000` | Most characters a windowed judge payload may use. The window narrows from the oldest turn until it fits, so one large tool result cannot decide judge cost and latency. Must be at least `256`. Ignored without `recent_turn_window`. | | `prompt` | No | packaged prompt | Replaces the capability prompt. The packaged schema is sent separately as structured-output configuration. | Escalation mode serves the weak target first and judges the completed turn. See @@ -242,6 +243,7 @@ how one route chooses between more than two models. | `classify_trigger` | No | `every_request` | When the judge runs. `every_request` judges every request, tool continuations included. `user_turn` judges each new user message and retains that target across intervening tool calls only when requests carry a session ID; without a session ID, it behaves like `every_request`. `new_session` judges once and reuses that target for the session. | | `message_hash_fallback` | No | `false` | Keys affinity on the first user message. Requires `classify_trigger = "new_session"`. | | `recent_turn_window` | No | unset | When unset, the judge sees the opening task and latest user follow-up, when present. When set, it also sees trailing turns. | +| `judge_char_budget` | No | `18000` | Most characters a windowed judge payload may use. The window narrows from the oldest turn until it fits, so one large tool result cannot decide judge cost and latency. Must be at least `256`. Ignored without `recent_turn_window`. | The selected JSON label must name a configured group. A label naming a target rather than a group, or a group you did not configure, falls back to diff --git a/docs/routing_algorithms/llm_classifier_routing.md b/docs/routing_algorithms/llm_classifier_routing.md index cbc893563..289546f2e 100644 --- a/docs/routing_algorithms/llm_classifier_routing.md +++ b/docs/routing_algorithms/llm_classifier_routing.md @@ -106,6 +106,7 @@ for the server merge behavior. | `base_threshold` | required | Lowest `p_solve` that routes a supported task to `weak_target`. Must be between `0` and `1`. | | `threshold_step` | `0.0` | Amount added for each boundary step. Must be finite and non-negative, and `base_threshold + 2 * threshold_step` must not exceed `1`. | | `recent_turn_window` | unset | When unset, the judge sees the opening user task and the latest user message when they differ. When set to `N`, it sees the opening user task and the last `N` conversation messages after that task. `0` keeps only the opening task. Client system and developer instructions are not shown to the judge. | +| `judge_char_budget` | `18000` | Most characters the windowed selection may use. A window is counted in turns, and one turn carrying a large tool result can be worth tens of thousands of characters, so the window narrows from the oldest turn until the payload fits. A task statement larger than the whole budget is clipped instead, marked with `...[trimmed]`. Must be at least `256`. Ignored when `recent_turn_window` is unset. | | `classify_trigger` | `every_request` | When the judge runs. `every_request` judges every request, tool continuations included. `user_turn` judges each new user message and holds that target across the tool calls between. `new_session` judges once and reuses that target for the session. | | `message_hash_fallback` | `false` | When session metadata is absent, keys affinity from the first user-message text. Requires `classify_trigger = "new_session"`. | | `prompt` | packaged capability prompt | Replaces the classifier's system prompt. The packaged verdict schema and routing policy remain active. | diff --git a/switchyard_rust/libsy.py b/switchyard_rust/libsy.py index fd768ba71..45f378682 100644 --- a/switchyard_rust/libsy.py +++ b/switchyard_rust/libsy.py @@ -63,7 +63,8 @@ class CustomClassifierConfig: """Configure schema-validated routing across runtime model groups. ``max_output_tokens`` must be positive. Enabling ``message_hash_fallback`` - requires ``session_affinity``. + requires ``session_affinity``. ``judge_char_budget`` caps the windowed judge + payload, must be at least 256, and is ignored without ``recent_turn_window``. """ def __init__( @@ -75,6 +76,7 @@ def __init__( session_affinity: bool = False, message_hash_fallback: bool = False, recent_turn_window: int | None = None, + judge_char_budget: int = 18_000, max_output_tokens: int = 4096, ) -> None: ... @@ -156,6 +158,8 @@ class TaskClassifierConfig: Thresholds must remain within ``[0, 1]``, ``max_output_tokens`` must be positive, and ``message_hash_fallback`` requires ``session_affinity``. + ``judge_char_budget`` caps the windowed judge payload, must be at least 256, + and is ignored without ``recent_turn_window``. """ def __init__( @@ -166,6 +170,7 @@ def __init__( session_affinity: bool = False, message_hash_fallback: bool = False, recent_turn_window: int | None = None, + judge_char_budget: int = 18_000, max_output_tokens: int = 4096, prompt: str | None = None, response_format_type: Literal["json_schema", "json_object"] = "json_schema",