diff --git a/CHANGELOG.md b/CHANGELOG.md index 76884e283..eb580239a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -72,6 +72,13 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Changed +- **Graded custom-classifier verdicts** — the `target_selector` policy takes an + optional `labels` map from verdict value to configured target, so a rubric can + grade a request (`simple` / `complex`) instead of naming a model and several + grades may share one target. Omitting `labels` keeps today's behavior, but when + it is set it is the whole verdict vocabulary and a verdict outside it falls back + to `default_target`. `CustomClassifierPolicy::TargetSelector` gains a field, so + external code matching that variant by its fields must be updated. (#348) - **`Algorithm::route` returns `Result`** — instead of the bare final `Result`, so callers observe the full routing outcome (see #458 for the design). (#459) diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs index 599e9f9e0..00e6f911c 100644 --- a/crates/libsy/src/algorithms/llm_class.rs +++ b/crates/libsy/src/algorithms/llm_class.rs @@ -381,6 +381,12 @@ pub enum CustomClassifierPolicy { TargetSelector { /// JSON Pointer evaluated against each schema-validated verdict. selector: String, + /// Maps each verdict value the judge may return onto a configured target label. + /// + /// `None` keeps the direct lookup, where the verdict is itself a target label. + /// When set, this map is the whole verdict vocabulary: several verdicts may share + /// one target, and a verdict outside the map falls back to `default_target`. + labels: Option>, }, } @@ -389,6 +395,7 @@ impl CustomClassifierPolicy { pub fn target_selector(selector: impl Into) -> Self { Self::TargetSelector { selector: selector.into(), + labels: None, } } } @@ -678,9 +685,38 @@ impl LlmTaskClassifier { } = config; let contract = ClassifierContract::from_inner_schema(&prompt, response_schema)?; let policy = match policy { - CustomClassifierPolicy::TargetSelector { selector } => { + CustomClassifierPolicy::TargetSelector { + selector, + labels: verdict_labels, + } => { + // Without `labels` a verdict is a target label. With them, the map is the + // whole vocabulary, so several grades may resolve to one target even though + // the target list itself still rejects a repeated model. + let verdict_map = match verdict_labels { + None => target_map, + Some(verdict_labels) if verdict_labels.is_empty() => { + return Err(LibsyError::AlgorithmError { + message: "custom classifier policy labels must not be empty" + .to_string(), + }); + } + Some(verdict_labels) => verdict_labels + .into_iter() + .map(|(verdict, label)| { + let target = target_map.get(&label).cloned().ok_or_else(|| { + LibsyError::AlgorithmError { + message: format!( + "custom classifier policy label {label:?} must be one of the configured targets" + ), + } + })?; + Ok((verdict, target)) + }) + .collect::>>()?, + }; CustomPolicyRuntime::TargetSelector(TargetSelectorPolicy::new( - selector, target_map, + selector, + verdict_map, )?) } }; @@ -1649,4 +1685,65 @@ mod tests { ); Ok(()) } + + fn custom_config_with_labels(labels: Option>) -> LlmClassifierConfig { + LlmClassifierConfig::Custom { + judge_target: ModelId::from("judge"), + targets: vec![ + ("worker".to_string(), ModelId::from("worker")), + ("reviewer".to_string(), ModelId::from("reviewer")), + ], + default_target: "worker".to_string(), + config: CustomClassifierConfig::new( + "classify the delegated task", + serde_json::json!({ + "type": "object", + "properties": { + "grade": {"type": "string", "enum": ["simple", "hard", "brutal"]} + }, + "required": ["grade"], + "additionalProperties": false + }), + CustomClassifierPolicy::TargetSelector { + selector: "/grade".to_string(), + labels, + }, + ), + } + } + + #[test] + fn graded_verdict_labels_may_share_one_routing_target() { + // Two grades routing to one target is the whole point of a rubric verdict: the + // target list still rejects a repeated model, but the verdict vocabulary must not. + let config = custom_config_with_labels(Some(BTreeMap::from([ + ("simple".to_string(), "worker".to_string()), + ("hard".to_string(), "reviewer".to_string()), + ("brutal".to_string(), "reviewer".to_string()), + ]))); + + assert!(LlmTaskClassifier::new(config).is_ok()); + } + + #[test] + fn graded_verdict_labels_are_checked_against_the_configured_targets() { + let empty = LlmTaskClassifier::new(custom_config_with_labels(Some(BTreeMap::new()))); + assert!( + matches!(&empty, Err(LibsyError::AlgorithmError { message }) + if message.contains("labels must not be empty")), + "{:?}", + empty.err() + ); + + let unknown = LlmTaskClassifier::new(custom_config_with_labels(Some(BTreeMap::from([( + "simple".to_string(), + "nope".to_string(), + )])))); + assert!( + matches!(&unknown, Err(LibsyError::AlgorithmError { message }) + if message.contains("policy label \"nope\"")), + "{:?}", + unknown.err() + ); + } } diff --git a/crates/libsy/src/algorithms/util/target_selector.rs b/crates/libsy/src/algorithms/util/target_selector.rs index 4e36b648a..6f32dc812 100644 --- a/crates/libsy/src/algorithms/util/target_selector.rs +++ b/crates/libsy/src/algorithms/util/target_selector.rs @@ -42,16 +42,20 @@ impl JudgePolicy for TargetSelectorPolicy { type Verdict = Value; fn to_classification(&self, verdict: Option<&Self::Verdict>) -> Classification { - let target = verdict + let label = verdict .and_then(|verdict| self.selector.resolve(verdict).ok()) - .and_then(Value::as_str) - .and_then(|label| self.targets.get(label)); - match target { + .and_then(Value::as_str); + match label.and_then(|label| self.targets.get(label)) { Some(target) => Classification::Scores(vec![Score { target: target.clone(), confidence: 1.0, }]), - None => Classification::Ambiguous(vec![]), + None => { + // Abstaining hands the request to `default_target`. Say which value missed: + // a verdict outside a configured `labels` map is the likeliest misconfiguration. + tracing::warn!(?label, "verdict is not a configured target or label"); + Classification::Ambiguous(vec![]) + } } } } diff --git a/crates/switchyard-runner/src/algorithm.rs b/crates/switchyard-runner/src/algorithm.rs index d3e5f78ac..6f5282c1c 100644 --- a/crates/switchyard-runner/src/algorithm.rs +++ b/crates/switchyard-runner/src/algorithm.rs @@ -65,6 +65,10 @@ pub enum ClassifierPolicyConfig { TargetSelector { /// JSON Pointer to the name, such as `/decision/target`. selector: String, + /// Maps each verdict value onto a configured target name. Omit it and the verdict + /// is the target name; set it and several verdicts may share one target. + #[serde(default)] + labels: Option>, }, } @@ -84,7 +88,9 @@ pub enum ClassifierMode { impl ClassifierPolicyConfig { fn into_libsy(self) -> CustomClassifierPolicy { match self { - Self::TargetSelector { selector } => CustomClassifierPolicy::target_selector(selector), + Self::TargetSelector { selector, labels } => { + CustomClassifierPolicy::TargetSelector { selector, labels } + } } } } diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rs index c1fea1464..a4f138780 100644 --- a/crates/switchyard-runner/src/config.rs +++ b/crates/switchyard-runner/src/config.rs @@ -662,6 +662,17 @@ classify_trigger = "new_session""#, configured } + /// The subagent custom classifier above, with a graded `policy.labels` map added. + fn with_subagent_classifier_labels(labels: &str) -> String { + let configured = with_subagent_llm_classifier(VALID_CONFIG, "passthrough", ""); + let policy = "selector = \"/target\" }"; + assert!(configured.contains(policy)); + configured.replace( + policy, + &format!("selector = \"/target\", labels = {labels} }}"), + ) + } + fn with_subagent_passthrough(config: &str, route: &str) -> String { format!("{config}\n[routes.{route}.subagents]\ntype = \"passthrough\"\ntarget = \"strong\"") } @@ -1058,6 +1069,14 @@ classifier_magic = true ), "route random context_window must be greater than zero", ), + ( + with_subagent_classifier_labels("{}"), + "labels must not be empty", + ), + ( + with_subagent_classifier_labels("{ simple = \"nope\" }"), + "policy label \"nope\"", + ), ]; for (toml, expected) in cases { @@ -1069,6 +1088,15 @@ classifier_magic = true } } + #[test] + fn accepts_graded_classifier_policy_labels() -> RunnerResult<()> { + // Two verdict grades may name the same target; only the target list stays unique. + runner_from_toml(&with_subagent_classifier_labels( + "{ simple = \"weak\", complex = \"strong\" }", + ))?; + Ok(()) + } + #[test] fn accepts_duplicate_target_model_ids_on_one_client() -> RunnerResult<()> { // Two targets share one model id on one client. The client keeps one and drops the diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs index 43e6abf7f..796beb02b 100644 --- a/crates/switchyard-server/tests/server.rs +++ b/crates/switchyard-server/tests/server.rs @@ -1513,6 +1513,119 @@ selector = "/decision/target" Ok(()) } +#[tokio::test] +async fn custom_classifier_maps_graded_verdict_labels_onto_targets() -> TestResult { + let upstream = MockUpstream::start().await?; + let state = load_test_config(&format!( + r#" +schema_version = 1 + +[llm_clients.upstream] +format = "openai_chat" +base_url = "{base_url}" + +[targets.classifier] +id = "model/classifier" +llm_client = "upstream" + +[targets.strong] +id = "model/strong" +llm_client = "upstream" + +[targets.middle] +id = "model/middle" +llm_client = "upstream" + +[targets.premium] +id = "model/premium" +llm_client = "upstream" + +[targets.weak] +id = "model/weak" +llm_client = "upstream" + +[routes.custom] +id = "switchyard/custom" +type = "llm_classifier" +mode = "custom" +classifier_target = "classifier" +targets = ["weak", "strong"] +default_target = "weak" +prompt = "CUSTOM MULTI TARGET" +response_schema = ''' +{{ + "type": "object", + "properties": {{ + "decision": {{ + "type": "object", + "properties": {{ + "target": {{"type": "string", "enum": ["weak", "middle", "strong", "premium"]}} + }}, + "required": ["target"], + "additionalProperties": false + }} + }}, + "required": ["decision"], + "additionalProperties": false +}} +''' + +[routes.custom.policy] +type = "target_selector" +selector = "/decision/target" +labels = {{ premium = "strong" }} +"#, + base_url = upstream.base_url + ))?; + let app = build_switchyard_router(state); + + for (task, selected) in [ + ("route this task", "model/strong"), + ("return an invalid verdict", "model/weak"), + ] { + let response = send( + &app, + "POST", + "/v1/chat/completions", + Some(json!({ + "model": "switchyard/custom", + "messages": [{"role": "user", "content": task}] + })), + ) + .await?; + assert_eq!(response.status, StatusCode::OK); + assert_eq!( + response + .headers + .get("x-model-router-selected-model") + .and_then(|value| value.to_str().ok()), + Some(selected) + ); + } + + let calls = upstream.calls.lock().await; + let judge_call = calls + .iter() + .find(|call| call["model"] == "model/classifier") + .ok_or("custom classifier target was not called")?; + let prompt = judge_call["messages"][0]["content"] + .as_str() + .ok_or("custom classifier prompt was not text")?; + assert_eq!(prompt, "CUSTOM MULTI TARGET"); + assert_eq!(judge_call["response_format"]["type"], "json_schema"); + assert_eq!( + judge_call["response_format"]["json_schema"]["name"], + "switchyard_classifier_response" + ); + assert_eq!(judge_call["response_format"]["json_schema"]["strict"], true); + assert_eq!( + judge_call["response_format"]["json_schema"]["schema"]["properties"]["decision"]["properties"] + ["target"]["enum"], + json!(["weak", "middle", "strong", "premium"]) + ); + Ok(()) +} + #[tokio::test] async fn classifier_contract_overrides_reach_every_server_mode() -> TestResult { let upstream = MockUpstream::start().await?; diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md index 0193ca680..ee0c0a247 100644 --- a/docs/reference/toml_schema.md +++ b/docs/reference/toml_schema.md @@ -215,6 +215,7 @@ policy selector, and routes to any configured target label. | `prompt` | Yes | — | Judge system prompt. The configured inner schema is sent separately as structured-output configuration. | | `response_schema` | Yes | — | Inner JSON Schema encoded as a TOML string. Switchyard adds the provider wrapper. | | `policy` | Yes | — | Policy table. `target_selector` accepts a JSON Pointer such as `/decision/target`. | +| `policy.labels` | No | unset | Maps each verdict value onto a configured target name, so several verdicts may share one target. Every value must name a configured target and the table must not be empty. When set it is the whole verdict vocabulary; a verdict outside it falls back to `default_target`. | | `classify_trigger` | No | `every_request` | When the judge runs. `every_request` judges every request, tool continuations included. `user_turn` judges each new user message and retains that target across intervening tool calls only when requests carry a session ID; without a session ID, it behaves like `every_request`. `new_session` judges once and reuses that target for the session. | | `message_hash_fallback` | No | `false` | Keys affinity on the first user message. Requires `classify_trigger = "new_session"`. | | `recent_turn_window` | No | unset | When unset, the judge sees the opening task and latest user follow-up, when present. When set, it also sees trailing turns. | diff --git a/docs/routing_algorithms/llm_classifier_routing.md b/docs/routing_algorithms/llm_classifier_routing.md index 4706175e3..ce459a2af 100644 --- a/docs/routing_algorithms/llm_classifier_routing.md +++ b/docs/routing_algorithms/llm_classifier_routing.md @@ -184,6 +184,53 @@ schema to the provider in a strict structured-output wrapper and validates the returned JSON again. `jsonptr` resolves the selector against that verdict. A missing, non-string, or unknown target falls back to `default_target`. +### Grading rubrics with `policy.labels` + +A rubric that asks for a judgement — difficulty, complexity, risk — rather than +a target name routes through an optional `labels` map. Several grades may share +one target, which `targets` cannot express because two entries resolving to one +model are rejected: + +```toml +[routes.assistant] +id = "assistant" +type = "llm_classifier" +mode = "custom" +classifier_target = "classifier" +targets = ["weak", "strong"] +default_target = "strong" +prompt = """ +Grade this request's difficulty. +Return JSON matching the response schema supplied with the request. +""" +response_schema = ''' +{ + "type": "object", + "properties": { + "recommended_tier": { + "type": "string", + "enum": ["simple", "medium", "complex", "reasoning"] + } + }, + "required": ["recommended_tier"], + "additionalProperties": false +} +''' + +[routes.assistant.policy] +type = "target_selector" +selector = "/recommended_tier" +labels = { simple = "weak", medium = "weak", complex = "strong", reasoning = "strong" } +``` + +Every value in `labels` must name a configured target and the table must not be +empty; both are checked when the deployment loads. One rubric then serves +several operating points, because only the map changes between them. + +With `labels` set, `response_schema` must enumerate the **verdict** values, not +the target names. Nothing validates that pairing, and getting it wrong makes +every verdict unroutable, so all traffic silently reaches `default_target`. + This separation applies to every classifier mode. Prompts containing the legacy `{{RESPONSE_SCHEMA}}` placeholder are rejected during configuration validation.