feat(libsy): map graded classifier verdicts onto routing targets (#348)

CHANGELOG.md

@@ -72,6 +72,13 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ### Changed
 
+- **Graded custom-classifier verdicts** — the `target_selector` policy takes an
+  optional `labels` map from verdict value to configured target, so a rubric can
+  grade a request (`simple` / `complex`) instead of naming a model and several
+  grades may share one target. Omitting `labels` keeps today's behavior, but when
+  it is set it is the whole verdict vocabulary and a verdict outside it falls back
+  to `default_target`. `CustomClassifierPolicy::TargetSelector` gains a field, so
+  external code matching that variant by its fields must be updated. (#348)
 - **`Algorithm::route` returns `Result<RoutingOutcome>`** — instead of the
   bare final `Result`, so callers observe the full routing outcome (see #458
   for the design). (#459)

crates/libsy/src/algorithms/llm_class.rs

@@ -381,6 +381,12 @@ pub enum CustomClassifierPolicy {
     TargetSelector {
         /// JSON Pointer evaluated against each schema-validated verdict.
         selector: String,
+        /// Maps each verdict value the judge may return onto a configured target label.
+        ///
+        /// `None` keeps the direct lookup, where the verdict is itself a target label.
+        /// When set, this map is the whole verdict vocabulary: several verdicts may share
+        /// one target, and a verdict outside the map falls back to `default_target`.
+        labels: Option<BTreeMap<String, String>>,
     },
 }
 
@@ -389,6 +395,7 @@ impl CustomClassifierPolicy {
     pub fn target_selector(selector: impl Into<String>) -> Self {
         Self::TargetSelector {
             selector: selector.into(),
+            labels: None,
         }
     }
 }
@@ -678,9 +685,38 @@ impl LlmTaskClassifier {
         } = config;
         let contract = ClassifierContract::from_inner_schema(&prompt, response_schema)?;
         let policy = match policy {
-            CustomClassifierPolicy::TargetSelector { selector } => {
+            CustomClassifierPolicy::TargetSelector {
+                selector,
+                labels: verdict_labels,
+            } => {
+                // Without `labels` a verdict is a target label. With them, the map is the
+                // whole vocabulary, so several grades may resolve to one target even though
+                // the target list itself still rejects a repeated model.
+                let verdict_map = match verdict_labels {
+                    None => target_map,
+                    Some(verdict_labels) if verdict_labels.is_empty() => {
+                        return Err(LibsyError::AlgorithmError {
+                            message: "custom classifier policy labels must not be empty"
+                                .to_string(),
+                        });
+                    }
+                    Some(verdict_labels) => verdict_labels
+                        .into_iter()
+                        .map(|(verdict, label)| {
+                            let target = target_map.get(&label).cloned().ok_or_else(|| {
+                                LibsyError::AlgorithmError {
+                                    message: format!(
+                                        "custom classifier policy label {label:?} must be one of the configured targets"
+                                    ),
+                                }
+                            })?;
+                            Ok((verdict, target))
+                        })
+                        .collect::<Result<BTreeMap<_, _>>>()?,
+                };
                 CustomPolicyRuntime::TargetSelector(TargetSelectorPolicy::new(
-                    selector, target_map,
+                    selector,
+                    verdict_map,
                 )?)
             }
         };
@@ -1649,4 +1685,65 @@ mod tests {
         );
         Ok(())
     }
+
+    fn custom_config_with_labels(labels: Option<BTreeMap<String, String>>) -> LlmClassifierConfig {
+        LlmClassifierConfig::Custom {
+            judge_target: ModelId::from("judge"),
+            targets: vec![
+                ("worker".to_string(), ModelId::from("worker")),
+                ("reviewer".to_string(), ModelId::from("reviewer")),
+            ],
+            default_target: "worker".to_string(),
+            config: CustomClassifierConfig::new(
+                "classify the delegated task",
+                serde_json::json!({
+                    "type": "object",
+                    "properties": {
+                        "grade": {"type": "string", "enum": ["simple", "hard", "brutal"]}
+                    },
+                    "required": ["grade"],
+                    "additionalProperties": false
+                }),
+                CustomClassifierPolicy::TargetSelector {
+                    selector: "/grade".to_string(),
+                    labels,
+                },
+            ),
+        }
+    }
+
+    #[test]
+    fn graded_verdict_labels_may_share_one_routing_target() {
+        // Two grades routing to one target is the whole point of a rubric verdict: the
+        // target list still rejects a repeated model, but the verdict vocabulary must not.
+        let config = custom_config_with_labels(Some(BTreeMap::from([
+            ("simple".to_string(), "worker".to_string()),
+            ("hard".to_string(), "reviewer".to_string()),
+            ("brutal".to_string(), "reviewer".to_string()),
+        ])));
+
+        assert!(LlmTaskClassifier::new(config).is_ok());
+    }
+
+    #[test]
+    fn graded_verdict_labels_are_checked_against_the_configured_targets() {
+        let empty = LlmTaskClassifier::new(custom_config_with_labels(Some(BTreeMap::new())));
+        assert!(
+            matches!(&empty, Err(LibsyError::AlgorithmError { message })
+                if message.contains("labels must not be empty")),
+            "{:?}",
+            empty.err()
+        );
+
+        let unknown = LlmTaskClassifier::new(custom_config_with_labels(Some(BTreeMap::from([(
+            "simple".to_string(),
+            "nope".to_string(),
+        )]))));
+        assert!(
+            matches!(&unknown, Err(LibsyError::AlgorithmError { message })
+                if message.contains("policy label \"nope\"")),
+            "{:?}",
+            unknown.err()
+        );
+    }
 }

crates/libsy/src/algorithms/util/target_selector.rs

@@ -42,16 +42,20 @@ impl JudgePolicy for TargetSelectorPolicy {
     type Verdict = Value;
 
     fn to_classification(&self, verdict: Option<&Self::Verdict>) -> Classification {
-        let target = verdict
+        let label = verdict
             .and_then(|verdict| self.selector.resolve(verdict).ok())
-            .and_then(Value::as_str)
-            .and_then(|label| self.targets.get(label));
-        match target {
+            .and_then(Value::as_str);
+        match label.and_then(|label| self.targets.get(label)) {
             Some(target) => Classification::Scores(vec![Score {
                 target: target.clone(),
                 confidence: 1.0,
             }]),
-            None => Classification::Ambiguous(vec![]),
+            None => {
+                // Abstaining hands the request to `default_target`. Say which value missed:
+                // a verdict outside a configured `labels` map is the likeliest misconfiguration.
+                tracing::warn!(?label, "verdict is not a configured target or label");
+                Classification::Ambiguous(vec![])
+            }
         }
     }
 }

crates/switchyard-runner/src/algorithm.rs

@@ -65,6 +65,10 @@ pub enum ClassifierPolicyConfig {
     TargetSelector {
         /// JSON Pointer to the name, such as `/decision/target`.
         selector: String,
+        /// Maps each verdict value onto a configured target name. Omit it and the verdict
+        /// is the target name; set it and several verdicts may share one target.
+        #[serde(default)]
+        labels: Option<BTreeMap<String, String>>,
     },
 }
 
@@ -84,7 +88,9 @@ pub enum ClassifierMode {
 impl ClassifierPolicyConfig {
     fn into_libsy(self) -> CustomClassifierPolicy {
         match self {
-            Self::TargetSelector { selector } => CustomClassifierPolicy::target_selector(selector),
+            Self::TargetSelector { selector, labels } => {
+                CustomClassifierPolicy::TargetSelector { selector, labels }
+            }
         }
     }
 }

crates/switchyard-runner/src/config.rs

@@ -662,6 +662,17 @@ classify_trigger = "new_session""#,
         configured
     }
 
+    /// The subagent custom classifier above, with a graded `policy.labels` map added.
+    fn with_subagent_classifier_labels(labels: &str) -> String {
+        let configured = with_subagent_llm_classifier(VALID_CONFIG, "passthrough", "");
+        let policy = "selector = \"/target\" }";
+        assert!(configured.contains(policy));
+        configured.replace(
+            policy,
+            &format!("selector = \"/target\", labels = {labels} }}"),
+        )
+    }
+
     fn with_subagent_passthrough(config: &str, route: &str) -> String {
         format!("{config}\n[routes.{route}.subagents]\ntype = \"passthrough\"\ntarget = \"strong\"")
     }
@@ -1058,6 +1069,14 @@ classifier_magic = true
                 ),
                 "route random context_window must be greater than zero",
             ),
+            (
+                with_subagent_classifier_labels("{}"),
+                "labels must not be empty",
+            ),
+            (
+                with_subagent_classifier_labels("{ simple = \"nope\" }"),
+                "policy label \"nope\"",
+            ),
         ];
 
         for (toml, expected) in cases {
@@ -1069,6 +1088,15 @@ classifier_magic = true
         }
     }
 
+    #[test]
+    fn accepts_graded_classifier_policy_labels() -> RunnerResult<()> {
+        // Two verdict grades may name the same target; only the target list stays unique.
+        runner_from_toml(&with_subagent_classifier_labels(
+            "{ simple = \"weak\", complex = \"strong\" }",
+        ))?;
+        Ok(())
+    }
+
     #[test]
     fn accepts_duplicate_target_model_ids_on_one_client() -> RunnerResult<()> {
         // Two targets share one model id on one client. The client keeps one and drops the

crates/switchyard-server/tests/server.rs

@@ -1513,6 +1513,119 @@ selector = "/decision/target"
     Ok(())
 }
 
+#[tokio::test]
+async fn custom_classifier_maps_graded_verdict_labels_onto_targets() -> TestResult {
+    let upstream = MockUpstream::start().await?;
+    let state = load_test_config(&format!(
+        r#"
+schema_version = 1
+
+[llm_clients.upstream]
+format = "openai_chat"
+base_url = "{base_url}"
+
+[targets.classifier]
+id = "model/classifier"
+llm_client = "upstream"
+
+[targets.strong]
+id = "model/strong"
+llm_client = "upstream"
+
+[targets.middle]
+id = "model/middle"
+llm_client = "upstream"
+
+[targets.premium]
+id = "model/premium"
+llm_client = "upstream"
+
+[targets.weak]
+id = "model/weak"
+llm_client = "upstream"
+
+[routes.custom]
+id = "switchyard/custom"
+type = "llm_classifier"
+mode = "custom"
+classifier_target = "classifier"
+targets = ["weak", "strong"]
+default_target = "weak"
+prompt = "CUSTOM MULTI TARGET"
+response_schema = '''
+{{
+  "type": "object",
+  "properties": {{
+    "decision": {{
+      "type": "object",
+      "properties": {{
+        "target": {{"type": "string", "enum": ["weak", "middle", "strong", "premium"]}}
+      }},
+      "required": ["target"],
+      "additionalProperties": false
+    }}
+  }},
+  "required": ["decision"],
+  "additionalProperties": false
+}}
+'''
+
+[routes.custom.policy]
+type = "target_selector"
+selector = "/decision/target"
+labels = {{ premium = "strong" }}
+"#,
+        base_url = upstream.base_url
+    ))?;
+    let app = build_switchyard_router(state);
+
+    for (task, selected) in [
+        ("route this task", "model/strong"),
+        ("return an invalid verdict", "model/weak"),
+    ] {
+        let response = send(
+            &app,
+            "POST",
+            "/v1/chat/completions",
+            Some(json!({
+                "model": "switchyard/custom",
+                "messages": [{"role": "user", "content": task}]
+            })),
+        )
+        .await?;
+        assert_eq!(response.status, StatusCode::OK);
+        assert_eq!(
+            response
+                .headers
+                .get("x-model-router-selected-model")
+                .and_then(|value| value.to_str().ok()),
+            Some(selected)
+        );
+    }
+
+    let calls = upstream.calls.lock().await;
+    let judge_call = calls
+        .iter()
+        .find(|call| call["model"] == "model/classifier")
+        .ok_or("custom classifier target was not called")?;
+    let prompt = judge_call["messages"][0]["content"]
+        .as_str()
+        .ok_or("custom classifier prompt was not text")?;
+    assert_eq!(prompt, "CUSTOM MULTI TARGET");
+    assert_eq!(judge_call["response_format"]["type"], "json_schema");
+    assert_eq!(
+        judge_call["response_format"]["json_schema"]["name"],
+        "switchyard_classifier_response"
+    );
+    assert_eq!(judge_call["response_format"]["json_schema"]["strict"], true);
+    assert_eq!(
+        judge_call["response_format"]["json_schema"]["schema"]["properties"]["decision"]["properties"]
+            ["target"]["enum"],
+        json!(["weak", "middle", "strong", "premium"])
+    );
+    Ok(())
+}
+
 #[tokio::test]
 async fn classifier_contract_overrides_reach_every_server_mode() -> TestResult {
     let upstream = MockUpstream::start().await?;

docs/reference/toml_schema.md

@@ -215,6 +215,7 @@ policy selector, and routes to any configured target label.
 | `prompt` | Yes | — | Judge system prompt. The configured inner schema is sent separately as structured-output configuration. |
 | `response_schema` | Yes | — | Inner JSON Schema encoded as a TOML string. Switchyard adds the provider wrapper. |
 | `policy` | Yes | — | Policy table. `target_selector` accepts a JSON Pointer such as `/decision/target`. |
+| `policy.labels` | No | unset | Maps each verdict value onto a configured target name, so several verdicts may share one target. Every value must name a configured target and the table must not be empty. When set it is the whole verdict vocabulary; a verdict outside it falls back to `default_target`. |
 | `classify_trigger` | No | `every_request` | When the judge runs. `every_request` judges every request, tool continuations included. `user_turn` judges each new user message and retains that target across intervening tool calls only when requests carry a session ID; without a session ID, it behaves like `every_request`. `new_session` judges once and reuses that target for the session. |
 | `message_hash_fallback` | No | `false` | Keys affinity on the first user message. Requires `classify_trigger = "new_session"`. |
 | `recent_turn_window` | No | unset | When unset, the judge sees the opening task and latest user follow-up, when present. When set, it also sees trailing turns. |