feat(libsy): Pass models at runtime not construction time- #630

Instead of giving the available models to the algorithm in `new` we pass them alongside the request in `run_stream` where they go in the `Driver`.

See #588

Assisted-by: Codex:GPT 5.6 Sol high
Signed-off-by: Graham King <grahamk@nvidia.com>

Author: grahamking

README.md

@@ -171,17 +171,15 @@ switchyard-protocol = { git = "https://github.com/NVIDIA-NeMo/Switchyard.git", b
 tokio = { version = "1", features = ["macros", "rt"] }
 ```
 
-**2. Construct an algorithm.** Target names are whatever your harness calls its
-models. This is the stage router from the benchmark; `random`,
-`llm_task_classifier`, and `llm_classifier` are built the same way.
+**2. Construct an algorithm.** Models are supplied when each request runs. This
+is the stage router from the benchmark; `random`, `llm_task_classifier`, and
+`llm_classifier` are built the same way.
 
 ```python
 from switchyard.libsy import LlmResponse, Step
 from switchyard.libsy.algorithms import stage_router
 
 algorithm = stage_router(
-    "capable",
-    "efficient",
     picker="efficient_first",
     confidence_threshold=0.5,
 )

benchmark/routing-profiles/tau2-telecom-custom-opus-qwen-aggressive.toml

@@ -50,19 +50,18 @@ llm_client = "openrouter"
 [routes.switchyard]
 id = "switchyard"
 type = "llm_classifier"
-classifier_target = "classifier"
 mode = "custom"
 recent_turn_window = 6
 # Re-classify when the user speaks again, and hold that target across the tool calls
 # in between, so a tool chain never switches tier mid-task.
 classify_trigger = "user_turn"
-targets = ["weak", "strong"]
-default_target = "strong"
+models = { judge = ["classifier"], capable = ["strong"], efficient = ["weak"], any = ["strong", "weak"] }
+default_target = "capable"
 response_schema = '''
 {
   "type": "object",
   "properties": {
-    "route": { "type": "string", "enum": ["weak", "strong"] },
+    "route": { "type": "string", "enum": ["efficient", "capable"] },
     "confidence": { "type": "number" },
     "abstain": { "type": "boolean" }
   },
@@ -74,10 +73,10 @@ prompt = '''
 You are a routing classifier inside a customer-service agent. Return exactly
 one JSON object:
 
-{"route": "weak" or "strong", "confidence": number 0..1, "abstain": boolean}
+{"route": "efficient" or "capable", "confidence": number 0..1, "abstain": boolean}
 
-State the route DIRECTLY: "weak" = the on-device assistant handles this turn;
-"strong" = escalate this turn to the frontier model.
+State the route DIRECTLY: "efficient" = the on-device assistant handles this turn;
+"capable" = escalate this turn to the frontier model.
 
 ROUTING BIAS: the WEAK tier is the DEFAULT — it handles nearly all support
 work end-to-end: lookups, standard actions, troubleshooting with known steps,

benchmark/routing-profiles/tau2-telecom-custom-opus-qwen-balanced.toml

@@ -50,19 +50,18 @@ llm_client = "openrouter"
 [routes.switchyard]
 id = "switchyard"
 type = "llm_classifier"
-classifier_target = "classifier"
 mode = "custom"
 recent_turn_window = 6
 # Re-classify when the user speaks again, and hold that target across the tool calls
 # in between, so a tool chain never switches tier mid-task.
 classify_trigger = "user_turn"
-targets = ["weak", "strong"]
-default_target = "strong"
+models = { judge = ["classifier"], capable = ["strong"], efficient = ["weak"], any = ["strong", "weak"] }
+default_target = "capable"
 response_schema = '''
 {
   "type": "object",
   "properties": {
-    "route": { "type": "string", "enum": ["weak", "strong"] },
+    "route": { "type": "string", "enum": ["efficient", "capable"] },
     "confidence": { "type": "number" },
     "abstain": { "type": "boolean" }
   },
@@ -76,20 +75,20 @@ You see a condensed view of the conversation: the original request, recent turns
 (including tool results), and the customer's newest message. Return exactly one
 JSON object:
 
-{"route": "weak" or "strong", "confidence": number 0..1, "abstain": boolean}
+{"route": "efficient" or "capable", "confidence": number 0..1, "abstain": boolean}
 
-State the route DIRECTLY: "weak" = the on-device assistant handles this turn;
-"strong" = escalate this turn to the frontier model. Decide for the customer's
+State the route DIRECTLY: "efficient" = the on-device assistant handles this turn;
+"capable" = escalate this turn to the frontier model. Decide for the customer's
 NEWEST request, using the recent turns as context.
 
-Route "weak" when the newest request is ROUTINE — the procedure is
+Route "efficient" when the newest request is ROUTINE — the procedure is
 clear and it's about executing it:
   account/order/status lookups, reading or relaying tool results, standard
   single-step actions (toggle a setting, resend a code, restart a service),
   collecting information from the customer, confirmations, pleasantries,
   straightforward troubleshooting with an obvious next step.
 
-Route "strong" when the newest request needs NON-OBVIOUS
+Route "capable" when the newest request needs NON-OBVIOUS
 JUDGMENT the routine tier may get wrong:
   applying or reconciling POLICY with multiple conditions (eligibility, refunds,
   exceptions, proration), conflicts between what the customer wants and what

crates/libsy-llm-client/src/run.rs

@@ -21,7 +21,7 @@ use http::StatusCode;
 use parking_lot::Mutex;
 use switchyard_libsy::{Algorithm, CallModel, LibsyError, Result, RoutingOutcome, drive};
 use switchyard_protocol::{
-    LlmClientError, ModelId, Request, Response, RoutedLlmClient, RoutingFallbackReason,
+    Category, LlmClientError, ModelId, Request, Response, RoutedLlmClient, RoutingFallbackReason,
 };
 use switchyard_translation::prepare_request_for_target;
 
@@ -44,6 +44,7 @@ pub async fn run(
     algorithm: Arc<dyn Algorithm>,
     clients: ClientRouter,
     request: Request,
+    models: HashMap<Category, Vec<ModelId>>,
     observer: Option<RunObserver>,
 ) -> Result<(ModelId, Response)> {
     let algorithm_name = algorithm.name().to_string();
@@ -52,7 +53,7 @@ pub async fn run(
     // This says if we have an observer, put Some(..) in routing_observations.
     // No observer means we don't want any routing_observations.
     let routing_observations = observer.as_ref().map(|_| Arc::new(Mutex::new(Vec::new())));
-    let outcome = drive(algorithm, request, {
+    let outcome = drive(algorithm, request, models, {
         let routing_observations = routing_observations.clone();
         move |call| serve(routing_clients.clone(), call, routing_observations.clone())
     })
@@ -110,9 +111,10 @@ pub async fn decide(
     algorithm: Arc<dyn Algorithm>,
     clients: ClientRouter,
     request: Request,
+    models: HashMap<Category, Vec<ModelId>>,
 ) -> Result<RoutingOutcome> {
     let routing_clients = clients.clone();
-    let mut outcome = drive(algorithm, request, move |call| {
+    let mut outcome = drive(algorithm, request, models, move |call| {
         serve(routing_clients.clone(), call, None)
     })
     .await?;
@@ -453,9 +455,7 @@ mod tests {
 
     use crate::{Backend, HttpBackendConfig, ModelConfig, TranslatingLlmClient};
 
-    struct CandidateAlgorithm {
-        models: Vec<ModelId>,
-    }
+    struct CandidateAlgorithm {}
 
     struct AnsweredAlgorithm {
         model: ModelId,
@@ -469,13 +469,14 @@ mod tests {
 
         async fn route(
             self: Arc<Self>,
-            _driver: Driver,
+            driver: Driver,
             request: Request,
         ) -> Result<RoutingOutcome> {
-            let selected_model = self.models.first().cloned().ok_or(LibsyError::NoTargets)?;
+            let models = driver.models_for(Category::Any);
+            let selected_model = models.first().cloned().ok_or(LibsyError::NoTargets)?;
             Ok(RoutingOutcome::route_to(
                 selected_model,
-                self.models.iter().skip(1).cloned().collect(),
+                models.iter().skip(1).cloned().collect(),
                 request,
             ))
         }
@@ -603,19 +604,27 @@ mod tests {
             requests: Mutex::new(Vec::new()),
             first,
         });
-        let algorithm = Arc::new(CandidateAlgorithm {
-            models: vec!["weak".into(), "strong".into()],
-        });
+        let algorithm = Arc::new(CandidateAlgorithm {});
+        let models = to_category_map(&["weak", "strong"]);
         let result = run(
             algorithm,
             ClientRouter::single(client.clone()),
             request(),
+            models,
             None,
         )
         .await;
         (client, result)
     }
 
+    fn to_category_map(names: &[&str]) -> HashMap<Category, Vec<ModelId>> {
+        [(
+            Category::Any,
+            names.iter().map(|name| ModelId::from(*name)).collect(),
+        )]
+        .into()
+    }
+
     #[tokio::test]
     async fn answered_outcome_does_not_make_a_second_model_call() -> Result<()> {
         let client = Arc::new(CandidateClient {
@@ -633,6 +642,7 @@ mod tests {
             }),
             ClientRouter::single(client.clone()),
             request(),
+            HashMap::new(),
             Some(observer),
         )
         .await?;
@@ -697,11 +707,10 @@ mod tests {
         );
 
         run(
-            Arc::new(CandidateAlgorithm {
-                models: vec!["weak".into(), "strong".into()],
-            }),
+            Arc::new(CandidateAlgorithm {}),
             clients,
             request(),
+            to_category_map(&["weak", "strong"]),
             None,
         )
         .await?;
@@ -733,6 +742,7 @@ mod tests {
             }),
             clients,
             request(),
+            HashMap::new(),
         )
         .await?;
 
@@ -763,11 +773,10 @@ mod tests {
         );
 
         let outcome = decide(
-            Arc::new(CandidateAlgorithm {
-                models: vec!["weak".into(), "strong".into()],
-            }),
+            Arc::new(CandidateAlgorithm {}),
             clients,
             request(),
+            to_category_map(&["weak", "strong"]),
         )
         .await?;
 
@@ -904,10 +913,15 @@ mod tests {
             ])
             .map_err(|error| LibsyError::external("building test client", error))?,
         );
-        let algorithm = Arc::new(CandidateAlgorithm {
-            models: vec!["weak".into(), "strong".into()],
-        });
-        run(algorithm, ClientRouter::single(client), request(), None).await?;
+        let algorithm = Arc::new(CandidateAlgorithm {});
+        run(
+            algorithm,
+            ClientRouter::single(client),
+            request(),
+            to_category_map(&["weak", "strong"]),
+            None,
+        )
+        .await?;
 
         assert_eq!(&*calls.lock(), &["weak", "weak", "weak", "strong"]);
         Ok(())
@@ -995,17 +1009,22 @@ mod tests {
             ])
             .expect("building test client"),
         );
-        let algorithm = Arc::new(CandidateAlgorithm {
-            models: vec!["weak".into(), "strong".into()],
-        });
+        let algorithm = Arc::new(CandidateAlgorithm {});
         let mut llm_request = text_request(Some("auto".to_string()), "hello".to_string());
         llm_request.stream = true;
         let request = Request {
             llm_request,
             raw_request: None,
             metadata: None,
         };
-        let result = run(algorithm, ClientRouter::single(client), request, None).await;
+        let result = run(
+            algorithm,
+            ClientRouter::single(client),
+            request,
+            to_category_map(&["weak", "strong"]),
+            None,
+        )
+        .await;
         (server, calls, result)
     }