diff --git a/CHANGELOG.md b/CHANGELOG.md index c6adadd03..4d7cc3206 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,11 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). on the Responses wire, `reasoning_effort` on Chat Completions), so a strong tier can run at `max` behind a client that sends `high`. `extra_body` only fills absent keys and could not do this. Rejected on Anthropic clients. +- **Target `model` distinct from `id`** — a target may name the provider model + it sends upstream separately from its routing id, so several targets can + address one provider model with different settings (effort, headers, + endpoint). Previously the runner kept a single target per model id and + dropped the rest. - **Raw Responses stream trace** — an opt-in trace of every upstream Responses event as received, under `RUST_LOG=switchyard_translation::responses::raw=trace`, for diagnosing provider-specific event shapes. (#646) diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rs index 2dacc6383..1ff42fc85 100644 --- a/crates/libsy-llm-client/src/client.rs +++ b/crates/libsy-llm-client/src/client.rs @@ -57,13 +57,22 @@ const INITIAL_RETRY_DELAY: Duration = Duration::from_millis(250); const MAX_RETRY_BACKOFF: Duration = Duration::from_secs(2); const MAX_RETRY_AFTER: Duration = Duration::from_secs(60); -/// How one model is served: the `default_backend` used when the request does not -/// pin a wire format, plus any `other_backends` reachable over additional formats. +/// How one routed model is served. +/// +/// `model_name` is the routing identity a route or target refers to, and the key the client +/// resolves a call by; `upstream_model`, when set, is the name the provider receives instead. +/// Requests go to `default_backend` unless they pin a wire format served by one of +/// `other_backends`. Several configs may point at one provider model under distinct routing +/// ids, each with its own backend settings. #[derive(Clone, Debug)] pub struct ModelConfig { model_name: ModelId, default_backend: Backend, other_backends: Option>, + /// Model name sent upstream when it differs from `model_name`. Lets two configs + /// with different backend settings (effort, headers, endpoint) address the same + /// provider model under distinct routing ids. + upstream_model: Option, } impl ModelConfig { @@ -78,8 +87,22 @@ impl ModelConfig { model_name: model_name.into(), default_backend, other_backends, + upstream_model: None, } } + + /// Sends `upstream_model` as the provider's model name instead of `model_name`. + pub fn with_upstream_model(mut self, upstream_model: impl Into) -> Self { + self.upstream_model = Some(upstream_model.into()); + self + } + + /// The model name the provider sees for this config. + fn upstream_name(&self) -> &str { + self.upstream_model + .as_deref() + .unwrap_or_else(|| self.model_name.as_ref()) + } } /// A model-bearing provider operation outside the normal completion endpoint. @@ -244,8 +267,14 @@ impl TranslatingLlmClient { .map_err(|error| LlmClientError::RequestEncoding(error.to_string()))?; // `encode_request` round-trips a preserved same-format body verbatim, // which keeps the caller's original `model`; force the resolved model so - // the upstream always sees the target id. - set_json_model(&mut body, model); + // the upstream always sees the configured provider model name. + let upstream_model = self + .model_to_config + .get(model) + .map(ModelConfig::upstream_name) + .unwrap_or_else(|| model.as_ref()) + .to_string(); + set_json_model(&mut body, &upstream_model); if matches!(backend, Backend::OpenAiResponses(_)) { sanitize_openai_responses_provider_body(&mut body); } @@ -1146,6 +1175,13 @@ mod tests { )] } + fn chat_map_with_upstream_model(base_url: &str, upstream: &str) -> Vec { + vec![ + ModelConfig::new("gpt-tier", Backend::OpenAiChat(config(base_url)), None) + .with_upstream_model(upstream), + ] + } + fn anthropic_map(base_url: &str) -> Vec { vec![ModelConfig::new( "claude", @@ -1682,6 +1718,43 @@ mod tests { Ok(()) } + /// A config's upstream model name, not its routing id, is what the provider receives. + #[tokio::test] + async fn upstream_model_name_replaces_the_routing_id_on_the_wire() + -> std::result::Result<(), Box> { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/chat/completions")) + .and(wiremock::matchers::body_partial_json( + json!({"model": "gpt"}), + )) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "id": "1", + "model": "gpt", + "choices": [{ + "index": 0, + "message": {"role": "assistant", "content": "ok"}, + "finish_reason": "stop" + }], + "usage": {} + }))) + .mount(&server) + .await; + let client = TranslatingLlmClient::new(&chat_map_with_upstream_model( + &format!("{}/v1", server.uri()), + "gpt", + ))?; + client + .call_rewrite_model_raw( + json!({"model": "client-facing", "messages": [{"role": "user", "content": "hi"}]}), + None, + Some(&ModelId::from("gpt-tier")), + WireFormat::OpenAiChat, + ) + .await?; + Ok(()) + } + #[tokio::test] async fn extra_body_adds_defaults_without_overriding_the_request() -> std::result::Result<(), Box> { diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rs index 316754723..c720bf76f 100644 --- a/crates/switchyard-runner/src/config.rs +++ b/crates/switchyard-runner/src/config.rs @@ -287,7 +287,7 @@ impl DeploymentConfig { ))); } } - model_configs.push(ModelConfig::new( + let mut model_config = ModelConfig::new( target.id.clone(), build_backend( &target.llm_client, @@ -296,7 +296,16 @@ impl DeploymentConfig { target.reasoning_effort.clone(), )?, None, - )); + ); + if let Some(model) = &target.model { + if model.trim().is_empty() { + return Err(RunnerError::configuration(format!( + "target {target_name} model must not be empty" + ))); + } + model_config = model_config.with_upstream_model(model.clone()); + } + model_configs.push(model_config); } let mut clients = BTreeMap::new(); @@ -530,7 +539,11 @@ struct LlmClientConfig { #[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] struct TargetConfig { + /// Routing id of the target; also the model name sent upstream unless `model` is set. id: ModelId, + /// Provider model name sent upstream when it differs from `id`, so several targets + /// (for example one per reasoning effort) can address the same provider model. + model: Option, llm_client: String, #[serde(default)] extra_body: BTreeMap, @@ -1109,6 +1122,21 @@ new = ["send_message"] Ok(()) } + #[test] + fn two_targets_can_share_an_upstream_model_under_distinct_ids() -> RunnerResult<()> { + let strong = "[targets.strong]\nid = \"strong/model\"\nllm_client = \"responses\""; + assert!(VALID_CONFIG.contains(strong)); + let aliased = VALID_CONFIG.replace( + strong, + "[targets.strong]\nid = \"strong-max\"\nmodel = \"strong/model\"\nllm_client = \"responses\"\n\n[targets.strong_low]\nid = \"strong-low\"\nmodel = \"strong/model\"\nllm_client = \"responses\"", + ); + runner_from_toml(&aliased)?; + + let blank = VALID_CONFIG.replace(strong, &format!("{strong}\nmodel = \" \"")); + assert!(error_message(&blank).contains("model must not be empty")); + Ok(()) + } + #[test] fn classifier_judge_completion_caps_are_configurable() -> RunnerResult<()> { let capability = VALID_CONFIG.replace( diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md index 7d4b8b914..ea7284509 100644 --- a/docs/reference/toml_schema.md +++ b/docs/reference/toml_schema.md @@ -86,8 +86,9 @@ calls an upstream. | Key | Required | Default | Meaning | |---|:---:|---|---| -| `id` | Yes | — | Exact model ID sent upstream. | +| `id` | Yes | — | Routing identifier of the target, unique per `llm_client`. Also the model ID sent upstream unless `model` is set. | | `llm_client` | Yes | — | Key under `[llm_clients]`. | +| `model` | No | same as `id` | Provider model name sent upstream when it differs from `id`. Lets several targets address one provider model with different settings, for example one target per reasoning effort; the routing id stays unique. | | `system_prompt` | No | unset | System prompt prepended when this target serves a completion. | | `extra_body` | No | `{}` | Values merged into the upstream request when the request does not already set that key. | | `reasoning_effort` | No | unset | Reasoning effort forced on every request to this target, replacing the value the caller sent (`reasoning.effort` on `openai_responses`, `reasoning_effort` on `openai_chat`). Rejected on `anthropic_messages` clients. Use it to run one target at a different effort than the client asked for, for example a strong tier at `max` behind a client that sends `high`. Two targets for the same model id on the same `llm_client` collapse into one, so give each effort tier its own `llm_clients` entry (same endpoint, different name). |