feat(relay): report routing outcome evidence

crates/libsy-llm-client/src/observation.rs

@@ -6,6 +6,7 @@
 use std::sync::Arc;
 use std::time::Duration;
 
+use switchyard_libsy::OutcomeMetadata;
 use switchyard_protocol::{ModelId, Usage};
 
 /// One completed model call observed while serving an algorithm run.
@@ -24,6 +25,8 @@ pub struct LlmCallObservation {
 /// One request-scoped observation emitted by the algorithm runner.
 #[derive(Clone, Debug)]
 pub enum RunObservation {
+    /// Metadata attached to the completed routing outcome.
+    Outcome(OutcomeMetadata),
     /// A completed model call requested by the algorithm for routing work.
     LlmCall(LlmCallObservation),
     /// A completed terminal model call made from the routing outcome.

crates/libsy-llm-client/src/run.rs

@@ -33,7 +33,8 @@ use crate::{metrics, observability};
 /// Run one request to completion, serving every offloaded model call with `client`.
 ///
 /// Returns the model selected by the algorithm and the final [`Response`]. `observer`, when
-/// present, receives each completed routing or answer call and the routing overhead.
+/// present, receives metadata for the routing outcome, each completed routing or answer call,
+/// and the routing overhead.
 ///
 /// `clients` resolves each offloaded call to the client for the target the algorithm
 /// selected — an algorithm may route among targets served by different providers, so this is
@@ -71,6 +72,11 @@ pub async fn run(
     metrics::record_routing_overhead(&algorithm_name, overhead);
 
     let selected_model_id = outcome.selected_model_id()?.clone();
+    if let Some(observer) = &observer
+        && let Some(metadata) = outcome.metadata
+    {
+        observer(RunObservation::Outcome(metadata));
+    }
     let (result, answer_duration) = if let Some(response) = outcome.response {
         (Ok(response), None)
     } else {
@@ -659,9 +665,13 @@ mod tests {
         assert!(matches!(observations[0], RunObservation::AnswerCall(_)));
         assert!(matches!(
             observations[1],
+            RunObservation::Outcome(ref metadata) if metadata.algorithm == "answered_test"
+        ));
+        assert!(matches!(
+            observations[2],
             RunObservation::RoutingOverhead(_)
         ));
-        assert_eq!(observations.len(), 2);
+        assert_eq!(observations.len(), 3);
         Ok(())
     }
 

crates/libsy-llm-client/tests/observability.rs

@@ -1211,15 +1211,19 @@ async fn observed_run_reports_one_successful_routed_call() -> switchyard_libsy::
         Some(Some(MODEL))
     );
     let observations = observations.lock();
-    assert_eq!(observations.len(), 2);
-    let RunObservation::AnswerCall(observation) = &observations[0] else {
+    assert_eq!(observations.len(), 3);
+    let RunObservation::Outcome(metadata) = &observations[0] else {
+        return Err(test_error("expected an outcome observation"));
+    };
+    assert_eq!(metadata.algorithm, ALGO);
+    let RunObservation::AnswerCall(observation) = &observations[1] else {
         return Err(test_error("expected an answer-call observation"));
     };
     assert_eq!(observation.selected_model, MODEL);
     assert!(observation.is_success);
     assert!(observation.usage.is_some());
     assert!(matches!(
-        observations[1],
+        observations[2],
         RunObservation::RoutingOverhead(_)
     ));
     Ok(())

crates/switchyard-nemo-relay-plugin/README.md

@@ -198,10 +198,12 @@ meaning requires a new schema version.
 | `switchyard.routing.requested` | `algorithm` |
 | `switchyard.routing.llm_call` | `call_index`, `selected_model`, `call_role`, `outcome`, `latency_ms` |
 | `switchyard.routing.overhead` | `latency_ms` |
-| `switchyard.routing.decision` | `algorithm`, `selected_model`, nullable `served_model`, nullable `fallback_used` |
-| `switchyard.routing.error` | `failure_kind`; optional `category`, `phase`, `upstream_status`, and `target` |
+| `switchyard.routing.decision` | `algorithm`, `outcome_id`, `selected_model`, nullable `served_model`, nullable `fallback_used`, and optional `evidence` |
+| `switchyard.routing.error` | `failure_kind`; route-execution failures also include `category`, `phase`, nullable `upstream_status` and `target`, and may include `outcome_id` and `evidence` |
 
 `served_model` and `fallback_used` are `null` when serving metadata is unavailable.
+`evidence` is an object containing the supported string fields `source`, `verdict`,
+`trigger`, and `reason_code`, and numeric fields `score`, `confidence`, and `threshold`.
 
 ## Failure policy
 

crates/switchyard-nemo-relay-plugin/src/runtime.rs

@@ -202,17 +202,25 @@ impl SwitchyardRuntime {
         });
         match route.execute(request, Some(observer)).await {
             Ok(output) => {
-                self.emit_observations(&mut events, take_observations(&observations), &metadata);
+                let mut outcome_fields = self.emit_observations(
+                    &mut events,
+                    take_observations(&observations),
+                    &metadata,
+                );
                 let served_model = output.response.served_model().map(|model| model.as_str());
+                let mut data = json!({
+                    "algorithm": route.algorithm_name(),
+                    "selected_model": output.selected_model.as_str(),
+                    "served_model": served_model,
+                    "fallback_used": served_model
+                        .map(|model| model != output.selected_model.as_str()),
+                });
+                if let Json::Object(data) = &mut data {
+                    data.append(&mut outcome_fields);
+                }
                 events.push(RoutingEvent::Mark(RoutingMark {
                     name: "switchyard.routing.decision".into(),
-                    data: json!({
-                        "algorithm": route.algorithm_name(),
-                        "selected_model": output.selected_model.as_str(),
-                        "served_model": served_model,
-                        "fallback_used": served_model
-                            .map(|model| model != output.selected_model.as_str()),
-                    }),
+                    data,
                     metadata,
                     severity: Some(LogSeverity::Info),
                 }));
@@ -222,11 +230,16 @@ impl SwitchyardRuntime {
                 }
             }
             Err(error) => {
-                self.emit_observations(&mut events, take_observations(&observations), &metadata);
+                let outcome_fields = self.emit_observations(
+                    &mut events,
+                    take_observations(&observations),
+                    &metadata,
+                );
                 self.route_execution_error_mark(
                     &mut events,
                     &error.execution_error_summary(),
                     None,
+                    outcome_fields,
                 );
                 Execution {
                     result: Err("Switchyard route execution failed".into()),
@@ -249,10 +262,20 @@ impl SwitchyardRuntime {
         events: &mut Vec<RoutingEvent>,
         observations: Vec<RunObservation>,
         metadata: &Json,
-    ) {
+    ) -> Map<String, Json> {
         let mut call_index = 0;
+        let mut outcome_fields = Map::new();
         for observation in observations {
             match observation {
+                RunObservation::Outcome(outcome) => {
+                    outcome_fields.insert(
+                        "outcome_id".into(),
+                        Json::String(outcome.outcome_id().into()),
+                    );
+                    if let Some(evidence) = evidence_for_mark(outcome.evidence) {
+                        outcome_fields.insert("evidence".into(), evidence);
+                    }
+                }
                 RunObservation::LlmCall(call) => {
                     call_index += 1;
                     self.routing_call_events(events, call, call_index, metadata);
@@ -287,6 +310,7 @@ impl SwitchyardRuntime {
                 }
             }
         }
+        outcome_fields
     }
 
     fn routing_call_events(
@@ -338,14 +362,31 @@ impl SwitchyardRuntime {
         events: &mut Vec<RoutingEvent>,
         summary: &RouteErrorSummary,
         metadata: Option<&Json>,
+        outcome_fields: Map<String, Json>,
     ) {
         let metadata = metadata
             .cloned()
             .unwrap_or_else(|| event_metadata(events).unwrap_or_else(|| Json::Object(Map::new())));
-        events.extend(route_execution_error_events(summary, metadata));
+        events.extend(route_execution_error_events(
+            summary,
+            metadata,
+            outcome_fields,
+        ));
     }
 }
 
+fn evidence_for_mark(evidence: Option<Json>) -> Option<Json> {
+    let Some(Json::Object(mut evidence)) = evidence else {
+        return None;
+    };
+    evidence.retain(|name, value| match name.as_str() {
+        "source" | "verdict" | "trigger" | "reason_code" => value.is_string(),
+        "score" | "confidence" | "threshold" => value.is_number(),
+        _ => false,
+    });
+    (!evidence.is_empty()).then_some(Json::Object(evidence))
+}
+
 pub(crate) fn emit_events(runtime: &PluginRuntime, events: Vec<RoutingEvent>) {
     for event in events {
         emit_event(runtime, event);
@@ -425,6 +466,7 @@ fn returned_events(
             for event in route_execution_error_events(
                 &stream_error_summary(error, served_model.as_ref()),
                 metadata.clone(),
+                Map::new(),
             ) {
                 emit_event(event);
             }
@@ -491,24 +533,40 @@ fn relay_stream_error(error: LlmStreamError) -> String {
     }
 }
 
-fn route_execution_error_mark(summary: &RouteErrorSummary, metadata: Json) -> RoutingMark {
+fn route_execution_error_mark(
+    summary: &RouteErrorSummary,
+    metadata: Json,
+    mut outcome_fields: Map<String, Json>,
+) -> RoutingMark {
+    let mut data = json!({
+        "failure_kind": "route_execution",
+        "category": summary.kind.as_str(),
+        "phase": summary.phase.as_str(),
+        "upstream_status": summary.upstream_status,
+        "target": summary.target.as_ref().map(|target| target.as_str()),
+    });
+    if let Json::Object(data) = &mut data {
+        data.append(&mut outcome_fields);
+    }
     RoutingMark {
         name: "switchyard.routing.error".into(),
-        data: json!({
-            "failure_kind": "route_execution",
-            "category": summary.kind.as_str(),
-            "phase": summary.phase.as_str(),
-            "upstream_status": summary.upstream_status,
-            "target": summary.target.as_ref().map(|target| target.as_str()),
-        }),
+        data,
         metadata,
         severity: Some(LogSeverity::Error),
     }
 }
 
-fn route_execution_error_events(summary: &RouteErrorSummary, metadata: Json) -> Vec<RoutingEvent> {
+fn route_execution_error_events(
+    summary: &RouteErrorSummary,
+    metadata: Json,
+    outcome_fields: Map<String, Json>,
+) -> Vec<RoutingEvent> {
     vec![
-        RoutingEvent::Mark(route_execution_error_mark(summary, metadata.clone())),
+        RoutingEvent::Mark(route_execution_error_mark(
+            summary,
+            metadata.clone(),
+            outcome_fields,
+        )),
         failure_metric(
             "route_execution",
             Some(summary.kind.as_str()),
@@ -894,12 +952,90 @@ mod tests {
             assert_eq!(decision["selected_model"], "target/model");
             assert_eq!(decision["served_model"], "target/model");
             assert_eq!(decision["fallback_used"], false);
+            assert!(decision["outcome_id"].is_string());
             let response = execution.result.expect("target call should succeed");
             assert_eq!(response["object"], "response");
             assert_eq!(response["model"], "target/model");
         }
     }
 
+    #[tokio::test]
+    async fn failed_answer_keeps_routing_outcome_in_error_mark() {
+        let server = MockServer::start().await;
+        Mock::given(method("POST"))
+            .and(path("/v1/chat/completions"))
+            .respond_with(ResponseTemplate::new(503).set_body_string("unavailable"))
+            .expect(2)
+            .mount(&server)
+            .await;
+        let deployment = json!({
+            "schema_version": 1,
+            "llm_clients": {
+                "local": {
+                    "format": "openai_chat",
+                    "base_url": format!("{}/v1", server.uri()),
+                    "max_retries": 0,
+                }
+            },
+            "targets": {
+                "strong": {"id": "poc/strong", "llm_client": "local"},
+                "weak": {"id": "poc/weak", "llm_client": "local"},
+            },
+            "routes": {
+                "stage": {
+                    "id": "switchyard/stage",
+                    "type": "stage_router",
+                    "capable_target": "strong",
+                    "efficient_target": "weak",
+                    "picker": "efficient_first",
+                    "confidence_threshold": 0.5,
+                }
+            },
+        });
+        let runtime = SwitchyardRuntime::new(crate::config::SwitchyardConfig {
+            priority: 0,
+            switchyard_config_path: None,
+            switchyard_config: Some(
+                deployment
+                    .as_object()
+                    .expect("deployment should be an object")
+                    .clone(),
+            ),
+        })
+        .expect("stage runtime should load");
+        let request = runtime
+            .decode_request(
+                WireFormat::OpenAiChat,
+                RelayRequest {
+                    headers: Map::new(),
+                    content: json!({
+                        "model": "switchyard/stage",
+                        "messages": [{"role": "user", "content": "hello"}],
+                    }),
+                },
+                false,
+            )
+            .expect("request should decode");
+
+        let execution = runtime
+            .execute_buffered(WireFormat::OpenAiChat, request)
+            .await;
+
+        assert!(execution.result.is_err());
+        let error = execution
+            .events
+            .iter()
+            .find_map(|event| match event {
+                RoutingEvent::Mark(mark) if mark.name == "switchyard.routing.error" => {
+                    Some(&mark.data)
+                }
+                RoutingEvent::Mark(_) | RoutingEvent::Metric(_) => None,
+            })
+            .expect("error mark should be emitted");
+        assert!(error["outcome_id"].is_string());
+        assert_eq!(error["evidence"], json!({"source": "fall_open"}));
+    }
+
     #[tokio::test]
     async fn buffered_responses_restore_codex_tool_namespaces() {
         let server = MockServer::start().await;
@@ -1019,6 +1155,7 @@ mod tests {
         let mark = route_execution_error_mark(
             &error.execution_error_summary(),
             json!({"session_id": "session"}),
+            Map::new(),
         );
 
         assert_eq!(mark.name, "switchyard.routing.error");
@@ -1033,6 +1170,33 @@ mod tests {
         assert!(!mark.data.to_string().contains(secret));
     }
 
+    #[test]
+    fn decision_evidence_keeps_only_documented_typed_fields() {
+        let evidence = evidence_for_mark(Some(json!({
+            "source": "llm-classifier",
+            "score": 0.9,
+            "confidence": "wrong type",
+            "threshold": 0.5,
+            "verdict": "continue",
+            "trigger": "turn",
+            "reason_code": "test",
+            "unknown": "patient name is Jane Doe",
+            "prompt": {"secret": "do not export"},
+        })));
+
+        assert_eq!(
+            evidence,
+            Some(json!({
+                "source": "llm-classifier",
+                "score": 0.9,
+                "threshold": 0.5,
+                "verdict": "continue",
+                "trigger": "turn",
+                "reason_code": "test",
+            }))
+        );
+    }
+
     #[test]
     fn routing_observations_emit_debug_marks_and_metrics() {
         let runtime = runtime_for("switchyard");