Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
137 changes: 131 additions & 6 deletions crates/agentic-server-core/src/executor/inference.rs
Original file line number Diff line number Diff line change
Expand Up @@ -190,7 +190,8 @@ pub fn call_inference(
}
}

/// Convert a successful upstream response body into normalized SSE data lines.
/// Convert a successful upstream response body into SSE data lines with a `data: ` prefix.
/// The single space after the field colon is optional on the wire.
pub(super) fn response_lines(
resp: reqwest::Response,
chunk_timeout: Duration,
Expand All @@ -215,12 +216,19 @@ pub(super) fn response_lines(
return;
}
};
for line in lines {
match line.as_str() {
"data: [DONE]" => return,
l if l.starts_with("data: ") => yield Ok(line),
_ => {}
for mut line in lines {
let Some(data) = line.strip_prefix("data:") else {
continue;
};
if data.strip_prefix(' ').unwrap_or(data) == "[DONE]" {
return;
}
// Keep one transport spelling for Responses and Messages consumers,
// without trimming whitespace that belongs to the field value.
if !data.starts_with(' ') {
line.insert("data:".len(), ' ');
}
yield Ok(line);
}
}
}
Expand All @@ -239,6 +247,123 @@ mod tests {

use super::*;

async fn read_sse_body(body: String, keep_open: bool) -> Vec<ExecutorResult<String>> {
let app = axum::Router::new().route(
"/v1/responses",
post(move || {
let body = body.clone();
async move {
let chunks = async_stream::stream! {
yield Ok::<_, Infallible>(Bytes::from(body));
if keep_open {
std::future::pending::<()>().await;
}
};
Response::builder()
.header("content-type", "text/event-stream")
.body(Body::from_stream(chunks))
.unwrap()
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let url = format!("http://{}/v1/responses", listener.local_addr().unwrap());
let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
let result = call_inference(
"{}".to_owned(),
url,
Arc::new(reqwest::Client::new()),
None,
Duration::from_secs(1),
)
.collect()
.await;
server.abort();
let _ = server.await;
result
}

#[tokio::test]
async fn sse_data_spacing_preserves_field_values() {
let lines = read_sse_body(
concat!(
":comment\r\nevent: ignored\r\nid: 1\r\nretry: 1000\r\n",
"Data: ignored\r\ndatabase: ignored\r\n data: ignored\r\n",
"data:{\"delta\":\"雪 ☃: data: text\"}\r\n\r\n",
"data: {\"spaced\":true}\n\n",
"data: {\"leading_space\":true}\n\n",
"data:\t{\"tab\":true}\n\n",
"data:\n\ndata: \n\ndata:{malformed}\n\n",
"data: [DONE]\n\ndata:[DONE]extra\n\n",
)
.to_owned(),
false,
)
.await
.into_iter()
.collect::<ExecutorResult<Vec<_>>>()
.unwrap();
assert_eq!(
lines,
[
"data: {\"delta\":\"雪 ☃: data: text\"}",
"data: {\"spaced\":true}",
"data: {\"leading_space\":true}",
"data: \t{\"tab\":true}",
"data: ",
"data: ",
"data: {malformed}",
"data: [DONE]",
"data: [DONE]extra",
]
);
}

#[tokio::test]
async fn sse_data_spacing_stops_at_both_done_markers() {
for separator in ["", " "] {
let body = format!("data: {{}}\n\ndata:{separator}[DONE]\r\n\r\ndata: ignored\n\n");
let lines = read_sse_body(body, true)
.await
.into_iter()
.collect::<ExecutorResult<Vec<_>>>()
.expect("[DONE] must stop without waiting for upstream EOF or a chunk timeout");
assert_eq!(lines, ["data: {}"]);
}
}

#[tokio::test]
async fn sse_data_spacing_preserves_line_limit() {
for separator in ["", " "] {
let prefix = format!("data:{separator}");
let line = format!("{prefix}{}", "x".repeat(MAX_SSE_LINE_BYTES - prefix.len()));
let accepted = read_sse_body(format!("{line}\n\n"), false).await;
assert_eq!(accepted.len(), 1);
assert_eq!(
accepted[0].as_ref().unwrap().len(),
MAX_SSE_LINE_BYTES + usize::from(separator.is_empty())
);

let rejected = read_sse_body(format!("{line}x\n\n"), false).await;
assert_eq!(rejected.len(), 1);
assert!(
rejected[0]
.as_ref()
.unwrap_err()
.to_string()
.contains("upstream SSE line exceeded")
);
}
}

#[tokio::test]
async fn sse_data_spacing_preserves_chunk_timeout_without_done() {
let lines = read_sse_body("data:{}\n\n".to_owned(), true).await;
assert_eq!(lines.len(), 2);
assert_eq!(lines[0].as_ref().unwrap(), "data: {}");
assert!(lines[1].as_ref().unwrap_err().to_string().contains("chunk timeout"));
}

async fn oversized_body_server(status: StatusCode) -> (String, tokio::task::JoinHandle<()>) {
let app = axum::Router::new().route(
"/v1/responses",
Expand Down
43 changes: 39 additions & 4 deletions crates/agentic-server-core/tests/messages_stream_test.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ use agentic_core::tool::{ToolRegistry, WebSearchHandler};
use agentic_core::types::messages::{GatewayToolMap, ToolParam, registry_tools};
use axum::extract::State;
use axum::response::{IntoResponse, Response};
use axum::routing::post;
use axum::routing::{get, post};
use axum::{Json, Router};
use futures::StreamExt;
use http::StatusCode;
Expand Down Expand Up @@ -146,7 +146,7 @@ async fn spawn_mock_vllm_stream_then_error(
async fn spawn_mock_search() -> (String, tokio::task::JoinHandle<()>) {
let app = Router::new().route(
"/v1/search",
post(|Json(_body): Json<Value>| async move {
get(|| async move {
Json(serde_json::json!({
"results": {"web": [{"url": "https://www.rust-lang.org/", "title": "Rust",
"description": "d", "snippets": ["Rust 1.89.0 is the latest stable release."]}], "news": []},
Expand Down Expand Up @@ -188,8 +188,28 @@ async fn run_test_messages_stream(

#[tokio::test]
async fn messages_stream_presents_one_message_and_hides_gateway_tool() {
let (vllm_url, upstream, _v) = spawn_mock_vllm_stream(cassette_turn_streams()).await;
let (search_url, _s) = spawn_mock_search().await;
assert_messages_stream_presents_one_message(cassette_turn_streams()).await;
}

#[tokio::test]
async fn messages_stream_accepts_unspaced_sse_data_through_gateway_tool_rounds() {
let streams = cassette_turn_streams()
.into_iter()
.map(|body| {
body.split_inclusive('\n')
.map(|line| {
line.strip_prefix("data: ")
.map_or_else(|| line.to_owned(), |data| format!("data:{data}"))
})
.collect()
})
.collect();
assert_messages_stream_presents_one_message(streams).await;
}

async fn assert_messages_stream_presents_one_message(streams: Vec<String>) {
let (vllm_url, upstream, vllm) = spawn_mock_vllm_stream(streams).await;
let (search_url, search) = spawn_mock_search().await;
let exec_ctx = build_exec_ctx(&vllm_url, &search_url).await;

let request = serde_json::json!({
Expand All @@ -210,13 +230,28 @@ async fn messages_stream_presents_one_message_and_hides_gateway_tool() {
let stream = run_test_messages_stream(request, registry, Arc::clone(&exec_ctx)).await;
let chunks: Vec<String> = stream.collect().await;
let sse = chunks.join("");
vllm.abort();
search.abort();
let _ = tokio::join!(vllm, search);

// Two upstream rounds ran (tool round + final).
assert_eq!(
upstream.calls.load(Ordering::SeqCst),
2,
"one tool round + one final round"
);
let requests = upstream.requests.lock().await;
let tool_output = &requests[1]["messages"].as_array().unwrap().last().unwrap()["content"][0];
assert_eq!(tool_output["type"], "tool_result");
assert_ne!(
tool_output["is_error"], true,
"gateway search must succeed: {tool_output}"
);
assert!(
tool_output["content"]
.to_string()
.contains("https://www.rust-lang.org/")
);

// Exactly one logical message lifecycle.
assert_eq!(
Expand Down
Loading