Skip to content

Commit ac99fc6

Browse files
committed
improve capture of openai messages
1 parent 2b44447 commit ac99fc6

5 files changed

Lines changed: 256 additions & 104 deletions

File tree

braintrust-sdk/instrumentation/langchain_1_8_0/src/main/java/dev/braintrust/instrumentation/langchain/v1_8_0/WrappedHttpClient.java

Lines changed: 5 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,9 @@
22

33
import static dev.braintrust.json.BraintrustJsonMapper.toJson;
44

5-
import com.fasterxml.jackson.databind.JsonNode;
65
import dev.braintrust.bootstrap.BraintrustBridge;
76
import dev.braintrust.instrumentation.InstrumentationSemConv;
7+
import dev.braintrust.instrumentation.SseResponseAccumulator;
88
import dev.braintrust.json.BraintrustJsonMapper;
99
import dev.langchain4j.exception.HttpException;
1010
import dev.langchain4j.http.client.HttpClient;
@@ -126,9 +126,7 @@ static class WrappedServerSentEventListener implements ServerSentEventListener {
126126
private final String providerName;
127127
private final long startNanos = System.nanoTime();
128128
private final AtomicLong timeToFirstTokenNanos = new AtomicLong();
129-
private final StringBuilder contentBuffer = new StringBuilder();
130-
private String finishReason = null;
131-
private JsonNode usageData = null;
129+
private final SseResponseAccumulator accumulator = new SseResponseAccumulator();
132130

133131
WrappedServerSentEventListener(
134132
ServerSentEventListener delegate, Span span, String providerName) {
@@ -186,48 +184,17 @@ private void accumulateChunk(String data) {
186184
if (timeToFirstTokenNanos.get() == 0L) {
187185
timeToFirstTokenNanos.compareAndExchange(0L, System.nanoTime() - startNanos);
188186
}
189-
JsonNode chunk = BraintrustJsonMapper.get().readTree(data);
190-
if (chunk.has("choices") && chunk.get("choices").size() > 0) {
191-
JsonNode choice = chunk.get("choices").get(0);
192-
if (choice.has("delta")) {
193-
JsonNode delta = choice.get("delta");
194-
if (delta.has("content")) {
195-
contentBuffer.append(delta.get("content").asText());
196-
}
197-
}
198-
if (choice.has("finish_reason") && !choice.get("finish_reason").isNull()) {
199-
finishReason = choice.get("finish_reason").asText();
200-
}
201-
}
202-
if (chunk.has("usage") && !chunk.get("usage").isNull()) {
203-
usageData = chunk.get("usage");
204-
}
187+
accumulator.merge(BraintrustJsonMapper.get().readTree(data));
205188
} catch (Exception e) {
206189
log.debug("Failed to parse SSE chunk: {}", data, e);
207190
}
208191
}
209192

210193
private void finalizeSpan() {
211194
try {
212-
var root = BraintrustJsonMapper.get().createObjectNode();
213-
214-
var choicesArray = BraintrustJsonMapper.get().createArrayNode();
215-
var choice = BraintrustJsonMapper.get().createObjectNode();
216-
choice.put("index", 0);
217-
if (finishReason != null) choice.put("finish_reason", finishReason);
218-
var message = BraintrustJsonMapper.get().createObjectNode();
219-
message.put("role", "assistant");
220-
message.put("content", contentBuffer.toString());
221-
choice.set("message", message);
222-
choicesArray.add(choice);
223-
root.set("choices", choicesArray);
224-
225-
if (usageData != null) {
226-
root.set("usage", usageData);
227-
}
228-
229195
Long ttft = timeToFirstTokenNanos.get();
230-
InstrumentationSemConv.tagLLMSpanResponse(span, providerName, toJson(root), ttft);
196+
InstrumentationSemConv.tagLLMSpanResponse(
197+
span, providerName, toJson(accumulator.build()), ttft);
231198
} catch (Exception e) {
232199
log.debug("Failed to finalize streaming span", e);
233200
}

braintrust-sdk/instrumentation/springai_1_0_0/src/main/java/dev/braintrust/instrumentation/springai/v1_0_0/BraintrustSpringAI.java

Lines changed: 7 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -553,81 +553,22 @@ static String reassembleSSEResponse(String rawSSE, Span span, StreamContext stre
553553
@SneakyThrows
554554
private static String reassembleOpenAISSE(String rawSSE, Span span, StreamContext streamCtx) {
555555
var mapper = dev.braintrust.json.BraintrustJsonMapper.get();
556-
var choices = mapper.createObjectNode();
557-
var usage = mapper.createObjectNode();
558-
String model = null;
559-
556+
// Generically merge chunks so every streamed field (content, reasoning, tool calls, ...) is
557+
// reconstructed — the shared accumulator merges tool-call deltas by index, which a naive
558+
// append-per-fragment loop cannot do.
559+
var accumulator = new dev.braintrust.instrumentation.SseResponseAccumulator();
560560
for (String line : rawSSE.split("\n")) {
561561
if (!line.startsWith("data: ") || line.equals("data: [DONE]")) {
562562
continue;
563563
}
564564
String json = line.substring("data: ".length()).trim();
565-
var chunk = mapper.readTree(json);
566-
567-
if (model == null && chunk.has("model") && !chunk.get("model").isNull()) {
568-
model = chunk.get("model").asText();
569-
}
570-
571-
if (chunk.has("choices")) {
572-
for (var choiceChunk : chunk.get("choices")) {
573-
int index = choiceChunk.has("index") ? choiceChunk.get("index").asInt() : 0;
574-
String indexKey = String.valueOf(index);
575-
576-
if (!choices.has(indexKey)) {
577-
var choice = mapper.createObjectNode();
578-
var message = mapper.createObjectNode();
579-
message.put("role", "assistant");
580-
message.put("content", "");
581-
choice.set("message", message);
582-
choice.put("index", index);
583-
choices.set(indexKey, choice);
584-
}
585-
586-
var choice = choices.get(indexKey);
587-
if (choiceChunk.has("delta")) {
588-
var delta = choiceChunk.get("delta");
589-
if (delta.has("content") && !delta.get("content").isNull()) {
590-
String existing = choice.get("message").get("content").asText();
591-
((com.fasterxml.jackson.databind.node.ObjectNode) choice.get("message"))
592-
.put("content", existing + delta.get("content").asText());
593-
}
594-
if (delta.has("tool_calls")) {
595-
if (!choice.get("message").has("tool_calls")) {
596-
((com.fasterxml.jackson.databind.node.ObjectNode)
597-
choice.get("message"))
598-
.set("tool_calls", mapper.createArrayNode());
599-
}
600-
for (var tc : delta.get("tool_calls")) {
601-
((com.fasterxml.jackson.databind.node.ArrayNode)
602-
choice.get("message").get("tool_calls"))
603-
.add(tc);
604-
}
605-
}
606-
}
607-
if (choiceChunk.has("finish_reason")
608-
&& !choiceChunk.get("finish_reason").isNull()) {
609-
((com.fasterxml.jackson.databind.node.ObjectNode) choice)
610-
.put("finish_reason", choiceChunk.get("finish_reason").asText());
611-
}
612-
}
613-
}
614-
615-
if (chunk.has("usage") && !chunk.get("usage").isNull()) {
616-
var u = chunk.get("usage");
617-
u.fields().forEachRemaining(entry -> usage.set(entry.getKey(), entry.getValue()));
618-
}
565+
accumulator.merge(mapper.readTree(json));
619566
}
620567

568+
var result = accumulator.build();
569+
String model = result.has("model") ? result.get("model").asText() : null;
621570
backfillModelMetadata(span, streamCtx, model);
622571

623-
var choicesArray = mapper.createArrayNode();
624-
choices.fields().forEachRemaining(entry -> choicesArray.add(entry.getValue()));
625-
626-
var result = mapper.createObjectNode();
627-
result.set("choices", choicesArray);
628-
if (usage.size() > 0) {
629-
result.set("usage", usage);
630-
}
631572
return dev.braintrust.json.BraintrustJsonMapper.toJson(result);
632573
}
633574

braintrust-sdk/src/main/java/dev/braintrust/instrumentation/InstrumentationSemConv.java

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -171,6 +171,13 @@ private static void tagOpenAIResponse(
171171
metrics.put("prompt_cached_tokens", details.get("cached_tokens"));
172172
}
173173
}
174+
// Reasoning tokens (Chat Completions API)
175+
if (usage.has("completion_tokens_details")) {
176+
JsonNode details = usage.get("completion_tokens_details");
177+
if (details.has("reasoning_tokens")) {
178+
metrics.put("completion_reasoning_tokens", details.get("reasoning_tokens"));
179+
}
180+
}
174181
// Responses API field names
175182
if (usage.has("input_tokens")) metrics.put("prompt_tokens", usage.get("input_tokens"));
176183
if (usage.has("output_tokens"))
Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
package dev.braintrust.instrumentation;
2+
3+
import com.fasterxml.jackson.databind.JsonNode;
4+
import com.fasterxml.jackson.databind.node.ArrayNode;
5+
import com.fasterxml.jackson.databind.node.ObjectNode;
6+
import dev.braintrust.json.BraintrustJsonMapper;
7+
import java.util.LinkedHashMap;
8+
import java.util.Map;
9+
10+
/**
11+
* Reconstructs a full (non-streaming) OpenAI-style chat completion response by generically merging
12+
* SSE chunks.
13+
*
14+
* <p>The merge is field-agnostic on purpose: every field the provider streams (content, reasoning,
15+
* tool calls, refusals, ...) is preserved rather than a hand-picked subset, so new streaming fields
16+
* are captured without changes here. Top-level fields (id, model, created, usage, ...) are
17+
* last-non-null-wins; each {@code choices[].delta} is merged into the reconstructed {@code
18+
* choices[].message}, where textual leaves are concatenated and structured leaves are merged
19+
* recursively.
20+
*
21+
* <p>Shared across instrumentation modules (langchain, spring-ai, ...) so there is one
22+
* reconstruction rather than divergent per-module copies. Instances are not thread-safe; feed one
23+
* accumulator the chunks of a single response from a single thread.
24+
*/
25+
public final class SseResponseAccumulator {
26+
private final ObjectNode responseRoot = BraintrustJsonMapper.get().createObjectNode();
27+
// Accumulated choices keyed by their "index" so multi-choice (n>1) streams merge correctly.
28+
private final Map<Integer, ObjectNode> choicesByIndex = new LinkedHashMap<>();
29+
30+
/** Merge one parsed SSE chunk into the reconstructed response. Non-objects are ignored. */
31+
public void merge(JsonNode chunk) {
32+
if (chunk == null || !chunk.isObject()) return;
33+
var fields = chunk.fields();
34+
while (fields.hasNext()) {
35+
var entry = fields.next();
36+
String name = entry.getKey();
37+
JsonNode value = entry.getValue();
38+
if ("choices".equals(name)) {
39+
if (value.isArray()) {
40+
value.forEach(this::mergeChoice);
41+
}
42+
} else if (!value.isNull()) {
43+
responseRoot.set(name, value);
44+
}
45+
}
46+
}
47+
48+
/** Build the reconstructed response object, assembling accumulated choices in index order. */
49+
public ObjectNode build() {
50+
var choicesArray = BraintrustJsonMapper.get().createArrayNode();
51+
choicesByIndex.values().forEach(choicesArray::add);
52+
responseRoot.set("choices", choicesArray);
53+
return responseRoot;
54+
}
55+
56+
private void mergeChoice(JsonNode choiceChunk) {
57+
if (!choiceChunk.isObject()) return;
58+
int index = choiceChunk.has("index") ? choiceChunk.get("index").asInt() : 0;
59+
ObjectNode choice =
60+
choicesByIndex.computeIfAbsent(
61+
index,
62+
i -> {
63+
var node = BraintrustJsonMapper.get().createObjectNode();
64+
node.put("index", i);
65+
return node;
66+
});
67+
var fields = choiceChunk.fields();
68+
while (fields.hasNext()) {
69+
var entry = fields.next();
70+
String name = entry.getKey();
71+
JsonNode value = entry.getValue();
72+
if ("delta".equals(name)) {
73+
// Streaming nests the message under "delta"; the reconstructed non-streaming shape
74+
// the UI expects uses "message".
75+
ObjectNode message =
76+
choice.has("message") && choice.get("message").isObject()
77+
? (ObjectNode) choice.get("message")
78+
: choice.putObject("message");
79+
deepMerge(message, value);
80+
} else if (!"index".equals(name) && !value.isNull()) {
81+
// finish_reason, logprobs, ... last-non-null-wins.
82+
choice.set(name, value);
83+
}
84+
}
85+
}
86+
87+
/**
88+
* Recursively merges {@code source} into {@code target}. Textual leaves are concatenated (so
89+
* streamed content / reasoning / tool-call arguments accumulate), nested objects are merged
90+
* key-by-key, and arrays whose elements carry an {@code index} (e.g. {@code tool_calls}) are
91+
* merged by that index. Other scalars are last-write-wins.
92+
*/
93+
private static void deepMerge(ObjectNode target, JsonNode source) {
94+
if (!source.isObject()) return;
95+
var fields = source.fields();
96+
while (fields.hasNext()) {
97+
var entry = fields.next();
98+
String name = entry.getKey();
99+
JsonNode value = entry.getValue();
100+
JsonNode existing = target.get(name);
101+
if (value.isTextual() && existing != null && existing.isTextual()) {
102+
target.put(name, existing.asText() + value.asText());
103+
} else if (value.isObject()) {
104+
if (existing != null && existing.isObject()) {
105+
deepMerge((ObjectNode) existing, value);
106+
} else {
107+
target.set(name, value.deepCopy());
108+
}
109+
} else if (value.isArray()) {
110+
mergeArray(target, name, value);
111+
} else if (!value.isNull()) {
112+
target.set(name, value);
113+
}
114+
}
115+
}
116+
117+
private static void mergeArray(ObjectNode target, String name, JsonNode sourceArray) {
118+
if (!(target.get(name) instanceof ArrayNode targetArray)) {
119+
target.set(name, sourceArray.deepCopy());
120+
return;
121+
}
122+
for (JsonNode element : sourceArray) {
123+
ObjectNode match =
124+
element.isObject() && element.has("index")
125+
? findByIndex(targetArray, element.get("index").asInt())
126+
: null;
127+
if (match != null) {
128+
deepMerge(match, element);
129+
} else {
130+
targetArray.add(element.deepCopy());
131+
}
132+
}
133+
}
134+
135+
private static ObjectNode findByIndex(ArrayNode array, int index) {
136+
for (JsonNode candidate : array) {
137+
if (candidate.isObject()
138+
&& candidate.has("index")
139+
&& candidate.get("index").asInt() == index) {
140+
return (ObjectNode) candidate;
141+
}
142+
}
143+
return null;
144+
}
145+
}

0 commit comments

Comments
 (0)