diff --git a/packages/dd-trace/src/llmobs/constants/tags.js b/packages/dd-trace/src/llmobs/constants/tags.js index f47b2808ede..d8f50d952b7 100644 --- a/packages/dd-trace/src/llmobs/constants/tags.js +++ b/packages/dd-trace/src/llmobs/constants/tags.js @@ -34,6 +34,7 @@ module.exports = { TOTAL_TOKENS_METRIC_KEY: 'total_tokens', CACHE_READ_INPUT_TOKENS_METRIC_KEY: 'cache_read_input_tokens', CACHE_WRITE_INPUT_TOKENS_METRIC_KEY: 'cache_write_input_tokens', + REASONING_OUTPUT_TOKENS_METRIC_KEY: 'reasoning_output_tokens', DROPPED_IO_COLLECTION_ERROR: 'dropped_io' } diff --git a/packages/dd-trace/src/llmobs/plugins/openai/index.js b/packages/dd-trace/src/llmobs/plugins/openai/index.js index 6a0fe14bfd4..59955dbe307 100644 --- a/packages/dd-trace/src/llmobs/plugins/openai/index.js +++ b/packages/dd-trace/src/llmobs/plugins/openai/index.js @@ -116,6 +116,10 @@ class OpenAiLLMObsPlugin extends LLMObsPlugin { metrics.cacheReadTokens = cacheReadTokens } } + // Reasoning tokens - Responses API returns `output_tokens_details`, `completion_tokens_details` + const reasoningOutputObject = tokenUsage.output_tokens_details ?? tokenUsage.completion_tokens_details + const reasoningOutputTokens = reasoningOutputObject?.reasoning_tokens ?? 0 + if (reasoningOutputTokens !== undefined) metrics.reasoningOutputTokens = reasoningOutputTokens } return metrics @@ -429,9 +433,6 @@ class OpenAiLLMObsPlugin extends LLMObsPlugin { if (response.tool_choice !== undefined) outputMetadata.tool_choice = response.tool_choice if (response.truncation !== undefined) outputMetadata.truncation = response.truncation if (response.text !== undefined) outputMetadata.text = response.text - if (response.usage?.output_tokens_details?.reasoning_tokens !== undefined) { - outputMetadata.reasoning_tokens = response.usage.output_tokens_details.reasoning_tokens - } this._tagger.tagMetadata(span, outputMetadata) // update the metadata with the output metadata } diff --git a/packages/dd-trace/src/llmobs/tagger.js b/packages/dd-trace/src/llmobs/tagger.js index 8a4ec1ae195..a8a3d7ba978 100644 --- a/packages/dd-trace/src/llmobs/tagger.js +++ b/packages/dd-trace/src/llmobs/tagger.js @@ -25,6 +25,7 @@ const { INPUT_TOKENS_METRIC_KEY, OUTPUT_TOKENS_METRIC_KEY, TOTAL_TOKENS_METRIC_KEY, + REASONING_OUTPUT_TOKENS_METRIC_KEY, INTEGRATION, DECORATOR, PROPAGATED_ML_APP_KEY @@ -164,6 +165,9 @@ class LLMObsTagger { case 'cacheWriteTokens': processedKey = CACHE_WRITE_INPUT_TOKENS_METRIC_KEY break + case 'reasoningOutputTokens': + processedKey = REASONING_OUTPUT_TOKENS_METRIC_KEY + break } if (typeof value === 'number') { diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_responses_post_01aba65f.yaml b/packages/dd-trace/test/llmobs/cassettes/openai/openai_responses_post_01aba65f.yaml new file mode 100644 index 00000000000..3027b66cd8d --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_responses_post_01aba65f.yaml @@ -0,0 +1,135 @@ +interactions: +- request: + body: '{"model":"gpt-5-mini","input":"Solve this step by step: What is 15 * 24?","max_output_tokens":500,"stream":false}' + headers: + ? !!python/object/apply:multidict._multidict.istr + - Accept + : - application/json + ? !!python/object/apply:multidict._multidict.istr + - Accept-Encoding + : - gzip, deflate + ? !!python/object/apply:multidict._multidict.istr + - Accept-Language + : - '*' + ? !!python/object/apply:multidict._multidict.istr + - Connection + : - keep-alive + Content-Length: + - '113' + ? !!python/object/apply:multidict._multidict.istr + - Content-Type + : - application/json + ? !!python/object/apply:multidict._multidict.istr + - User-Agent + : - OpenAI/JS 6.8.1 + ? !!python/object/apply:multidict._multidict.istr + - X-Stainless-Arch + : - arm64 + ? !!python/object/apply:multidict._multidict.istr + - X-Stainless-Lang + : - js + ? !!python/object/apply:multidict._multidict.istr + - X-Stainless-OS + : - MacOS + ? !!python/object/apply:multidict._multidict.istr + - X-Stainless-Package-Version + : - 6.8.1 + ? !!python/object/apply:multidict._multidict.istr + - X-Stainless-Retry-Count + : - '0' + ? !!python/object/apply:multidict._multidict.istr + - X-Stainless-Runtime + : - node + ? !!python/object/apply:multidict._multidict.istr + - X-Stainless-Runtime-Version + : - v22.21.1 + ? !!python/object/apply:multidict._multidict.istr + - sec-fetch-mode + : - cors + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: "{\n \"id\": \"resp_0e28e5318868c83a0169308b613bdc819184e3bca92ada65e6\",\n + \ \"object\": \"response\",\n \"created_at\": 1764789089,\n \"status\": + \"completed\",\n \"background\": false,\n \"billing\": {\n \"payer\": + \"developer\"\n },\n \"error\": null,\n \"incomplete_details\": null,\n + \ \"instructions\": null,\n \"max_output_tokens\": 500,\n \"max_tool_calls\": + null,\n \"model\": \"gpt-5-mini-2025-08-07\",\n \"output\": [\n {\n \"id\": + \"rs_0e28e5318868c83a0169308b617fa08191b6726155d23d010d\",\n \"type\": + \"reasoning\",\n \"summary\": []\n },\n {\n \"id\": \"msg_0e28e5318868c83a0169308b63623c819198d6e2c9b0eb26b0\",\n + \ \"type\": \"message\",\n \"status\": \"completed\",\n \"content\": + [\n {\n \"type\": \"output_text\",\n \"annotations\": + [],\n \"logprobs\": [],\n \"text\": \"Method 1 \\u2014 distributive + property:\\n15 \\u00d7 24 = 15 \\u00d7 (20 + 4)\\n= 15 \\u00d7 20 + 15 \\u00d7 + 4\\n= 300 + 60\\n= 360\\n\\nMethod 2 \\u2014 split the other way:\\n24 \\u00d7 + 15 = 24 \\u00d7 (10 + 5)\\n= 24 \\u00d7 10 + 24 \\u00d7 5\\n= 240 + 120\\n= + 360\\n\\nAnswer: 360.\"\n }\n ],\n \"role\": \"assistant\"\n + \ }\n ],\n \"parallel_tool_calls\": true,\n \"previous_response_id\": + null,\n \"prompt_cache_key\": null,\n \"prompt_cache_retention\": null,\n + \ \"reasoning\": {\n \"effort\": \"medium\",\n \"summary\": null\n },\n + \ \"safety_identifier\": null,\n \"service_tier\": \"default\",\n \"store\": + false,\n \"temperature\": 1.0,\n \"text\": {\n \"format\": {\n \"type\": + \"text\"\n },\n \"verbosity\": \"medium\"\n },\n \"tool_choice\": + \"auto\",\n \"tools\": [],\n \"top_logprobs\": 0,\n \"top_p\": 1.0,\n \"truncation\": + \"disabled\",\n \"usage\": {\n \"input_tokens\": 20,\n \"input_tokens_details\": + {\n \"cached_tokens\": 0\n },\n \"output_tokens\": 232,\n \"output_tokens_details\": + {\n \"reasoning_tokens\": 128\n },\n \"total_tokens\": 252\n },\n + \ \"user\": null,\n \"metadata\": {}\n}" + headers: + CF-RAY: + - 9a855ebe0d40cb6a-BOS + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Wed, 03 Dec 2025 19:11:33 GMT + Server: + - cloudflare + Set-Cookie: + - __cf_bm=tGToqitV1eFrOrI1DCsm6GfeeW0ajMWiagCcgm7EZ84-1764789093-1.0.1.1-d.Bwuc72eKGJd7bZL0hQRAOgBqhp15Rk0H9FzUo.0s8hqVVPBeKHE39I5EaaTi2YZdtQyCFyHmd3iILpZJKskSfEdJ3MtfEHr_4Ktk7yDw8; + path=/; expires=Wed, 03-Dec-25 19:41:33 GMT; domain=.api.openai.com; HttpOnly; + Secure; SameSite=None + - _cfuvid=NO0A21ruRpV_Vd_9ghyhNxI_FfJlolr0EqD7FHB3tOs-1764789093127-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: + - datadog-staging + openai-processing-ms: + - '3895' + openai-project: + - proj_gt6TQZPRbZfoY2J9AQlEJMpd + openai-version: + - '2020-10-01' + x-envoy-upstream-service-time: + - '3900' + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '180000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '179999985' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_d13dea53ffc94a628dfa63719a5bcbb9 + status: + code: 200 + message: OK +version: 1 diff --git a/packages/dd-trace/test/llmobs/plugins/openai/openaiv3.spec.js b/packages/dd-trace/test/llmobs/plugins/openai/openaiv3.spec.js index 60d0337bad4..81db0fe11be 100644 --- a/packages/dd-trace/test/llmobs/plugins/openai/openaiv3.spec.js +++ b/packages/dd-trace/test/llmobs/plugins/openai/openaiv3.spec.js @@ -61,7 +61,8 @@ describe('integrations', () => { metrics: { input_tokens: MOCK_NUMBER, output_tokens: MOCK_NUMBER, - total_tokens: MOCK_NUMBER + total_tokens: MOCK_NUMBER, + reasoning_output_tokens: 0 }, modelName: 'gpt-3.5-turbo-instruct:20230824-v2', modelProvider: 'openai', @@ -113,6 +114,7 @@ describe('integrations', () => { ], metrics: { cache_read_input_tokens: 0, + reasoning_output_tokens: 0, input_tokens: MOCK_NUMBER, output_tokens: MOCK_NUMBER, total_tokens: MOCK_NUMBER @@ -146,7 +148,9 @@ describe('integrations', () => { { text: 'hello world' } ], outputValue: '[1 embedding(s) returned]', - metrics: { input_tokens: MOCK_NUMBER, output_tokens: 0, total_tokens: MOCK_NUMBER }, + metrics: { + input_tokens: MOCK_NUMBER, output_tokens: 0, total_tokens: MOCK_NUMBER, reasoning_output_tokens: 0 + }, modelName: 'text-embedding-ada-002-v2', modelProvider: 'openai', metadata: { encoding_format: 'base64' }, diff --git a/packages/dd-trace/test/llmobs/plugins/openai/openaiv4.spec.js b/packages/dd-trace/test/llmobs/plugins/openai/openaiv4.spec.js index 6d603a44f84..cf88ef58e78 100644 --- a/packages/dd-trace/test/llmobs/plugins/openai/openaiv4.spec.js +++ b/packages/dd-trace/test/llmobs/plugins/openai/openaiv4.spec.js @@ -81,7 +81,9 @@ describe('integrations', () => { outputMessages: [ { content: MOCK_STRING, role: '' } ], - metrics: { input_tokens: MOCK_NUMBER, output_tokens: MOCK_NUMBER, total_tokens: MOCK_NUMBER }, + metrics: { + input_tokens: MOCK_NUMBER, output_tokens: MOCK_NUMBER, total_tokens: MOCK_NUMBER, reasoning_output_tokens: 0 + }, modelName: 'gpt-3.5-turbo-instruct:20230824-v2', modelProvider: 'openai', metadata: { @@ -128,6 +130,7 @@ describe('integrations', () => { ], metrics: { cache_read_input_tokens: 0, + reasoning_output_tokens: 0, input_tokens: MOCK_NUMBER, output_tokens: MOCK_NUMBER, total_tokens: MOCK_NUMBER @@ -161,7 +164,9 @@ describe('integrations', () => { { text: 'hello world' } ], outputValue: '[1 embedding(s) returned]', - metrics: { input_tokens: MOCK_NUMBER, output_tokens: 0, total_tokens: MOCK_NUMBER }, + metrics: { + input_tokens: MOCK_NUMBER, output_tokens: 0, total_tokens: MOCK_NUMBER, reasoning_output_tokens: 0 + }, modelName: 'text-embedding-ada-002-v2', modelProvider: 'openai', metadata: { encoding_format: 'base64' }, @@ -220,6 +225,7 @@ describe('integrations', () => { tags: { ml_app: 'test', integration: 'openai' }, metrics: { cache_read_input_tokens: 0, + reasoning_output_tokens: 0, input_tokens: MOCK_NUMBER, output_tokens: MOCK_NUMBER, total_tokens: MOCK_NUMBER @@ -268,7 +274,12 @@ describe('integrations', () => { outputMessages: [ { content: '\n\nHello! How can I assist you?', role: '' } ], - metrics: { input_tokens: MOCK_NUMBER, output_tokens: MOCK_NUMBER, total_tokens: MOCK_NUMBER }, + metrics: { + input_tokens: MOCK_NUMBER, + output_tokens: MOCK_NUMBER, + total_tokens: MOCK_NUMBER, + reasoning_output_tokens: 0 + }, modelName: 'gpt-3.5-turbo-instruct:20230824-v2', modelProvider: 'openai', metadata: { @@ -329,6 +340,7 @@ describe('integrations', () => { ], metrics: { cache_read_input_tokens: 0, + reasoning_output_tokens: 0, input_tokens: MOCK_NUMBER, output_tokens: MOCK_NUMBER, total_tokens: MOCK_NUMBER @@ -413,6 +425,7 @@ describe('integrations', () => { tags: { ml_app: 'test', integration: 'openai' }, metrics: { cache_read_input_tokens: 0, + reasoning_output_tokens: 0, input_tokens: MOCK_NUMBER, output_tokens: MOCK_NUMBER, total_tokens: MOCK_NUMBER @@ -599,6 +612,7 @@ describe('integrations', () => { ], metrics: { cache_read_input_tokens: 0, + reasoning_output_tokens: 0, input_tokens: 1221, output_tokens: 100, total_tokens: 1321 @@ -647,6 +661,7 @@ describe('integrations', () => { output_tokens: 100, total_tokens: 1320, cache_read_input_tokens: 1152, + reasoning_output_tokens: 0 }, modelName: 'gpt-4o-2024-08-06', modelProvider: 'openai', @@ -689,7 +704,8 @@ describe('integrations', () => { input_tokens: MOCK_NUMBER, output_tokens: MOCK_NUMBER, total_tokens: MOCK_NUMBER, - cache_read_input_tokens: 0 + cache_read_input_tokens: 0, + reasoning_output_tokens: 0 }, modelName: 'gpt-4o-mini-2024-07-18', modelProvider: 'openai', @@ -700,7 +716,6 @@ describe('integrations', () => { tool_choice: 'auto', truncation: 'disabled', text: { format: { type: 'text' }, verbosity: 'medium' }, - reasoning_tokens: 0, stream: false }, tags: { ml_app: 'test', integration: 'openai' } @@ -739,7 +754,8 @@ describe('integrations', () => { input_tokens: MOCK_NUMBER, output_tokens: MOCK_NUMBER, total_tokens: MOCK_NUMBER, - cache_read_input_tokens: 0 + cache_read_input_tokens: 0, + reasoning_output_tokens: 0 }, modelName: 'gpt-4o-mini-2024-07-18', modelProvider: 'openai', @@ -750,7 +766,6 @@ describe('integrations', () => { tool_choice: 'auto', truncation: 'disabled', text: { format: { type: 'text' }, verbosity: 'medium' }, - reasoning_tokens: 0, stream: true }, tags: { ml_app: 'test', integration: 'openai' } @@ -957,6 +972,52 @@ describe('integrations', () => { ]) }) }) + + it('submits a response span with reasoning tokens', async function () { + if (semifies(realVersion, '<4.87.0')) { + this.skip() + } + + await openai.responses.create({ + model: 'gpt-5-mini', + input: 'Solve this step by step: What is 15 * 24?', + max_output_tokens: 500, + stream: false + }) + + const { apmSpans, llmobsSpans } = await getEvents() + assertLlmObsSpanEvent(llmobsSpans[0], { + span: apmSpans[0], + spanKind: 'llm', + name: 'OpenAI.createResponse', + inputMessages: [ + { role: 'user', content: 'Solve this step by step: What is 15 * 24?' } + ], + outputMessages: [ + { role: 'reasoning', content: MOCK_STRING }, + { role: 'assistant', content: MOCK_STRING } + ], + metrics: { + input_tokens: MOCK_NUMBER, + output_tokens: MOCK_NUMBER, + total_tokens: MOCK_NUMBER, + cache_read_input_tokens: MOCK_NUMBER, + reasoning_output_tokens: 128 + }, + modelName: 'gpt-5-mini-2025-08-07', + modelProvider: 'openai', + metadata: { + max_output_tokens: 500, + top_p: 1, + temperature: 1, + tool_choice: 'auto', + truncation: 'disabled', + text: { format: { type: 'text' }, verbosity: 'medium' }, + stream: false + }, + tags: { ml_app: 'test', integration: 'openai' } + }) + }) }) }) })