Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 15 additions & 5 deletions lib/instrumentation/aws-sdk/v3/bedrock.js
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ const {
LlmChatCompletionMessage,
LlmChatCompletionSummary,
LlmEmbedding,
LlmError,
LlmErrorMessage,
BedrockCommand,
BedrockResponse,
StreamHandler
Expand Down Expand Up @@ -174,8 +174,13 @@ function recordChatCompletionMessages({
recordEvent({ agent, type: 'LlmChatCompletionSummary', msg: summary })

if (err) {
const llmError = new LlmError({ bedrockResponse, err, summary })
agent.errors.add(transaction, err, llmError)
const llmErrorMessage = new LlmErrorMessage(
{ response: bedrockResponse,
cause: err,
summary,
useNameAsCode: true }
)
agent.errors.add(transaction, err, llmErrorMessage)
}
}

Expand Down Expand Up @@ -221,8 +226,13 @@ function recordEmbeddingMessage({
}

if (err) {
const llmError = new LlmError({ bedrockResponse, err, embedding: embeddings.length === 1 ? embeddings[0] : undefined })
agent.errors.add(transaction, err, llmError)
const llmErrorMessage = new LlmErrorMessage(
{ response: bedrockResponse,
cause: err,
embedding: embeddings.length === 1 ? embeddings[0] : undefined,
useNameAsCode: true }
)
agent.errors.add(transaction, err, llmErrorMessage)
}
}

Expand Down
29 changes: 0 additions & 29 deletions lib/llm-events/aws-bedrock/error.js

This file was deleted.

2 changes: 1 addition & 1 deletion lib/llm-events/aws-bedrock/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ module.exports = {
LlmChatCompletionSummary: require('./chat-completion-summary'),
LlmEmbedding: require('./embedding'),
LlmEvent: require('./event'),
LlmError: require('./error'),
LlmErrorMessage: require('../error-message'),
StreamHandler: require('./stream-handler'),
ConverseStreamHandler: require('./converse-stream-handler')
}
52 changes: 32 additions & 20 deletions lib/llm-events/error-message.js
Original file line number Diff line number Diff line change
Expand Up @@ -12,43 +12,55 @@
module.exports = class LlmErrorMessage {
/**
* @param {object} params Constructor parameters
* @param {object} [params.response] Instance of an incoming message.
* @param {object} [params.cause] An instance of the OpenAI error object.
* @param {object} params.response Instance of an incoming message.
* @param {object} params.cause An instance of the LLM error object.
* @param {LlmChatCompletionSummary} [params.summary] Details about the
* conversation if it was a chat completion conversation.
* @param {LlmEmbedding} [params.embedding] Details about the conversation
* if it was an embedding conversation.
* @param {LlmVectorStoreSearch} [params.vectorsearch] Details about the vector
* search if it was a vector search event.
* @param {LlmTool} [params.tool] Details about the tool event if it was a tool event.
* @param {boolean} [params.useNameAsCode] defaults to false, only Bedrock sets it to true so far
*/
constructor({ response, cause, summary, embedding, vectorsearch, tool } = {}) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think I realized that this class is overloaded. We handle specific assignment of attributes depending on params and/or further processing. It seems like most LLMs will either use keys on response or error(cause) to assign attributes but in the case of gemini it parses error and in case of bedrock is uses the error.name for the code instead of error.code

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I was thinking of maybe having error message classes that inherit from LlmErrorMessage for Bedrock and Gemini. I was planning to do in the AIM subscriber refactor.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

for sure, that's what I was referring to. it can wait until then

// For @google/genai only, cause does not have the `error` or `status` fields,
// but it does have `message` with the info we need. So, we need to parse
// the relevant fields from cause.message to get `status` and `error`.
let parsedError
const isGeminiVendor = embedding?.vendor === 'gemini' || summary?.vendor === 'gemini'
if (isGeminiVendor && cause?.message) {
try {
// Extract the JSON portion of the cause.message
const jsonStartIndex = cause.message.indexOf('{')
const jsonString = cause.message.substring(jsonStartIndex)
parsedError = JSON.parse(jsonString)?.error
} catch {
parsedError = undefined
}
}
this['http.statusCode'] = response?.status ?? cause?.status ?? parsedError?.code
constructor({ response, cause, summary = {}, embedding = {}, vectorsearch = {}, tool = {}, useNameAsCode = false } = {}) {
this['http.statusCode'] = response?.statusCode ?? response?.status ?? cause?.status
this['error.message'] = cause?.message
this['error.code'] = response?.code ?? cause?.error?.code ?? parsedError?.code
this['error.code'] = response?.code ?? cause?.error?.code
if (useNameAsCode) {
this['error.code'] = cause?.name
}
this['error.param'] = response?.param ?? cause?.error?.param
this.completion_id = summary?.id
this.embedding_id = embedding?.id
this.vector_store_id = vectorsearch?.id
this.tool_id = tool?.id

if (embedding?.vendor === 'gemini' || summary?.vendor === 'gemini') {
this._handleGemini(cause)
}
}

get [Symbol.toStringTag]() {
return 'LlmErrorMessage'
}

/**
* For `@google/genai` only, `cause` does not have the `error` or `status` fields,
* but it does have `message` with the info we need. So, we need to parse
* the relevant fields from cause.message to get `status` and `error`.
* @param {object} cause error object
*/
_handleGemini(cause) {
if (cause?.message) {
try {
const jsonStartIndex = cause.message.indexOf('{')
const jsonString = cause.message.substring(jsonStartIndex)
const parsedError = JSON.parse(jsonString)?.error

this['http.statusCode'] = parsedError?.code
this['error.code'] = parsedError?.code
} catch { }
}
}
}
15 changes: 9 additions & 6 deletions test/unit/llm-events/aws-bedrock/error.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -7,26 +7,29 @@

const test = require('node:test')
const assert = require('node:assert')
const LlmError = require('../../../../lib/llm-events/aws-bedrock/error')
const LlmErrorMessage = require('#agentlib/llm-events/error-message.js')

test.beforeEach((ctx) => {
ctx.nr = {}
ctx.nr.bedrockResponse = {
ctx.nr.response = {
statusCode: 400
}

ctx.nr.err = {
// The cause of the error.
ctx.nr.cause = {
message: 'No soup for you',
name: 'SoupRule'
}

ctx.nr.summary = {
id: 'completion-id'
}

ctx.nr.useNameAsCode = true
})

test('create creates a new instance', (t) => {
const err = new LlmError(t.nr)
const err = new LlmErrorMessage(t.nr)
assert.equal(err['http.statusCode'], 400)
assert.equal(err['error.message'], 'No soup for you')
assert.equal(err['error.code'], 'SoupRule')
Expand All @@ -37,7 +40,7 @@ test('create creates a new instance', (t) => {
test('create error with embedding_id', (t) => {
delete t.nr.summary
t.nr.embedding = { id: 'embedding-id' }
const err = new LlmError(t.nr)
const err = new LlmErrorMessage(t.nr)
assert.equal(err['http.statusCode'], 400)
assert.equal(err['error.message'], 'No soup for you')
assert.equal(err['error.code'], 'SoupRule')
Expand All @@ -46,7 +49,7 @@ test('create error with embedding_id', (t) => {
})

test('empty error', () => {
const err = new LlmError()
const err = new LlmErrorMessage()
assert.ok(!err['http.statusCode'])
assert.ok(!err['error.message'])
assert.ok(!err['error.code'])
Expand Down
Loading