From e5116cc3f37e5935b4fe3815a89a6de71ddd8c8a Mon Sep 17 00:00:00 2001 From: Jay/Fienna Liang Date: Sun, 12 Jul 2026 19:27:31 +0800 Subject: [PATCH] feat: expose safe model run failures --- docs/guides/programmatic/turn-results.md | 17 +++++++ .../01-hosted-service/agent-service.ts | 3 +- .../02-http-sse-api/contracts.ts | 12 +++++ examples/sdk/05-hosted-agent/README.md | 19 +++---- .../control-plane/session-runtime.test.ts | 38 ++++++++++++-- .../integration/core/agent-loop.test.ts | 39 ++++++++++++++ .../integration/core/run-agent.test.ts | 18 +++++-- .../memory/memory-integration.test.ts | 3 +- .../agent-model-turn-retry-service.test.ts | 36 +++++++++++++ src/core/agent/finish/index.ts | 1 + src/core/agent/finish/run-finisher.ts | 31 +++++++---- src/core/agent/finish/types.ts | 9 +++- src/core/agent/model/README.md | 39 ++++++++++++++ .../agent/model/model-turn-retry-service.ts | 51 +++++++++++++++++-- src/core/agent/model/model-turn-service.ts | 8 ++- src/core/chat/engine/README.md | 5 +- src/core/chat/engine/turn-result.ts | 3 +- src/core/chat/engine/turns/service.ts | 1 + src/core/live/types.ts | 3 +- src/core/runtime/loop/checkpoint.ts | 1 + src/core/runtime/loop/service.ts | 1 + src/core/runtime/loop/types.ts | 3 +- src/core/types.ts | 25 ++++++++- src/index.ts | 2 + 24 files changed, 326 insertions(+), 42 deletions(-) create mode 100644 src/__tests__/unit/core/agent-model-turn-retry-service.test.ts create mode 100644 src/core/agent/model/README.md diff --git a/docs/guides/programmatic/turn-results.md b/docs/guides/programmatic/turn-results.md index b3314d24..2df4c6b5 100644 --- a/docs/guides/programmatic/turn-results.md +++ b/docs/guides/programmatic/turn-results.md @@ -8,6 +8,7 @@ const result = await engine.turns.submit({ sessionId, prompt, host }) console.log(result.outcome) console.log(result.summary) +console.log(result.failure) console.log(result.traceFile) console.log(result.artifacts.map((artifact) => artifact.id)) console.log(result.toolResults.map((entry) => entry.call.tool)) @@ -17,6 +18,10 @@ Fields: - `outcome`: why the turn stopped. - `summary`: persisted assistant summary for the turn. +- `failure`: optional safe structured category for a failed model run. It has + the shape `{ source: 'model', code }` and never contains credentials or raw + provider messages. Use this field for stable product behavior instead of + parsing `summary`. - `session`: updated persisted session. - `traceFile`: path to the persisted raw trace file. - `artifacts`: current artifacts associated with the session. @@ -25,3 +30,15 @@ Fields: Use these fields for product summaries and run artifacts. Reach for raw trace files only when your host needs lower-level evidence or custom analysis. + +For example, a hosted product can translate a rejected user-supplied model +credential into its own API error without coupling to an OpenAI error string: + +```ts +if (result.failure?.code === 'authentication') { + throw new ProductModelCredentialError() +} +``` + +Model failure codes currently distinguish `authentication`, `permission`, +`rate_limit`, `request`, `transport`, `empty_response`, and `unknown`. diff --git a/examples/sdk/05-hosted-agent/01-hosted-service/agent-service.ts b/examples/sdk/05-hosted-agent/01-hosted-service/agent-service.ts index d92bbe54..44b90cbd 100644 --- a/examples/sdk/05-hosted-agent/01-hosted-service/agent-service.ts +++ b/examples/sdk/05-hosted-agent/01-hosted-service/agent-service.ts @@ -34,7 +34,7 @@ export type HostedAgentRunAccepted = { sessionId: string; }; -export type HostedAgentResult = Pick; +export type HostedAgentResult = Pick; export type HostedAgentRunStreamItem = ConversationRunStreamItem; @@ -109,6 +109,7 @@ export class HostedAgentService { projectResult: (result): HostedAgentResult => ({ outcome: result.outcome, summary: result.summary, + ...(result.failure ? { failure: result.failure } : {}), }), projectError: () => ({ code: 'run_failed', diff --git a/examples/sdk/05-hosted-agent/02-http-sse-api/contracts.ts b/examples/sdk/05-hosted-agent/02-http-sse-api/contracts.ts index d84f0541..67131313 100644 --- a/examples/sdk/05-hosted-agent/02-http-sse-api/contracts.ts +++ b/examples/sdk/05-hosted-agent/02-http-sse-api/contracts.ts @@ -37,6 +37,18 @@ const HostedAgentActivitySchema = z.object({ const HostedAgentResultSchema = z.object({ outcome: z.string().min(1), summary: z.string(), + failure: z.object({ + source: z.literal('model'), + code: z.enum([ + 'authentication', + 'permission', + 'rate_limit', + 'request', + 'transport', + 'empty_response', + 'unknown', + ]), + }).optional(), }); export const HostedAgentRunProtocol = new ConversationRunProtocolCodec({ diff --git a/examples/sdk/05-hosted-agent/README.md b/examples/sdk/05-hosted-agent/README.md index 1838a8ee..7d311927 100644 --- a/examples/sdk/05-hosted-agent/README.md +++ b/examples/sdk/05-hosted-agent/README.md @@ -125,10 +125,10 @@ configuration, injected repositories, approval decisions, and process lifecycle. The example's `projectResult` callback reduces Heddle's internal turn result to -`outcome` and `summary`. In a product, this callback is also the place to await -authorized state persistence or reconciliation before clients observe success. -Projection failure becomes the run's error terminal; it cannot race behind a -premature successful result. +`outcome`, `summary`, and the optional safe `failure` category. In a product, +this callback is also the place to await authorized state persistence or +reconciliation before clients observe success. Projection failure becomes the +run's error terminal; it cannot race behind a premature successful result. The adjacent `projectError` callback keeps raw model, tool, provider, and persistence failures on the host side. Remote clients receive only the @@ -188,11 +188,12 @@ registers host-owned Express routes and composes - closing an SSE connection aborts only that subscription, not the run; - cancel is a separate, authenticated operation. -The hosted service deliberately projects terminal results to public `outcome` -and `summary` fields before replay. The API schema validates that boundary -again. Trace paths, artifacts, tool results, and internal session state are not -serialized to the browser. Extend the public projection and Zod schema with -only the product data the client is authorized to receive. +The hosted service deliberately projects terminal results to public `outcome`, +`summary`, and optional safe `failure` fields before replay. The API schema +validates that boundary again. Trace paths, artifacts, tool results, and +internal session state are not serialized to the browser. Extend the public +projection and Zod schema with only the product data the client is authorized +to receive. The session read/reset endpoints are example host operations used by the React stage. They project persisted visible messages plus the process-local active diff --git a/src/__tests__/integration/control-plane/session-runtime.test.ts b/src/__tests__/integration/control-plane/session-runtime.test.ts index 2c2d921e..2b93d65d 100644 --- a/src/__tests__/integration/control-plane/session-runtime.test.ts +++ b/src/__tests__/integration/control-plane/session-runtime.test.ts @@ -406,6 +406,32 @@ describe('conversation turn lifecycle', () => { ]); }); + it('returns the safe model failure category to programmatic hosts', async () => { + const storage = createConversationTurnStorage(); + vi.spyOn(agentLoopModule.AgentLoopRuntimeService, 'run').mockResolvedValue(createLoopResult({ + workspaceRoot: storage.workspaceRoot, + prompt: 'Use a rejected credential.', + summary: 'LLM error: Model authentication failed', + outcome: 'error', + failure: { source: 'model', code: 'authentication' }, + }) as never); + + const turnResult = await EngineConversationTurnService.run({ + workspaceRoot: storage.workspaceRoot, + stateRoot: storage.stateRoot, + traceDir: join(storage.stateRoot, 'traces'), + sessionStoragePath: storage.sessionStoragePath, + sessionId: storage.sessionId, + prompt: 'Use a rejected credential.', + apiKey: 'rejected-key', + memoryMaintenanceMode: 'none', + artifactRoot: storage.artifactRoot, + artifactsEnabled: true, + }); + + expect(turnResult.failure).toEqual({ source: 'model', code: 'authentication' }); + }); + it('clears the session lease when the run loop fails', async () => { const storage = createConversationTurnStorage(); vi.spyOn(agentLoopModule.AgentLoopRuntimeService, 'run').mockRejectedValue(new Error('loop failed')); @@ -469,8 +495,11 @@ function createLoopResult(args: { workspaceRoot: string; prompt: string; summary: string; + outcome?: RunResult['outcome']; + failure?: RunResult['failure']; trace?: RunResult['trace']; }) { + const outcome = args.outcome ?? 'done'; const trace: RunResult['trace'] = args.trace ?? [ { type: 'assistant.turn', @@ -481,8 +510,9 @@ function createLoopResult(args: { }, { type: 'run.finished', - outcome: 'done', + outcome, summary: args.summary, + ...(args.failure ? { failure: args.failure } : {}), step: 1, timestamp: '2026-05-03T00:00:02.000Z', }, @@ -493,8 +523,9 @@ function createLoopResult(args: { ]; return { - outcome: 'done', + outcome, summary: args.summary, + ...(args.failure ? { failure: args.failure } : {}), trace, transcript, model: 'gpt-5.4', @@ -509,8 +540,9 @@ function createLoopResult(args: { workspaceRoot: args.workspaceRoot, startedAt: '2026-05-03T00:00:00.000Z', finishedAt: '2026-05-03T00:00:02.000Z', - outcome: 'done', + outcome, summary: args.summary, + ...(args.failure ? { failure: args.failure } : {}), transcript, trace, }, diff --git a/src/__tests__/integration/core/agent-loop.test.ts b/src/__tests__/integration/core/agent-loop.test.ts index 6b2db422..07ff1fee 100644 --- a/src/__tests__/integration/core/agent-loop.test.ts +++ b/src/__tests__/integration/core/agent-loop.test.ts @@ -117,6 +117,45 @@ describe('AgentLoopRuntimeService.run', () => { }); }); + it('propagates safe model failures through loop state and terminal activity', async () => { + const events: AgentLoopEvent[] = []; + const fakeLlm: LlmAdapter = { + info: { + provider: 'openai', + model: 'gpt-test', + capabilities: { + toolCalls: true, + systemMessages: true, + reasoningSummaries: false, + parallelToolCalls: true, + }, + }, + async chat(): Promise { + throw Object.assign(new Error('Unauthorized'), { status: 401 }); + }, + }; + + const result = await AgentLoopRuntimeService.run({ + goal: 'Use a rejected credential.', + llm: fakeLlm, + tools: [], + includeDefaultTools: false, + logger: silentLogger, + workspaceRoot: resolve('/tmp/heddle-loop-failure-test'), + onEvent: (event) => events.push(event), + }); + + expect(result.failure).toEqual({ source: 'model', code: 'authentication' }); + expect(result.state.failure).toEqual({ source: 'model', code: 'authentication' }); + expect(events.at(-1)).toMatchObject({ + type: 'loop.finished', + failure: { source: 'model', code: 'authentication' }, + state: { + failure: { source: 'model', code: 'authentication' }, + }, + }); + }); + it('emits assistant stream events through the programmatic event stream', async () => { const events: AgentHeartbeatEvent[] = []; const fakeLlm: LlmAdapter = { diff --git a/src/__tests__/integration/core/run-agent.test.ts b/src/__tests__/integration/core/run-agent.test.ts index c26c8733..8c5f7b1d 100644 --- a/src/__tests__/integration/core/run-agent.test.ts +++ b/src/__tests__/integration/core/run-agent.test.ts @@ -104,10 +104,11 @@ describe('AgentRunService.run', () => { it('records an error outcome when the LLM chat throws a non-retryable error', async () => { let calls = 0; + const providerSecret = 'sk-provider-error-sentinel'; const fakeLlm: LlmAdapter = { async chat(): Promise { calls += 1; - throw Object.assign(new Error('Unauthorized'), { status: 401 }); + throw Object.assign(new Error(`Unauthorized: ${providerSecret}`), { status: 401 }); }, }; @@ -120,9 +121,15 @@ describe('AgentRunService.run', () => { }); expect(result.outcome).toBe('error'); - expect(result.summary).toBe('LLM error: Unauthorized'); + expect(result.summary).toBe('LLM error: Model authentication failed'); + expect(result.failure).toEqual({ source: 'model', code: 'authentication' }); expect(calls).toBe(1); - expect(result.trace.some((event) => event.type === 'run.finished')).toBe(true); + expect(result.trace.at(-1)).toMatchObject({ + type: 'run.finished', + outcome: 'error', + failure: { source: 'model', code: 'authentication' }, + }); + expect(JSON.stringify(result)).not.toContain(providerSecret); }); it('retries transient LLM transport errors before returning the assistant response', async () => { @@ -158,14 +165,14 @@ describe('AgentRunService.run', () => { reason: 'transport_error', attempt: 1, maxAttempts: 5, - message: 'fetch failed', + message: 'Model provider is temporarily unavailable', }), expect.objectContaining({ type: 'model.retry', reason: 'transport_error', attempt: 2, maxAttempts: 5, - message: 'fetch failed', + message: 'Model provider is temporarily unavailable', }), ]); }); @@ -189,6 +196,7 @@ describe('AgentRunService.run', () => { expect(result.outcome).toBe('error'); expect(result.summary).toBe('Model returned an empty response after 3 attempts'); + expect(result.failure).toEqual({ source: 'model', code: 'empty_response' }); expect(calls).toBe(3); expect(result.trace.filter((event) => event.type === 'model.retry')).toHaveLength(2); }); diff --git a/src/__tests__/integration/memory/memory-integration.test.ts b/src/__tests__/integration/memory/memory-integration.test.ts index e2c8f367..7e490b80 100644 --- a/src/__tests__/integration/memory/memory-integration.test.ts +++ b/src/__tests__/integration/memory/memory-integration.test.ts @@ -116,8 +116,9 @@ describe('memory maintenance integration', () => { expect(result.events.map((event) => event.type)).toEqual(['memory.maintenance_started', 'memory.maintenance_failed']); expect(result.events[1]).toMatchObject({ type: 'memory.maintenance_failed', - error: expect.stringContaining('maintainer unavailable'), + error: 'LLM error: Model request failed', }); + expect(JSON.stringify(result.events)).not.toContain('maintainer unavailable'); }); it('serializes maintenance runs per memory root', async () => { diff --git a/src/__tests__/unit/core/agent-model-turn-retry-service.test.ts b/src/__tests__/unit/core/agent-model-turn-retry-service.test.ts new file mode 100644 index 00000000..09389a49 --- /dev/null +++ b/src/__tests__/unit/core/agent-model-turn-retry-service.test.ts @@ -0,0 +1,36 @@ +import { describe, expect, it } from 'vitest'; +import { AgentModelTurnRetryService } from '@/core/agent/model/index.js'; + +describe('AgentModelTurnRetryService', () => { + it.each([ + [401, 'authentication', false], + [403, 'permission', false], + [400, 'request', false], + [429, 'rate_limit', true], + [503, 'transport', true], + [418, 'unknown', false], + ] as const)('classifies HTTP status %s as %s', (status, code, retryable) => { + const decision = AgentModelTurnRetryService.resolve({ + kind: 'error', + error: Object.assign(new Error('provider message'), { status }), + }); + + expect(decision).toMatchObject({ + retryable, + failure: { source: 'model', code }, + }); + expect(decision.failure).not.toHaveProperty('message'); + }); + + it('classifies network failures without exposing provider details', () => { + const decision = AgentModelTurnRetryService.resolve({ + kind: 'error', + error: Object.assign(new Error('fetch failed with secret-value'), { code: 'ECONNRESET' }), + }); + + expect(decision.failure).toEqual({ source: 'model', code: 'transport' }); + expect(decision.message).toBe('Model provider is temporarily unavailable'); + expect(decision.message).not.toContain('secret-value'); + expect(JSON.stringify(decision.failure)).not.toContain('secret-value'); + }); +}); diff --git a/src/core/agent/finish/index.ts b/src/core/agent/finish/index.ts index 2efc3c3a..3e382c53 100644 --- a/src/core/agent/finish/index.ts +++ b/src/core/agent/finish/index.ts @@ -3,6 +3,7 @@ export type { AgentRunFinishedValue, FinishAgentRunArgs, FinishAgentRunLogging, + FinishAgentRunOptions, FinishAssistantResponseArgs, MaybeFinishInterruptedArgs, } from './types.js'; diff --git a/src/core/agent/finish/run-finisher.ts b/src/core/agent/finish/run-finisher.ts index 3da45aea..d19a2721 100644 --- a/src/core/agent/finish/run-finisher.ts +++ b/src/core/agent/finish/run-finisher.ts @@ -3,7 +3,7 @@ import { HeddleEventType } from '@/core/event-types.js'; import type { LlmResponse } from '@/core/llm/types.js'; import type { RunResult, StopReason } from '@/core/types.js'; import type { AgentRunContext } from '../types.js'; -import type { FinishAgentRunLogging } from './types.js'; +import type { FinishAgentRunOptions } from './types.js'; /** * Owns terminal run outcomes and final RunResult shaping. @@ -19,8 +19,10 @@ export class AgentRunFinisher { static finishInterrupted(context: AgentRunContext, logMessage: string): RunResult { return AgentRunFinisher.finish(context, 'interrupted', INTERRUPTED_SUMMARY, { - logLevel: 'info', - logMessage, + logging: { + logLevel: 'info', + logMessage, + }, }); } @@ -40,15 +42,19 @@ export class AgentRunFinisher { context.messages.push({ role: 'assistant', content: response.content }); return AgentRunFinisher.finish(context, 'done', response.content, { - logLevel: 'info', - logMessage: 'Agent run finished', + logging: { + logLevel: 'info', + logMessage: 'Agent run finished', + }, }); } static maxSteps(context: AgentRunContext): RunResult { return AgentRunFinisher.finish(context, 'max_steps', `Reached maximum step limit (${context.maxSteps})`, { - logLevel: 'warn', - logMessage: 'Budget exhausted', + logging: { + logLevel: 'warn', + logMessage: 'Budget exhausted', + }, }); } @@ -56,19 +62,23 @@ export class AgentRunFinisher { context: AgentRunContext, outcome: StopReason, summary: string, - logging?: FinishAgentRunLogging, + options: FinishAgentRunOptions = {}, ): RunResult { context.state.outcome = outcome; context.state.summary = summary; - if (logging) { - context.log[logging.logLevel]({ step: context.state.step, outcome, maxSteps: context.maxSteps }, logging.logMessage); + if (options.logging) { + context.log[options.logging.logLevel]( + { step: context.state.step, outcome, maxSteps: context.maxSteps }, + options.logging.logMessage, + ); } context.live.trace({ type: HeddleEventType.runFinished, outcome, summary, + ...(options.failure ? { failure: options.failure } : {}), step: context.state.step, timestamp: context.now(), }); @@ -76,6 +86,7 @@ export class AgentRunFinisher { return { outcome, summary, + ...(options.failure ? { failure: options.failure } : {}), trace: context.trace.getTrace(), transcript: context.messages.slice(1), usage: context.state.usage, diff --git a/src/core/agent/finish/types.ts b/src/core/agent/finish/types.ts index 4de1e675..3b074ec2 100644 --- a/src/core/agent/finish/types.ts +++ b/src/core/agent/finish/types.ts @@ -1,4 +1,4 @@ -import type { StopReason, RunResult } from '@/core/types.js'; +import type { RunFailure, StopReason, RunResult } from '@/core/types.js'; import type { LlmResponse } from '@/core/llm/types.js'; import type { AgentRunContext } from '../types.js'; @@ -7,11 +7,16 @@ export type FinishAgentRunLogging = { logMessage: string; }; +export type FinishAgentRunOptions = { + failure?: RunFailure; + logging?: FinishAgentRunLogging; +}; + export type FinishAgentRunArgs = { context: AgentRunContext; outcome: StopReason; summary: string; - logging?: FinishAgentRunLogging; + options?: FinishAgentRunOptions; }; export type FinishAssistantResponseArgs = { diff --git a/src/core/agent/model/README.md b/src/core/agent/model/README.md new file mode 100644 index 00000000..27c6a87e --- /dev/null +++ b/src/core/agent/model/README.md @@ -0,0 +1,39 @@ +# Agent Model Turn Service + +This folder owns one model request inside an agent run, including streaming, +usage accumulation, retry policy, and safe terminal failure classification. + +## Owns + +- Calling the configured `LlmAdapter` for the current turn. +- Retrying only model requests that are safe to repeat. Tool calls and whole + agent runs are never replayed by this service. +- Converting provider status and transport signals into the stable + host-facing `RunFailure` category. +- Replacing raw provider diagnostics with a stable safe message before the + value reaches logs, retry traces, or the human-readable run summary. + +## Boundary + +`AgentModelTurnRetryService` is the single owner of model-failure +classification. Product hosts should branch on `result.failure`, not parse +provider messages or reimplement HTTP-status maps. + +The failure contract deliberately contains only a source and code. Raw model +errors are used transiently for structured status and retry detection, then +discarded. Never add credentials, request bodies, response bodies, or provider +messages to the failure or retry contracts: they can flow through traces, +checkpoints, logs, live events, and remote API results. + +Provider-specific retry behavior belongs here when it can be derived from +structured adapter errors. Product-specific copy and HTTP/API error mapping +belong in the consuming host. + +## Adjacent Owners + +- `src/core/llm/` owns provider adapters and their raw response/error behavior. +- `src/core/agent/finish/` owns the final `RunResult` and `run.finished` trace. +- `src/core/runtime/loop/` propagates the finished result into loop state and + live activity. +- `src/core/chat/engine/turns/` exposes the safe category to programmatic + conversation hosts. diff --git a/src/core/agent/model/model-turn-retry-service.ts b/src/core/agent/model/model-turn-retry-service.ts index 40e4a458..4f1c19fc 100644 --- a/src/core/agent/model/model-turn-retry-service.ts +++ b/src/core/agent/model/model-turn-retry-service.ts @@ -3,12 +3,14 @@ import includes from 'lodash/includes.js'; import isNumber from 'lodash/isNumber.js'; import isString from 'lodash/isString.js'; import type { LlmResponse } from '@/core/llm/types.js'; +import type { ModelRunFailureCode, RunFailure } from '@/core/types.js'; export type AgentModelTurnRetryReason = 'transport_error' | 'empty_response'; export type AgentModelTurnRetryDecision = { retryable: boolean; reason?: AgentModelTurnRetryReason; + failure?: RunFailure; maxAttempts: number; message: string; }; @@ -24,6 +26,30 @@ const RETRY_MAX_DELAY_MS = 4_000; const RETRYABLE_STATUS_CODES = new Set([408, 409, 425, 429, 500, 502, 503, 504]); const NON_RETRYABLE_STATUS_CODES = new Set([400, 401, 403, 404, 422]); +const MODEL_FAILURE_BY_STATUS = new Map([ + [400, 'request'], + [401, 'authentication'], + [403, 'permission'], + [404, 'request'], + [408, 'transport'], + [409, 'transport'], + [422, 'request'], + [425, 'transport'], + [429, 'rate_limit'], + [500, 'transport'], + [502, 'transport'], + [503, 'transport'], + [504, 'transport'], +]); +const MODEL_FAILURE_MESSAGE = new Map([ + ['authentication', 'Model authentication failed'], + ['permission', 'Model access was denied'], + ['rate_limit', 'Model provider rate limit reached'], + ['request', 'Model request was rejected'], + ['transport', 'Model provider is temporarily unavailable'], + ['empty_response', 'Model returned an empty response'], + ['unknown', 'Model request failed'], +]); const RETRYABLE_ERROR_CODES = new Set([ 'ECONNRESET', @@ -82,38 +108,45 @@ export class AgentModelTurnRetryService { return { retryable: true, reason: 'empty_response', + failure: AgentModelTurnRetryService.modelFailure('empty_response'), maxAttempts: EMPTY_RESPONSE_RETRY_ATTEMPTS, message: 'Model returned an empty response', }; } private static resolveError(error: unknown): AgentModelTurnRetryDecision { - const message = AgentModelTurnRetryService.formatErrorMessage(error); + const providerMessage = AgentModelTurnRetryService.formatErrorMessage(error); const status = AgentModelTurnRetryService.readStatusCode(error); + const failureCode = status === undefined ? 'unknown' : MODEL_FAILURE_BY_STATUS.get(status) ?? 'unknown'; + const failure = AgentModelTurnRetryService.modelFailure(failureCode); + const message = AgentModelTurnRetryService.safeMessage(failureCode); if (status !== undefined && NON_RETRYABLE_STATUS_CODES.has(status)) { - return { retryable: false, maxAttempts: 1, message }; + return { retryable: false, failure, maxAttempts: 1, message }; } if (status !== undefined && RETRYABLE_STATUS_CODES.has(status)) { return { retryable: true, reason: 'transport_error', + failure, maxAttempts: TRANSPORT_RETRY_ATTEMPTS, message, }; } - if (AgentModelTurnRetryService.hasRetryableErrorCode(error) || AgentModelTurnRetryService.hasRetryableMessage(message)) { + if (AgentModelTurnRetryService.hasRetryableErrorCode(error) + || AgentModelTurnRetryService.hasRetryableMessage(providerMessage)) { return { retryable: true, reason: 'transport_error', + failure: AgentModelTurnRetryService.modelFailure('transport'), maxAttempts: TRANSPORT_RETRY_ATTEMPTS, - message, + message: AgentModelTurnRetryService.safeMessage('transport'), }; } - return { retryable: false, maxAttempts: 1, message }; + return { retryable: false, failure, maxAttempts: 1, message }; } private static readStatusCode(error: unknown): number | undefined { @@ -143,4 +176,12 @@ export class AgentModelTurnRetryService { private static formatErrorMessage(error: unknown): string { return error instanceof Error ? error.message : String(error); } + + private static modelFailure(code: ModelRunFailureCode): RunFailure { + return { source: 'model', code }; + } + + private static safeMessage(code: ModelRunFailureCode): string { + return MODEL_FAILURE_MESSAGE.get(code) ?? 'Model request failed'; + } } diff --git a/src/core/agent/model/model-turn-service.ts b/src/core/agent/model/model-turn-service.ts index 44706ffb..8833fdf1 100644 --- a/src/core/agent/model/model-turn-service.ts +++ b/src/core/agent/model/model-turn-service.ts @@ -35,7 +35,12 @@ export class AgentModelTurnService { const retry = AgentModelTurnRetryService.resolve({ kind: 'response', response }); if (!retry.retryable || attempt >= retry.maxAttempts) { return retry.retryable - ? AgentRunFinisher.finish(context, 'error', `${retry.message} after ${attempt} attempts`) + ? AgentRunFinisher.finish( + context, + 'error', + `${retry.message} after ${attempt} attempts`, + { failure: retry.failure }, + ) : response; } @@ -53,6 +58,7 @@ export class AgentModelTurnService { context, 'error', attempt > 1 ? `LLM error after ${attempt} attempts: ${retry.message}` : `LLM error: ${retry.message}`, + { failure: retry.failure }, ); } diff --git a/src/core/chat/engine/README.md b/src/core/chat/engine/README.md index be2f982d..ac3b8681 100644 --- a/src/core/chat/engine/README.md +++ b/src/core/chat/engine/README.md @@ -22,7 +22,7 @@ not each invent their own version. - Host-extension composition for SDK integrations, including curated MCP tool surfaces and artifact behavior. - Host-facing turn result summaries, including trace file, completed tool - results, and session artifacts. + results, session artifacts, and safe model-failure categories. ## Domain Ownership Rule @@ -94,6 +94,9 @@ Use this pattern when it fits the domain: - Put shared compaction behavior in `compaction/`. - Put public turn result vocabulary in `turn-result.ts` so SDK hosts do not parse trace or artifact storage for common summaries. +- Read `result.failure` for product error handling. Provider error + classification belongs to `src/core/agent/model/`; hosts must not parse + `result.summary` or duplicate provider status maps. - Put host-extension policy in `host-extension.ts` or focused helpers such as `mcp-host-extension.ts`; runtime toolkits should receive resolved policy, not re-read host-extension config. diff --git a/src/core/chat/engine/turn-result.ts b/src/core/chat/engine/turn-result.ts index be4918a9..34490ffa 100644 --- a/src/core/chat/engine/turn-result.ts +++ b/src/core/chat/engine/turn-result.ts @@ -1,5 +1,5 @@ import type { RuntimeArtifact } from '@/core/artifacts/index.js'; -import type { ToolCall, ToolResult } from '@/core/types.js'; +import type { RunFailure, ToolCall, ToolResult } from '@/core/types.js'; import type { ChatSession } from '@/core/chat/types.js'; export type ConversationTurnToolResult = { @@ -13,6 +13,7 @@ export type ConversationTurnToolResult = { export type ConversationTurnResultSummary = { outcome: string; summary: string; + failure?: RunFailure; session: ChatSession; traceFile?: string; artifacts: RuntimeArtifact[]; diff --git a/src/core/chat/engine/turns/service.ts b/src/core/chat/engine/turns/service.ts index c76527eb..510c1f56 100644 --- a/src/core/chat/engine/turns/service.ts +++ b/src/core/chat/engine/turns/service.ts @@ -180,6 +180,7 @@ export class EngineConversationTurnService implements ConversationTurnService { return { outcome: resultForPersistence.outcome, summary: persisted.summary, + ...(resultForPersistence.failure ? { failure: resultForPersistence.failure } : {}), session: persisted.session, traceFile: persisted.traceFile, artifacts: EngineConversationTurnService.listTurnArtifacts({ diff --git a/src/core/live/types.ts b/src/core/live/types.ts index bccd23dd..05906224 100644 --- a/src/core/live/types.ts +++ b/src/core/live/types.ts @@ -1,7 +1,7 @@ import type { LlmProvider, LlmUsage } from '@/core/llm/types.js'; import { HeddleEventType } from '@/core/event-types.js'; import type { AgentPlanState } from '@/core/agent/planning/index.js'; -import type { StopReason, ToolCall, ToolResult } from '@/core/types.js'; +import type { RunFailure, StopReason, ToolCall, ToolResult } from '@/core/types.js'; export type ConversationActivityCorrelation = { runId?: string; @@ -125,6 +125,7 @@ export type ConversationLoopFinishedActivity = { runId: string; outcome: StopReason; summary: string; + failure?: RunFailure; usage?: LlmUsage; timestamp: string; }; diff --git a/src/core/runtime/loop/checkpoint.ts b/src/core/runtime/loop/checkpoint.ts index 5bdbf08d..53005a46 100644 --- a/src/core/runtime/loop/checkpoint.ts +++ b/src/core/runtime/loop/checkpoint.ts @@ -27,6 +27,7 @@ export class AgentLoopCheckpointService { finishedAt: args.finishedAt, outcome: args.result.outcome, summary: args.result.summary, + ...(args.result.failure ? { failure: args.result.failure } : {}), usage: args.result.usage, transcript: args.result.transcript, trace: args.result.trace, diff --git a/src/core/runtime/loop/service.ts b/src/core/runtime/loop/service.ts index 1ce66172..421f7309 100644 --- a/src/core/runtime/loop/service.ts +++ b/src/core/runtime/loop/service.ts @@ -124,6 +124,7 @@ export class AgentLoopRuntimeService { runId, outcome: result.outcome, summary: result.summary, + ...(result.failure ? { failure: result.failure } : {}), usage: result.usage, state, timestamp: finishedAt, diff --git a/src/core/runtime/loop/types.ts b/src/core/runtime/loop/types.ts index 92aa4ee4..b0ddf8f6 100644 --- a/src/core/runtime/loop/types.ts +++ b/src/core/runtime/loop/types.ts @@ -14,7 +14,7 @@ import type { ConversationPlanUpdatedActivity, } from '@/core/live/index.js'; import type { ChatMessage, LlmAdapter, LlmProvider, LlmUsage, ReasoningEffort } from '@/core/llm/types.js'; -import type { RunResult, StopReason, ToolCall, ToolDefinition, TraceEvent } from '@/core/types.js'; +import type { RunFailure, RunResult, StopReason, ToolCall, ToolDefinition, TraceEvent } from '@/core/types.js'; export type AgentLoopStatus = 'finished'; @@ -29,6 +29,7 @@ export type AgentLoopState = { finishedAt: string; outcome: StopReason; summary: string; + failure?: RunFailure; usage?: LlmUsage; transcript: ChatMessage[]; trace: TraceEvent[]; diff --git a/src/core/types.ts b/src/core/types.ts index 972cc38e..e7c20469 100644 --- a/src/core/types.ts +++ b/src/core/types.ts @@ -60,6 +60,21 @@ export type ToolResult = { */ export type StopReason = 'done' | 'max_steps' | 'error' | 'interrupted'; +/** Safe host-facing classification for a failed run. Never contains provider messages or credentials. */ +export type ModelRunFailureCode = + | 'authentication' + | 'permission' + | 'rate_limit' + | 'request' + | 'transport' + | 'empty_response' + | 'unknown'; + +export type RunFailure = { + source: 'model'; + code: ModelRunFailureCode; +}; + /** * A single event in the run trace. Discriminated union on `type`. */ @@ -177,7 +192,14 @@ export type TraceEvent = metadata: Record; timestamp: string; } - | { type: typeof HeddleEventType.runFinished; outcome: StopReason; summary: string; step: number; timestamp: string }; + | { + type: typeof HeddleEventType.runFinished; + outcome: StopReason; + summary: string; + failure?: RunFailure; + step: number; + timestamp: string; + }; /** * The result returned from `AgentRunService.run`. @@ -185,6 +207,7 @@ export type TraceEvent = export type RunResult = { outcome: StopReason; summary: string; + failure?: RunFailure; trace: TraceEvent[]; transcript: ChatMessage[]; usage?: LlmUsage; diff --git a/src/index.ts b/src/index.ts index e4d7431a..9b181774 100644 --- a/src/index.ts +++ b/src/index.ts @@ -50,7 +50,9 @@ export { DEFAULT_OPENAI_MODEL, DEFAULT_ANTHROPIC_MODEL } from './core/config.js' // --------------------------------------------------------------------------- export type { RunInput, + RunFailure, RunResult, + ModelRunFailureCode, ToolDefinition, ToolCall, ToolResult,