diff --git a/plugins/admin/public/index.html b/plugins/admin/public/index.html index becbb6f75..84a71a317 100644 --- a/plugins/admin/public/index.html +++ b/plugins/admin/public/index.html @@ -8516,6 +8516,7 @@

Confirm governance change

const phaseSel = urlToState().phase; if (phaseSel) return renderPhaseDrill(root, d, phaseSel); renderCacheHealth(root, d); + renderSpend(root, d); const phases = d.phases || []; root.appendChild( openableTable( @@ -8560,6 +8561,20 @@

Confirm governance change

]), ); } + function renderSpend(root, d) { + const sp = d.spend; + if (!sp || !(sp.samples || 0)) return; + root.appendChild( + statline([ + fmtUsd(sp.costUsdTotal) + " model spend", + (sp.turnsWithKnownCost || 0) + " turns with known cost", + fmtTokens(sp.outputTokensTotal || 0) + " output tokens", + (sp.samples || 0) + " turns with spend data", + ]), + ); + } + const fmtUsd = (n) => + n == null ? "—" : "$" + (Math.round((n || 0) * 100) / 100).toFixed(2); const fmtX = (n) => (n == null ? "—" : String(Math.round(n * 10) / 10)); function renderTurnMix(root, d) { const an = d.anatomy; diff --git a/src/admin/metrics-sink.ts b/src/admin/metrics-sink.ts index df9cb1743..8dfba6411 100644 --- a/src/admin/metrics-sink.ts +++ b/src/admin/metrics-sink.ts @@ -36,6 +36,8 @@ export interface TurnMetricSample { cacheRead?: number; cacheWrite?: number; uncachedInput?: number; + outputTokens?: number; + costUsd?: number; } export function cacheHitRatio(s: { cacheRead?: number; cacheWrite?: number; uncachedInput?: number }): number | null { diff --git a/src/admin/postgres-metrics-sink.ts b/src/admin/postgres-metrics-sink.ts index 29187f828..57bb48161 100644 --- a/src/admin/postgres-metrics-sink.ts +++ b/src/admin/postgres-metrics-sink.ts @@ -37,6 +37,8 @@ const COLUMNS: readonly EventColumn[] = [ ["cache_read", "cacheRead", "BIGINT", "number"], ["cache_write", "cacheWrite", "BIGINT", "number"], ["uncached_input", "uncachedInput", "BIGINT", "number"], + ["output_tokens", "outputTokens", "BIGINT", "number"], + ["cost_usd", "costUsd", "DOUBLE PRECISION", "number"], ]; const EXTRA_SCHEMA_STATEMENTS = [ diff --git a/src/api/routes/admin/observability.ts b/src/api/routes/admin/observability.ts index aee13d633..e856cec04 100644 --- a/src/api/routes/admin/observability.ts +++ b/src/api/routes/admin/observability.ts @@ -150,6 +150,18 @@ export async function metrics(ctx: ApiCtx): Promise { uncachedInputTotal, }; + // Turn-level spend, straight off the usage the harnesses already meter: + // output tokens and provider-computed cost land on turn_metrics next to + // the cache telemetry, so an operator no longer joins session_llm_requests + // by hand to know what a scope or model spends. + const costSamples = samples.filter((s) => s.costUsd !== undefined || s.outputTokens !== undefined); + const spend = { + samples: costSamples.length, + turnsWithKnownCost: samples.filter((s) => s.costUsd !== undefined).length, + costUsdTotal: samples.reduce((n, s) => n + (s.costUsd ?? 0), 0), + outputTokensTotal: samples.reduce((n, s) => n + (s.outputTokens ?? 0), 0), + }; + const phaseFields: [string, (s: TurnMetricSample) => number | undefined][] = [ ["total", (s) => s.totalMs], ["ttft", (s) => s.ttftMs], @@ -225,6 +237,7 @@ export async function metrics(ctx: ApiCtx): Promise { series, anatomy, cache, + spend, phases, }); } diff --git a/src/core/orchestrator.ts b/src/core/orchestrator.ts index 3a17db7ab..a5e86ba4f 100644 --- a/src/core/orchestrator.ts +++ b/src/core/orchestrator.ts @@ -2796,6 +2796,9 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator { uncachedInput: result.cacheUsage.uncachedInput, } : {}), + ...(result.costUsage + ? { outputTokens: result.costUsage.outputTokens, costUsd: result.costUsage.costUsd } + : {}), }); const onTurnEnd = memoryStrategy.onTurnEnd?.bind(memoryStrategy); if (!pausing && useMemory && memoryPolicy.capture !== "off" && onTurnEnd) { diff --git a/src/harness/claude-harness.ts b/src/harness/claude-harness.ts index 2edbb26eb..8d09c77b3 100644 --- a/src/harness/claude-harness.ts +++ b/src/harness/claude-harness.ts @@ -764,11 +764,12 @@ export function createClaudeHarness(opts: ClaudeHarnessOptions = {}): Harness { const usageTotals = [...callUsage.values()].reduce( (acc, usage) => { acc.input += usage.input; + acc.output += usage.output; acc.cacheRead += usage.cacheRead; acc.cacheWrite += usage.cacheWrite; return acc; }, - { input: 0, cacheRead: 0, cacheWrite: 0 }, + { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, ); return { reply, @@ -793,6 +794,12 @@ export function createClaudeHarness(opts: ClaudeHarnessOptions = {}): Harness { finalResult.usage.cache_creation_input_tokens, ), }, + // The SDK result carries no cost total, so spend is surfaced only + // when the per-message accumulator saw cost deltas; the fallback + // branch keeps output tokens but reports no known cost. + costUsage: callUsage.size + ? { outputTokens: usageTotals.output, costUsd: lastTotalCostUsd } + : { outputTokens: finalResult.usage.output_tokens ?? 0, costUsd: 0 }, ...(tapeWriteFailed ? { tapeWriteFailed: true } : {}), }; } finally { diff --git a/src/harness/harness.ts b/src/harness/harness.ts index 77940e80a..018e0b6d0 100644 --- a/src/harness/harness.ts +++ b/src/harness/harness.ts @@ -109,6 +109,7 @@ export interface HarnessTurnResult { pausedOnApproval?: boolean; modelCalls?: number; cacheUsage?: { cacheRead: number; cacheWrite: number; uncachedInput: number }; + costUsage?: { outputTokens: number; costUsd: number }; compileMs?: number; tapeWriteFailed?: boolean; } diff --git a/src/harness/mock-harness.ts b/src/harness/mock-harness.ts index 319732612..36ca80574 100644 --- a/src/harness/mock-harness.ts +++ b/src/harness/mock-harness.ts @@ -682,6 +682,10 @@ export function createMockHarness(): Harness { }), { cacheRead: 0, cacheWrite: 0, uncachedInput: 0 }, ); + const costUsage = steps.reduce( + (acc, u) => ({ outputTokens: acc.outputTokens + u.output, costUsd: acc.costUsd + u.costUsd }), + { outputTokens: 0, costUsd: 0 }, + ); const base = collected.length ? { reply, @@ -691,7 +695,7 @@ export function createMockHarness(): Harness { } : { reply, modelCalls }; if (muteReply) base.reply = ""; - return { ...base, ...(silent ? { silent: true as const } : {}), cacheUsage }; + return { ...base, ...(silent ? { silent: true as const } : {}), cacheUsage, costUsage }; }, shouldRespond(detect: HarnessDetectInput): Promise { diff --git a/src/harness/pi-harness.ts b/src/harness/pi-harness.ts index 73ff2ba74..6d86e5758 100644 --- a/src/harness/pi-harness.ts +++ b/src/harness/pi-harness.ts @@ -574,6 +574,21 @@ function piUsageToCallUsage(u: PiUsageShape | undefined): LlmCallUsage | null { }; } +function sumCostUsage( + stats: ReadonlyArray<{ usage: LlmCallUsage | null }>, +): { outputTokens: number; costUsd: number } | null { + let saw = false; + let outputTokens = 0; + let costUsd = 0; + for (const s of stats) { + if (!s.usage) continue; + saw = true; + outputTokens += s.usage.output; + costUsd += s.usage.costUsd; + } + return saw ? { outputTokens, costUsd } : null; +} + function sumCacheUsage( stats: ReadonlyArray<{ usage: LlmCallUsage | null }>, ): { cacheRead: number; cacheWrite: number; uncachedInput: number } | null { @@ -1989,6 +2004,7 @@ export function createPiHarness(opts?: PiHarnessOptions): Harness { }); await checkpointSubturn(finalEntry.seq); const cacheUsage = sumCacheUsage(callStats); + const costUsage = sumCostUsage(callStats); const base = { reply, stopped: true as const, @@ -1996,7 +2012,7 @@ export function createPiHarness(opts?: PiHarnessOptions): Harness { compileMs, ...(tapeWriteFailed ? { tapeWriteFailed: true } : {}), }; - return cacheUsage ? { ...base, cacheUsage } : base; + return { ...base, ...(cacheUsage ? { cacheUsage } : {}), ...(costUsage ? { costUsage } : {}) }; } const closingText = recoveryDead ? "" : (piLastAssistantTextOrThrow(entry.agentSession) ?? ""); const closingTextWithWaiver = [closingText, grindWaiverNote].filter(Boolean).join("\n\n"); @@ -2011,6 +2027,7 @@ export function createPiHarness(opts?: PiHarnessOptions): Harness { const pendingApprovals = entry.ref.pendingApprovals ?? []; const modelCalls = entry.ref.modelCalls ?? 0; const cacheUsage = sumCacheUsage(callStats); + const costUsage = sumCostUsage(callStats); const silent = entry.ref.silentRequested ? { silent: true as const } : {}; const base = pendingApprovals.length ? { @@ -2023,7 +2040,7 @@ export function createPiHarness(opts?: PiHarnessOptions): Harness { compileMs, } : { reply, ...(tapeWriteFailed ? { tapeWriteFailed: true } : {}), modelCalls, ...silent, compileMs }; - return cacheUsage ? { ...base, cacheUsage } : base; + return { ...base, ...(cacheUsage ? { cacheUsage } : {}), ...(costUsage ? { costUsage } : {}) }; } finally { removeIsolatedDirs(entry); } diff --git a/test/turn-metrics-cost.test.ts b/test/turn-metrics-cost.test.ts new file mode 100644 index 000000000..215e29ea3 --- /dev/null +++ b/test/turn-metrics-cost.test.ts @@ -0,0 +1,79 @@ +import "./support/auto-fake-sprites.ts"; + +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { AddressInfo } from "node:net"; +import { createInsecureTestServer } from "../src/api/server.ts"; +import { buildApp } from "../src/wiring.ts"; +import type { TurnRequest } from "../src/types.ts"; +import { testConfig } from "./support/test-config.ts"; +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; + +function start() { + const built = buildApp(testConfig({ dataDir: mkdtempSync(join(tmpdir(), "cost-obs-")) })); + const server = createInsecureTestServer(built.app, { + admin: built.admin, + sessions: built.sessions, + auditLog: built.auditLog, + metrics: built.metrics, + runs: built.runs, + }); + server.listen(0); + const base = `http://localhost:${(server.address() as AddressInfo).port}`; + return { base, built, close: () => new Promise((r) => server.close(() => r())) }; +} + +const ALICE = { "x-admin-actor": "admin-alice@default-org" }; +const getJson = async (base: string, path: string, headers: Record = ALICE): Promise => + (await fetch(base + path, { headers })).json(); + +test("metrics: turn spend (output tokens + cost) is surfaced on the metrics endpoint", async () => { + const s = start(); + try { + const turn: TurnRequest = { + surface: "test", + actor: { externalId: "U1" }, + conversation: { kind: "dm", threadRef: "dm:U1:c1" }, + text: "hello there", + }; + assert.equal((await s.built.app.turn(turn)).status, "ok"); + + const m = await getJson(s.base, "/v1/admin/metrics?scope=org:default-org"); + assert.ok(m.spend, "the metrics response carries a spend block"); + assert.ok(m.spend.samples >= 1, "the turn carried spend telemetry"); + assert.ok(m.spend.turnsWithKnownCost >= 1, "the mock harness reports known (if zero) cost"); + assert.ok(m.spend.outputTokensTotal > 0, "output tokens are summed per turn"); + assert.equal(typeof m.spend.costUsdTotal, "number", "cost total is numeric even when the mock reports 0"); + } finally { + await s.close(); + } +}); + +test("metrics: an empty scope yields a present-but-empty spend block", async () => { + const s = start(); + try { + const m = await getJson(s.base, "/v1/admin/metrics?scope=personal:nobody"); + assert.ok(m.spend, "spend block is always present"); + assert.equal(m.spend.samples, 0); + assert.equal(m.spend.turnsWithKnownCost, 0); + assert.equal(m.spend.outputTokensTotal, 0); + assert.equal(m.spend.costUsdTotal, 0); + } finally { + await s.close(); + } +}); + +test("metrics viewer: the spend line renders cost and output totals beside cache health", () => { + const html = readFileSync(fileURLToPath(new URL("../plugins/admin/public/index.html", import.meta.url)), "utf8").split(String.fromCharCode(13, 10)).join(String.fromCharCode(10)); + assert.ok(html.includes("function renderSpend(root, d)"), "viewer has a spend renderer"); + assert.ok( + html.includes("renderCacheHealth(root, d);" + String.fromCharCode(10) + " renderSpend(root, d);"), + "the spend line renders directly beside cache health on the metrics page", + ); + assert.ok(html.includes("fmtUsd(sp.costUsdTotal)"), "cost total is formatted as currency"); + assert.ok(html.includes("fmtTokens(sp.outputTokensTotal || 0)"), "output tokens are formatted"); +});