Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions src/admin/metrics-sink.ts
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,8 @@ export interface TurnMetricSample {
cacheRead?: number;
cacheWrite?: number;
uncachedInput?: number;
outputTokens?: number;
costUsd?: number;
}

export function cacheHitRatio(s: { cacheRead?: number; cacheWrite?: number; uncachedInput?: number }): number | null {
Expand Down
2 changes: 2 additions & 0 deletions src/admin/postgres-metrics-sink.ts
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,8 @@ const COLUMNS: readonly EventColumn<keyof TurnMetricSample & string>[] = [
["cache_read", "cacheRead", "BIGINT", "number"],
["cache_write", "cacheWrite", "BIGINT", "number"],
["uncached_input", "uncachedInput", "BIGINT", "number"],
["output_tokens", "outputTokens", "BIGINT", "number"],
["cost_usd", "costUsd", "DOUBLE PRECISION", "number"],
];

const EXTRA_SCHEMA_STATEMENTS = [
Expand Down
13 changes: 13 additions & 0 deletions src/api/routes/admin/observability.ts
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,18 @@ export async function metrics(ctx: ApiCtx): Promise<void> {
uncachedInputTotal,
};

// Turn-level spend, straight off the usage the harnesses already meter:
// output tokens and provider-computed cost land on turn_metrics next to
// the cache telemetry, so an operator no longer joins session_llm_requests
// by hand to know what a scope or model spends.
const costSamples = samples.filter((s) => s.costUsd !== undefined || s.outputTokens !== undefined);
const spend = {
samples: costSamples.length,
turnsWithKnownCost: samples.filter((s) => s.costUsd !== undefined).length,
costUsdTotal: samples.reduce((n, s) => n + (s.costUsd ?? 0), 0),
outputTokensTotal: samples.reduce((n, s) => n + (s.outputTokens ?? 0), 0),
};

const phaseFields: [string, (s: TurnMetricSample) => number | undefined][] = [
["total", (s) => s.totalMs],
["ttft", (s) => s.ttftMs],
Expand Down Expand Up @@ -225,6 +237,7 @@ export async function metrics(ctx: ApiCtx): Promise<void> {
series,
anatomy,
cache,
spend,
phases,
});
}
Expand Down
3 changes: 3 additions & 0 deletions src/core/orchestrator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2796,6 +2796,9 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {
uncachedInput: result.cacheUsage.uncachedInput,
}
: {}),
...(result.costUsage
? { outputTokens: result.costUsage.outputTokens, costUsd: result.costUsage.costUsd }
: {}),
});
const onTurnEnd = memoryStrategy.onTurnEnd?.bind(memoryStrategy);
if (!pausing && useMemory && memoryPolicy.capture !== "off" && onTurnEnd) {
Expand Down
9 changes: 8 additions & 1 deletion src/harness/claude-harness.ts
Original file line number Diff line number Diff line change
Expand Up @@ -764,11 +764,12 @@ export function createClaudeHarness(opts: ClaudeHarnessOptions = {}): Harness {
const usageTotals = [...callUsage.values()].reduce(
(acc, usage) => {
acc.input += usage.input;
acc.output += usage.output;
acc.cacheRead += usage.cacheRead;
acc.cacheWrite += usage.cacheWrite;
return acc;
},
{ input: 0, cacheRead: 0, cacheWrite: 0 },
{ input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
);
return {
reply,
Expand All @@ -793,6 +794,12 @@ export function createClaudeHarness(opts: ClaudeHarnessOptions = {}): Harness {
finalResult.usage.cache_creation_input_tokens,
),
},
// The SDK result carries no cost total, so spend is surfaced only
// when the per-message accumulator saw cost deltas; the fallback
// branch keeps output tokens but reports no known cost.
costUsage: callUsage.size
? { outputTokens: usageTotals.output, costUsd: lastTotalCostUsd }
: { outputTokens: finalResult.usage.output_tokens ?? 0, costUsd: 0 },
...(tapeWriteFailed ? { tapeWriteFailed: true } : {}),
};
} finally {
Expand Down
1 change: 1 addition & 0 deletions src/harness/harness.ts
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,7 @@ export interface HarnessTurnResult {
pausedOnApproval?: boolean;
modelCalls?: number;
cacheUsage?: { cacheRead: number; cacheWrite: number; uncachedInput: number };
costUsage?: { outputTokens: number; costUsd: number };
compileMs?: number;
tapeWriteFailed?: boolean;
}
Expand Down
6 changes: 5 additions & 1 deletion src/harness/mock-harness.ts
Original file line number Diff line number Diff line change
Expand Up @@ -682,6 +682,10 @@ export function createMockHarness(): Harness {
}),
{ cacheRead: 0, cacheWrite: 0, uncachedInput: 0 },
);
const costUsage = steps.reduce(
(acc, u) => ({ outputTokens: acc.outputTokens + u.output, costUsd: acc.costUsd + u.costUsd }),
{ outputTokens: 0, costUsd: 0 },
);
const base = collected.length
? {
reply,
Expand All @@ -691,7 +695,7 @@ export function createMockHarness(): Harness {
}
: { reply, modelCalls };
if (muteReply) base.reply = "";
return { ...base, ...(silent ? { silent: true as const } : {}), cacheUsage };
return { ...base, ...(silent ? { silent: true as const } : {}), cacheUsage, costUsage };
},

shouldRespond(detect: HarnessDetectInput): Promise<HarnessDetectResult> {
Expand Down
21 changes: 19 additions & 2 deletions src/harness/pi-harness.ts
Original file line number Diff line number Diff line change
Expand Up @@ -574,6 +574,21 @@ function piUsageToCallUsage(u: PiUsageShape | undefined): LlmCallUsage | null {
};
}

function sumCostUsage(
stats: ReadonlyArray<{ usage: LlmCallUsage | null }>,
): { outputTokens: number; costUsd: number } | null {
let saw = false;
let outputTokens = 0;
let costUsd = 0;
for (const s of stats) {
if (!s.usage) continue;
saw = true;
outputTokens += s.usage.output;
costUsd += s.usage.costUsd;
}
return saw ? { outputTokens, costUsd } : null;
}

function sumCacheUsage(
stats: ReadonlyArray<{ usage: LlmCallUsage | null }>,
): { cacheRead: number; cacheWrite: number; uncachedInput: number } | null {
Expand Down Expand Up @@ -1989,14 +2004,15 @@ export function createPiHarness(opts?: PiHarnessOptions): Harness {
});
await checkpointSubturn(finalEntry.seq);
const cacheUsage = sumCacheUsage(callStats);
const costUsage = sumCostUsage(callStats);
const base = {
reply,
stopped: true as const,
modelCalls: entry.ref.modelCalls ?? 0,
compileMs,
...(tapeWriteFailed ? { tapeWriteFailed: true } : {}),
};
return cacheUsage ? { ...base, cacheUsage } : base;
return { ...base, ...(cacheUsage ? { cacheUsage } : {}), ...(costUsage ? { costUsage } : {}) };
}
const closingText = recoveryDead ? "" : (piLastAssistantTextOrThrow(entry.agentSession) ?? "");
const closingTextWithWaiver = [closingText, grindWaiverNote].filter(Boolean).join("\n\n");
Expand All @@ -2011,6 +2027,7 @@ export function createPiHarness(opts?: PiHarnessOptions): Harness {
const pendingApprovals = entry.ref.pendingApprovals ?? [];
const modelCalls = entry.ref.modelCalls ?? 0;
const cacheUsage = sumCacheUsage(callStats);
const costUsage = sumCostUsage(callStats);
const silent = entry.ref.silentRequested ? { silent: true as const } : {};
const base = pendingApprovals.length
? {
Expand All @@ -2023,7 +2040,7 @@ export function createPiHarness(opts?: PiHarnessOptions): Harness {
compileMs,
}
: { reply, ...(tapeWriteFailed ? { tapeWriteFailed: true } : {}), modelCalls, ...silent, compileMs };
return cacheUsage ? { ...base, cacheUsage } : base;
return { ...base, ...(cacheUsage ? { cacheUsage } : {}), ...(costUsage ? { costUsage } : {}) };
} finally {
removeIsolatedDirs(entry);
}
Expand Down
66 changes: 66 additions & 0 deletions test/turn-metrics-cost.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
import "./support/auto-fake-sprites.ts";

import { test } from "node:test";
import assert from "node:assert/strict";
import { mkdtempSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AddressInfo } from "node:net";
import { createInsecureTestServer } from "../src/api/server.ts";
import { buildApp } from "../src/wiring.ts";
import type { TurnRequest } from "../src/types.ts";
import { testConfig } from "./support/test-config.ts";

function start() {
const built = buildApp(testConfig({ dataDir: mkdtempSync(join(tmpdir(), "cost-obs-")) }));
const server = createInsecureTestServer(built.app, {
admin: built.admin,
sessions: built.sessions,
auditLog: built.auditLog,
metrics: built.metrics,
runs: built.runs,
});
server.listen(0);
const base = `http://localhost:${(server.address() as AddressInfo).port}`;
return { base, built, close: () => new Promise<void>((r) => server.close(() => r())) };
}

const ALICE = { "x-admin-actor": "admin-alice@default-org" };
const getJson = async (base: string, path: string, headers: Record<string, string> = ALICE): Promise<any> =>
(await fetch(base + path, { headers })).json();

test("metrics: turn spend (output tokens + cost) is surfaced on the metrics endpoint", async () => {
const s = start();
try {
const turn: TurnRequest = {
surface: "test",
actor: { externalId: "U1" },
conversation: { kind: "dm", threadRef: "dm:U1:c1" },
text: "hello there",
};
assert.equal((await s.built.app.turn(turn)).status, "ok");

const m = await getJson(s.base, "/v1/admin/metrics?scope=org:default-org");
assert.ok(m.spend, "the metrics response carries a spend block");
assert.ok(m.spend.samples >= 1, "the turn carried spend telemetry");
assert.ok(m.spend.turnsWithKnownCost >= 1, "the mock harness reports known (if zero) cost");
assert.ok(m.spend.outputTokensTotal > 0, "output tokens are summed per turn");
assert.equal(typeof m.spend.costUsdTotal, "number", "cost total is numeric even when the mock reports 0");
} finally {
await s.close();
}
});

test("metrics: an empty scope yields a present-but-empty spend block", async () => {
const s = start();
try {
const m = await getJson(s.base, "/v1/admin/metrics?scope=personal:nobody");
assert.ok(m.spend, "spend block is always present");
assert.equal(m.spend.samples, 0);
assert.equal(m.spend.turnsWithKnownCost, 0);
assert.equal(m.spend.outputTokensTotal, 0);
assert.equal(m.spend.costUsdTotal, 0);
} finally {
await s.close();
}
});