From 63aea5c2f40ca20813cc3565638c106f38538ba2 Mon Sep 17 00:00:00 2001 From: Gary Pinkham Date: Tue, 15 Sep 2026 21:05:46 -0400 Subject: [PATCH] Cost Grok agents from the usage file their CLI already writes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A Grok agent ran fine and cost $0.00 forever. The telemetry env that makes an agent push OTel is injected for Claude Code alone (hive.ts `ensureAgent`), and a Grok agent writes no Claude transcript, so both of the collector's sources came back empty — no fleet numbers, and no row in cost-ledger.jsonl at all. On this machine that is 21,542 ledger rows, every one of them a Claude model, and none for the Grok agent that had been working alongside them. Its real numbers were on disk the whole time. The Grok CLI keeps a cumulative per-session snapshot at ~/.grok/sessions/// usage.json, and both halves of that path are already resolved: the Grok hook bridge normalizes its camelCase payload to `session_id` (GROK_HOOK_SHIM), which `recordSession` stores, so resolveCwd/resolveSessionId answer for a Grok agent exactly as they do for a Claude one. So this adds a reader, not any plumbing. Cost comes from the file too — `costUsdTicks`, 10^10 ticks to the dollar per Grok's own docs — so there are no Grok prices to maintain here. The dedup needs care. `appendCostLedger` is fed a running total, so re-appending an unchanged sample writes the same row forever; that is #56, which left 2,417 identical rows. The transcript fallback dodges it by reporting an empty session id to disqualify itself from the ledger, but this provider cannot — it needs a real session id to be accounted at all. Hence CumulativeSampleGate: admit a row only when the numbers move. An idle Grok agent re-reads the same file every beat and is correctly silent. `ts` is not in the signature, because when only the write time changed there is no new cost to record. The Claude path is untouched. `grokFallback` returns null for a Claude agent — its session id is not a directory under ~/.grok/sessions — and the ledger gate short-circuits on provider before the gate is ever consulted. Closes #535 Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01YSioheUKVkWkZ3aRxjYnaJ --- src/main/index.ts | 20 ++++- src/main/telemetry.ts | 69 +++++++++++++++- src/main/usage.ts | 38 +++++++++ test/grok-usage-ledger.test.cjs | 141 ++++++++++++++++++++++++++++++++ 4 files changed, 265 insertions(+), 3 deletions(-) create mode 100644 test/grok-usage-ledger.test.cjs diff --git a/src/main/index.ts b/src/main/index.ts index 40634ac6f..e80298c63 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -28,7 +28,7 @@ import { linkWorktreeDeps, unlinkWorktreeDeps } from './worktreeDeps'; import { HiveManager, type AgentMeta, type HiveMessage, type HiveTask } from './hive'; import { HookServer } from './hooks'; import { CircuitBreaker, type BreakerInput } from './breaker'; -import type { UsageProvider } from './usage'; +import { CumulativeSampleGate, type UsageProvider } from './usage'; import { MemoryManager } from './memory'; import { KnowledgeManager } from './knowledge'; import { MemoryReflector, type ReflectSettings } from './reflect'; @@ -260,6 +260,12 @@ const telemetry = new TelemetryCollector({ // untouched; telemetry has a transcript fallback built in, so it works before any // live OTel arrives. const usageProvider: UsageProvider = telemetry; +// Grok agents are costed from a cumulative file snapshot (telemetry.ts +// `grokFallback`), so an idle one re-reads identical totals every beat. Their +// session id is real, so the liveness gate below cannot filter that — this +// does, by admitting a row only when the numbers move. Claude's live OTel path +// does not consult it. +const grokLedgerGate = new CumulativeSampleGate(); // Circuit breaker (Lane A #6.6b) — the REAL policy (replaces Lane C's interim // glue). POLICY only; the heartbeat beat feeds it signals (via usageProvider) + // enforces its decisions. Config read live so a settings change applies next beat. @@ -454,6 +460,9 @@ function teardownPty(id: string): void { try { breaker.forget(agentId); } catch { /* best-effort */ } // A replacement using this id needs a new usage counter, not the dead PTY's. try { telemetry.forgetAgent(agentId); } catch { /* best-effort */ } + // Same reason, for the Grok ledger gate: a respawned agent's first sample + // must be admitted rather than matched against the dead one's last row. + try { grokLedgerGate.forget(agentId); } catch { /* best-effort */ } // W1 — kill this agent's proxy-bridge sidecar (qwen), if any, so a dead // PTY never leaves an orphan loopback listener. No-op for non-proxy agents. try { hive.stopProxyBridge(agentId); } catch (e) { console.error('[hive] stopProxyBridge failed:', e); } @@ -1212,7 +1221,14 @@ function runBreakerBeat(progressWindowMs: number): void { // (2,417 dupes observed). A truthy sessionId is set only by a live session // (aggregateLive picks the most-recent live session id), so this gates on // "is there a live session" without changing any live-agent behavior. - if (sample?.sessionId) hive.appendCostLedger(sample); // ledger covers everyone incl. god + if (sample?.sessionId) { + // A Grok sample's session id is always truthy, so for that provider #56's + // duplicate-row risk moves from "is there a live session" to "did anything + // change". Short-circuits before the gate for everyone else, leaving the + // live-OTel path exactly as it was. + const moved = a.provider !== 'grok' || grokLedgerGate.admits(sample); + if (moved) hive.appendCostLedger(sample); // ledger covers everyone incl. god + } // Second source for the resume key. recordSession() is otherwise reachable // ONLY from the hook shim, so any window where hooks don't land leaves the // registry with no sessionId and "Restart & Continue" refuses to continue — diff --git a/src/main/telemetry.ts b/src/main/telemetry.ts index 29dee3a2f..be9c1ced6 100644 --- a/src/main/telemetry.ts +++ b/src/main/telemetry.ts @@ -26,7 +26,10 @@ * the security boundary. Runs in the Electron main process; deliberately free of * any `electron` import so it can be smoke-tested as a plain Node module. */ +import { readFileSync } from 'node:fs'; import { createServer, type IncomingMessage, type Server, type ServerResponse } from 'node:http'; +import { homedir } from 'node:os'; +import { join } from 'node:path'; import { readAgentUsage } from './transcript'; import { normalizeModel } from './pricing'; @@ -188,7 +191,11 @@ export class TelemetryCollector { getAgentUsage(agentId: string): AgentUsageSample | null { const live = this.aggregateLive(agentId); if (live) return live; - return this.transcriptFallback(agentId); + // Grok before the transcript read: a hook-bridge agent has no Claude + // transcript to fall back TO, and its own file carries a real cost rather + // than an estimate. Returns null for everyone else, so the Claude path is + // reached unchanged. + return this.grokFallback(agentId) ?? this.transcriptFallback(agentId); } /** Push (additive, OTel-only). Fires the agent's fresh aggregate whenever new @@ -449,6 +456,66 @@ export class TelemetryCollector { }; } + /** + * Grok's own cumulative usage snapshot, for hook-bridge agents. + * + * A Grok agent never pushes OTel and has no Claude transcript to read, because + * the telemetry env is injected for Claude Code alone (hive.ts `ensureAgent`). + * So both existing sources return nothing and the agent costs $0.00 forever, + * while its real numbers sit on disk the whole time. The Grok CLI writes them + * per session at + * + * ~/.grok/sessions///usage.json + * + * and BOTH path parts are already resolved: the Grok hook bridge normalizes + * its camelCase payload to `session_id` (hive.ts GROK_HOOK_SHIM), which + * `recordSession` stores, so `resolveSessionId` and `resolveCwd` answer for a + * Grok agent exactly as they do for a Claude one. Nothing new is plumbed here + * — this only reads a file the CLI already maintains. + * + * Null for a Claude agent: its session id is not a directory under + * `~/.grok/sessions`, so the path does not exist and the read throws. + * + * Unlike `transcriptFallback`, this returns the REAL session id rather than + * ''. That is deliberate — an empty id is how the transcript path stays out of + * the ledger, and the whole point here is to get IN. The duplicate-row risk + * that '' was guarding against is handled where the row is written, by the + * delta gate in index.ts. + */ + private grokFallback(agentId: string): AgentUsageSample | null { + const cwd = this.resolveCwd?.(agentId); + const sessionId = this.resolveSessionId?.(agentId); + if (!cwd || !sessionId) return null; + const file = join(homedir(), '.grok', 'sessions', encodeURIComponent(cwd), sessionId, 'usage.json'); + try { + const parsed = JSON.parse(readFileSync(file, 'utf8')) as { + updatedAt?: unknown; + session?: Record; + }; + const totals = parsed.session; + if (!totals) return null; + const ts = Date.parse(String(parsed.updatedAt ?? '')); + return { + agentId, + sessionId, + ts: Number.isFinite(ts) ? ts : Date.now(), + input: numAttr(totals.inputTokens), + output: numAttr(totals.outputTokens), + cacheRead: numAttr(totals.cachedReadTokens), + cacheCreation: numAttr(totals.cacheCreationTokens), + model: normalizeModel(String(totals.primaryModelId ?? '')), + // Grok reports cost in integer ticks, 10^10 to the dollar (its own + // docs, user-guide/17-sessions.md). Taken as given, never recomputed — + // same rule the Claude live path follows for Claude's own figure. + usd: numAttr(totals.costUsdTicks) / 1e10 + }; + } catch { + // No file (any non-Grok agent), or it is mid-write. Both mean "no data + // from this source", which is what the next fallback is for. + return null; + } + } + private publishUsage(agentId: string): void { const sample = this.aggregateLive(agentId); if (!sample) return; diff --git a/src/main/usage.ts b/src/main/usage.ts index 0ce2a0a43..31e12d306 100644 --- a/src/main/usage.ts +++ b/src/main/usage.ts @@ -100,3 +100,41 @@ export class StubUsageProvider implements UsageProvider { }; } } + +/** + * Duplicate gate for CUMULATIVE file-snapshot samples (Grok). + * + * `appendCostLedger` is fed a running TOTAL, not an increment, so re-appending + * an unchanged sample writes the same row over and over. That is #56: the + * transcript fallback did exactly this and left 2,417 identical rows behind, + * which is why it now reports an empty `sessionId` to disqualify itself from + * the ledger. + * + * The Grok provider cannot use that trick — it needs a real session id to be + * accounted at all — so it needs this instead: append only when the numbers + * actually moved. An idle Grok agent re-reads the same `usage.json` on every + * beat and is correctly silent. + * + * `ts` is deliberately NOT part of the signature. It tracks when the file was + * written, not what it says, and a sample whose timestamp is the only thing to + * have changed carries no new cost. + */ +export class CumulativeSampleGate { + private readonly last = new Map(); + + /** True when this sample differs from the last one admitted for the agent. */ + admits(sample: AgentUsageSample): boolean { + const signature = [ + sample.sessionId, sample.input, sample.output, + sample.cacheRead, sample.cacheCreation, sample.model, sample.usd + ].join('|'); + if (this.last.get(sample.agentId) === signature) return false; + this.last.set(sample.agentId, signature); + return true; + } + + /** Drop an agent's memory (archived/despawned) so the map cannot grow forever. */ + forget(agentId: string): void { + this.last.delete(agentId); + } +} diff --git a/test/grok-usage-ledger.test.cjs b/test/grok-usage-ledger.test.cjs new file mode 100644 index 000000000..8fa7cc98a --- /dev/null +++ b/test/grok-usage-ledger.test.cjs @@ -0,0 +1,141 @@ +'use strict'; + +/** + * #535 — Grok agents run fine but never reach the cost ledger. + * + * The telemetry env that makes an agent push OTel is injected for Claude Code + * alone (hive.ts `ensureAgent`), and a Grok agent writes no Claude transcript, + * so both of the collector's sources came back empty and every Grok agent read + * as $0.00 / 0 tok forever — in fleet.json and by its total absence from + * cost-ledger.jsonl. Its real numbers were on disk the whole time, in the Grok + * CLI's own per-session `usage.json`. + * + * These cover the two halves of the fix: reading that file, and not writing the + * same row to the ledger over and over once it can be read (the cumulative + * snapshot is exactly the shape that produced #56's 2,417 duplicates). + * + * Sandboxes HOME so the path resolves into a throwaway dir rather than the + * developer's real ~/.grok (mirrors test/telemetry-session-fallback.test.cjs). + */ + +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); + +const FAKE_HOME = fs.mkdtempSync(path.join(os.tmpdir(), 'home-')); +process.env.HOME = FAKE_HOME; +process.env.USERPROFILE = FAKE_HOME; + +const loadTs = require('./load-ts.cjs'); +const { TelemetryCollector } = loadTs('src/main/telemetry.ts'); +const { CumulativeSampleGate } = loadTs('src/main/usage.ts'); + +const AGENT = 'ryan-mu1mcvvx'; +const CWD = '/Users/someone/dev/portfolio'; +const SESSION = '01a0a15a-0176-7f22-9a00-c81bcd4e3940'; + +/** The real shape `grok usage ` persists, trimmed to what we read. */ +function writeUsageFile(totals) { + const dir = path.join(FAKE_HOME, '.grok', 'sessions', encodeURIComponent(CWD), SESSION); + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync( + path.join(dir, 'usage.json'), + JSON.stringify({ + sessionId: SESSION, + updatedAt: '2026-09-14T20:23:52.899124+00:00', + session: { + inputTokens: 1782920, + outputTokens: 14247, + cachedReadTokens: 1524736, + cacheCreationTokens: 0, + reasoningTokens: 11404, + totalTokens: 1797167, + modelCalls: 24, + // 10^10 ticks to the dollar → $0.5388. + costUsdTicks: 5388341200, + primaryModelId: 'grok-4.6-build', + ...totals + } + }) + ); +} + +function collector({ cwd = CWD, sessionId = SESSION } = {}) { + return new TelemetryCollector({ + resolveCwd: () => cwd, + resolveSessionId: () => sessionId + }); +} + +test('a Grok agent is costed from the CLI’s own usage.json', () => { + writeUsageFile(); + + const sample = collector().getAgentUsage(AGENT); + + assert.ok(sample, 'no sample — the Grok agent would show $0.00 forever (#535)'); + assert.equal(sample.input, 1782920); + assert.equal(sample.output, 14247); + assert.equal(sample.cacheRead, 1524736); + assert.equal(sample.cacheCreation, 0); + assert.equal(sample.model, 'grok-4.6-build'); + // Ticks, not dollars: reading the raw number would bill this at $5.4 billion. + assert.equal(Number(sample.usd.toFixed(4)), 0.5388); +}); + +test('the sample carries a real session id, so the ledger accepts it', () => { + writeUsageFile(); + + const sample = collector().getAgentUsage(AGENT); + + // index.ts appends only `if (sample?.sessionId)`. The transcript fallback + // returns '' on purpose to stay out; this one has to get in. + assert.equal(sample.sessionId, SESSION); +}); + +test('no usage.json means no data, not a zeroed sample', () => { + // Every Claude agent takes this path: its session id is not a directory + // under ~/.grok/sessions, so the read throws and the next fallback runs. + const sample = collector({ sessionId: 'claude-session-with-no-grok-file' }).getAgentUsage(AGENT); + + assert.equal(sample, null); +}); + +test('an unknown cwd or session id is not guessed at', () => { + writeUsageFile(); + + // null, not undefined — undefined would take the helper's default and quietly + // assert nothing. + assert.equal(collector({ cwd: null }).getAgentUsage(AGENT), null); + assert.equal(collector({ sessionId: null }).getAgentUsage(AGENT), null); +}); + +test('an idle Grok agent does not re-append the same ledger row (#56)', () => { + writeUsageFile(); + const gate = new CumulativeSampleGate(); + const read = () => collector().getAgentUsage(AGENT); + + assert.equal(gate.admits(read()), true, 'first sample must be recorded'); + // The beat fires every ~30s whether or not the agent did anything. + assert.equal(gate.admits(read()), false); + assert.equal(gate.admits(read()), false); + + // The agent takes a turn; the totals move and the row is real again. + writeUsageFile({ outputTokens: 20000, costUsdTicks: 6000000000 }); + assert.equal(gate.admits(read()), true); + assert.equal(gate.admits(read()), false); +}); + +test('the gate keeps agents apart and forgets on request', () => { + writeUsageFile(); + const gate = new CumulativeSampleGate(); + const sample = collector().getAgentUsage(AGENT); + + assert.equal(gate.admits(sample), true); + assert.equal(gate.admits({ ...sample, agentId: 'someone-else' }), true, 'per-agent, not global'); + assert.equal(gate.admits(sample), false); + + gate.forget(AGENT); + assert.equal(gate.admits(sample), true, 'a respawned agent starts clean'); +});