[codex] Add LLM observability guards

agent/__tests__/llm-error-fallback.test.ts

@@ -23,4 +23,19 @@ describe("LLM error handling — no fabricated summaries", () => {
   it("imports agentLlmErrorTotal from metrics", () => {
     expect(serverSource).toContain("agentLlmErrorTotal");
   });
+
+  it("logs LLM errors with queryable structured fields", () => {
+    expect(serverSource).toContain('event: "llm_error"');
+    expect(serverSource).toContain("model: LLM_MODEL");
+    expect(serverSource).toContain("latency_ms");
+    expect(serverSource).toContain("requestId: getRequestId()");
+    expect(serverSource).toContain("error,");
+  });
+
+  it("records per-call LLM count and latency metrics", () => {
+    expect(serverSource).toContain("agentLlmCallsTotal.inc");
+    expect(serverSource).toContain("agentLlmLatencySeconds.observe");
+    expect(serverSource).toContain('status: "success"');
+    expect(serverSource).toContain('status: "error"');
+  });
 });

agent/__tests__/no-console.test.ts

@@ -0,0 +1,24 @@
+import { readdirSync, readFileSync, statSync } from "node:fs";
+import { join } from "node:path";
+import { describe, expect, it } from "vitest";
+
+function collectAgentSources(dir: string): string[] {
+  return readdirSync(dir).flatMap((entry) => {
+    const path = join(dir, entry);
+    const stat = statSync(path);
+    if (stat.isDirectory()) {
+      return entry === "__tests__" ? [] : collectAgentSources(path);
+    }
+    return /\.(ts|tsx|js|mjs|cjs)$/.test(entry) ? [path] : [];
+  });
+}
+
+describe("agent console logging guard", () => {
+  it("keeps agent runtime code on the structured logger", () => {
+    const offenders = collectAgentSources("agent").filter((path) =>
+      /\bconsole\./.test(readFileSync(path, "utf8")),
+    );
+
+    expect(offenders).toEqual([]);
+  });
+});

agent/server.ts

@@ -31,6 +31,9 @@ import {
   agentLlmTokensTotal,
   agentLlmIterationTokens,
   agentLlmContextUsageRatio,
+  agentLlmCallsTotal,
+  agentLlmLatencySeconds,
+  agentLlmErrorTotal,
 } from "../shared/metrics.ts";
 import {
   comparePharmacyPrices,
@@ -280,6 +283,16 @@ function calculateMaxTokens(iteration: number, toolCallCount: number, previousTo
   return LLM_MAX_TOKENS_SIMPLE; // 1024
 }
 
+function serializeLlmError(error: unknown): { name?: string; message: string } {
+  if (error instanceof Error) {
+    return { name: error.name, message: error.message };
+  }
+  if (typeof error === "object" && error !== null && "message" in error) {
+    return { message: String((error as { message: unknown }).message) };
+  }
+  return { message: String(error) };
+}
+
 // Run the agent with a task — full agentic loop
 async function runAgent(task: string) {
   const userTask = _scrubSession ? scrubText(task, _scrubSession) : task;
@@ -298,6 +311,7 @@ async function runAgent(task: string) {
 
   for (let iteration = 0; iteration < 15; iteration++) {
     let response;
+    const llmStartedAt = Date.now();
     try {
       // Determine temperature based on whether this is a tool-call round or final summary
       // Tool-call rounds use temperature=0 for deterministic tool selection
@@ -315,33 +329,36 @@ async function runAgent(task: string) {
         tools: LLM_TOOLS,
         messages,
       });
-    } catch (llmErr: any) {
-      logger.error({ err: llmErr.message, iteration }, "LLM API error");
+      const latencyMs = Date.now() - llmStartedAt;
+      agentLlmCallsTotal.inc({ model: LLM_MODEL, status: "success" });
+      agentLlmLatencySeconds.observe({ model: LLM_MODEL, status: "success" }, latencyMs / 1000);
+    } catch (llmErr: unknown) {
+      const latencyMs = Date.now() - llmStartedAt;
+      const error = serializeLlmError(llmErr);
+      agentLlmCallsTotal.inc({ model: LLM_MODEL, status: "error" });
+      agentLlmLatencySeconds.observe({ model: LLM_MODEL, status: "error" }, latencyMs / 1000);
+      logger.error(
+        {
+          event: "llm_error",
+          model: LLM_MODEL,
+          latency_ms: latencyMs,
+          requestId: getRequestId(),
+          iteration,
+          error,
+        },
+        "LLM API error",
+      );
       agentLlmErrorTotal.inc();
       finalResponse = JSON.stringify({
         status: "llm_error",
         toolCallsCompleted: toolCalls.length,
-        message: `LLM API error: ${llmErr.message}. Agent run was interrupted — not all tool calls may have completed.`,
+        message: `LLM API error: ${error.message}. Agent run was interrupted — not all tool calls may have completed.`,
         toolCalls: toolCalls.map(tc => ({
           tool: tc.tool,
           input: tc.input,
           result: tc.result,
         })),
       });
-      if (toolCalls.length > 0 && !finalResponse) {
-        finalResponse = toolCalls.map(tc => {
-          if (tc.result?.error) return `${tc.tool}: ${tc.result.error}`;
-          if (tc.result?.ok === false && tc.result?.reason) return `${tc.tool}: ${tc.result.reason}`;
-          if (tc.tool === "compare_pharmacy_prices" && (tc.result as any)?.cheapest) return `${(tc.result as any).drug}: cheapest at $${(tc.result as any).cheapest.price} (${(tc.result as any).cheapest.pharmacyName}), save $${(tc.result as any).potentialSavings}/mo`;
-          if (tc.tool === "audit_medical_bill" && (tc.result as any)?.totalOvercharge) return `Bill audit: $${(tc.result as any).totalOvercharge} in overcharges found (${(tc.result as any).errorCount} errors)`;
-          if (tc.tool === "check_drug_interactions" && (tc.result as any)?.summary) return (tc.result as any).summary;
-          if (tc.tool === "pay_for_medication" && (tc.result as any)?.success) return `Paid $${(tc.result as any).transaction.amount} for ${(tc.result as any).transaction.description}`;
-          if (tc.tool === "pay_bill" && (tc.result as any)?.success) return `Paid bill: $${(tc.result as any).transaction.amount}`;
-          return `${tc.tool}: completed`;
-        }).join("\n");
-      } else if (!finalResponse) {
-        finalResponse = `LLM error: ${llmErr.message}`;
-      }
       break;
     }
 

agent/tools.ts

@@ -1691,7 +1691,7 @@ export async function payBill(
     };
 
     let stellarTx = await buildStellarTx();
-    console.log(`  [Stellar] Signer verified: ${agentKeypair.publicKey().slice(0, 8)}...`);
+    logger.info({ signerPrefix: agentKeypair.publicKey().slice(0, 8) }, "[Stellar] Signer verified");
 
     const result = await submitTransactionWithRetry(horizonServer, stellarTx, 2, 35000, buildStellarTx);
 

docker-compose.override.yml

@@ -39,6 +39,7 @@ services:
       - "9090:9090"
     volumes:
       - ./docker/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - ./docker/prometheus/alerts.yml:/etc/prometheus/alerts.yml:ro
     # Optional: set METRICS_TOKEN in .env and Prometheus will send it as a bearer token.
     # See docker/prometheus/prometheus.yml for scrape_config bearer_token usage.
 

docker-compose.yml

@@ -88,6 +88,7 @@ services:
       - "--storage.tsdb.retention.time=7d"
     volumes:
       - ./docker/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - ./docker/prometheus/alerts.yml:/etc/prometheus/alerts.yml:ro
       - prometheus-data:/prometheus
     ports:
       - "9090:9090"

docker/prometheus/alerts.yml

@@ -0,0 +1,15 @@
+groups:
+  - name: careguard-agent-llm
+    rules:
+      - alert: CareGuardHighLlmErrorRate
+        expr: |
+          sum(rate(agent_llm_error_total[10m]))
+            /
+          clamp_min(sum(rate(agent_llm_calls_total[10m])), 1) > 0.05
+        for: 10m
+        labels:
+          severity: warning
+          service: careguard-agent
+        annotations:
+          summary: CareGuard LLM error rate is above 5%
+          description: More than 5% of agent LLM calls have failed over the last 10 minutes.

docker/prometheus/prometheus.yml

@@ -2,6 +2,9 @@ global:
   scrape_interval: 5s
   evaluation_interval: 5s
 
+rule_files:
+  - /etc/prometheus/alerts.yml
+
 scrape_configs:
   # When running via docker compose, the server is reachable as `server` on
   # the shared bridge network. For host-only Prometheus (no compose), uncomment

docs/agent/llm-config.md

@@ -156,9 +156,13 @@ WARN: LLM token usage exceeds 50% of budget threshold
 Check token consumption in metrics:
 ```
 agent_llm_tokens_total{kind="prompt"} / agent_llm_tokens_total{kind="completion"}
-agentLlmContextUsageRatio (warns at 80% of context window)
+agent_llm_context_usage_ratio (warns at 80% of context window)
+agent_llm_calls_total{model="llama-3.3-70b-versatile",status="success|error"}
+agent_llm_latency_seconds_bucket{model="llama-3.3-70b-versatile",status="success|error"}
 ```
 
+Prometheus also evaluates `CareGuardHighLlmErrorRate` from `docker/prometheus/alerts.yml`, which fires when `agent_llm_error_total / agent_llm_calls_total` stays above 5% for 10 minutes.
+
 Run token analysis:
 ```bash
-pnpm run check-llm-budget  # Shows token consumption by query type
+pnpm run check-llm-budget  # Shows token consumption by query type

eslint.agent.config.mjs

@@ -0,0 +1,20 @@
+import tseslint from "typescript-eslint";
+
+export default tseslint.config(
+  {
+    ignores: ["agent/__tests__/**"],
+  },
+  {
+    files: ["agent/**/*.{ts,tsx}"],
+    languageOptions: {
+      parser: tseslint.parser,
+      parserOptions: {
+        ecmaVersion: "latest",
+        sourceType: "module",
+      },
+    },
+    rules: {
+      "no-console": "error",
+    },
+  },
+);