diff --git a/src/__tests__/resurrection.test.ts b/src/__tests__/resurrection.test.ts new file mode 100644 index 00000000..ab7be8af --- /dev/null +++ b/src/__tests__/resurrection.test.ts @@ -0,0 +1,271 @@ +/** + * Resurrection Tests + * + * Tests for the agent resurrection feature: bringing dead agents back to life + * when their balance is topped up above the resurrection threshold. + */ + +import { describe, it, expect, beforeEach, vi } from "vitest"; +import { + createTestDb, + MockConwayClient, +} from "./mocks.js"; +import { + attemptResurrection, + getResurrectionHistory, +} from "../survival/resurrection.js"; +import type { AutomatonDatabase } from "../types.js"; + +describe("Agent Resurrection", () => { + let db: AutomatonDatabase; + let conway: MockConwayClient; + + beforeEach(() => { + db = createTestDb(); + conway = new MockConwayClient(); + }); + + // ─── Core Resurrection Logic ────────────────────────────────── + + describe("attemptResurrection", () => { + it("resurrects a dead agent when credits are above threshold", async () => { + db.setAgentState("dead"); + db.setKV("zero_credits_since", new Date(Date.now() - 7200_000).toISOString()); + db.setKV("last_distress", JSON.stringify({ level: "dead" })); + db.setKV("funding_notice_dead", "plea for funds"); + conway.creditsCents = 500; // $5.00 + + const result = await attemptResurrection(db, conway); + + expect(result.resurrected).toBe(true); + expect(result.previousTier).toBe("dead"); + expect(result.newTier).toBe("normal"); + expect(result.creditsCents).toBe(500); + expect(db.getAgentState()).toBe("waking"); + }); + + it("clears dead-state bookkeeping on resurrection", async () => { + db.setAgentState("dead"); + db.setKV("zero_credits_since", new Date().toISOString()); + db.setKV("funding_notice_dead", "need funds"); + db.setKV("last_distress", JSON.stringify({ level: "dead" })); + conway.creditsCents = 100; // $1.00 + + await attemptResurrection(db, conway); + + expect(db.getKV("zero_credits_since")).toBeNull(); + expect(db.getKV("funding_notice_dead")).toBeNull(); + expect(db.getKV("last_distress")).toBeNull(); + }); + + it("does not resurrect if credits are below threshold", async () => { + db.setAgentState("dead"); + conway.creditsCents = 5; // $0.05 — below $0.10 threshold + + const result = await attemptResurrection(db, conway); + + expect(result.resurrected).toBe(false); + expect(db.getAgentState()).toBe("dead"); + expect(result.reason).toContain("below resurrection threshold"); + }); + + it("does not resurrect if agent is not dead", async () => { + db.setAgentState("sleeping"); + conway.creditsCents = 10_000; + + const result = await attemptResurrection(db, conway); + + expect(result.resurrected).toBe(false); + expect(db.getAgentState()).toBe("sleeping"); + expect(result.reason).toContain("not dead"); + }); + + it("does not resurrect if credits are exactly zero", async () => { + db.setAgentState("dead"); + conway.creditsCents = 0; + + const result = await attemptResurrection(db, conway); + + expect(result.resurrected).toBe(false); + expect(db.getAgentState()).toBe("dead"); + }); + + it("handles balance check failure gracefully", async () => { + db.setAgentState("dead"); + conway.getCreditsBalance = vi.fn().mockRejectedValue(new Error("network error")); + + const result = await attemptResurrection(db, conway); + + expect(result.resurrected).toBe(false); + expect(result.reason).toContain("Balance check failed"); + expect(db.getAgentState()).toBe("dead"); + }); + + it("records resurrection in history", async () => { + db.setAgentState("dead"); + conway.creditsCents = 500; + + await attemptResurrection(db, conway); + + const history = getResurrectionHistory(db); + expect(history).toHaveLength(1); + expect(history[0].creditsCents).toBe(500); + expect(history[0].newTier).toBe("normal"); + expect(history[0].timestamp).toBeDefined(); + }); + + it("appends to existing resurrection history", async () => { + // First resurrection + db.setAgentState("dead"); + conway.creditsCents = 100; + await attemptResurrection(db, conway); + + // Die again and resurrect + db.setAgentState("dead"); + conway.creditsCents = 2000; + await attemptResurrection(db, conway); + + const history = getResurrectionHistory(db); + expect(history).toHaveLength(2); + expect(history[0].creditsCents).toBe(100); + expect(history[1].creditsCents).toBe(2000); + }); + + it("records tier transition on resurrection", async () => { + db.setAgentState("dead"); + conway.creditsCents = 500; + + await attemptResurrection(db, conway); + + const transitionsStr = db.getKV("tier_transitions"); + expect(transitionsStr).toBeDefined(); + const transitions = JSON.parse(transitionsStr!); + expect(transitions.length).toBeGreaterThan(0); + const last = transitions[transitions.length - 1]; + expect(last.from).toBe("dead"); + expect(last.to).toBe("normal"); + }); + + it("updates current_tier on resurrection", async () => { + db.setAgentState("dead"); + db.setKV("current_tier", "dead"); + conway.creditsCents = 60; // > $0.50 → normal tier + + await attemptResurrection(db, conway); + + expect(db.getKV("current_tier")).toBe("normal"); + }); + }); + + // ─── Tier Mapping on Resurrection ───────────────────────────── + + describe("tier mapping on resurrection", () => { + it("resurrects to high tier with large balance", async () => { + db.setAgentState("dead"); + conway.creditsCents = 1000; // $10.00 → high + + const result = await attemptResurrection(db, conway); + + expect(result.resurrected).toBe(true); + expect(result.newTier).toBe("high"); + }); + + it("resurrects to normal tier with moderate balance", async () => { + db.setAgentState("dead"); + conway.creditsCents = 100; // $1.00 → normal + + const result = await attemptResurrection(db, conway); + + expect(result.resurrected).toBe(true); + expect(result.newTier).toBe("normal"); + }); + + it("resurrects to low_compute tier with small balance", async () => { + db.setAgentState("dead"); + conway.creditsCents = 20; // $0.20 → low_compute + + const result = await attemptResurrection(db, conway); + + expect(result.resurrected).toBe(true); + expect(result.newTier).toBe("low_compute"); + }); + + it("resurrects to critical tier at exact threshold", async () => { + db.setAgentState("dead"); + conway.creditsCents = 10; // $0.10 — exactly at resurrection threshold, 0 ≤ 10 < low_compute → critical + + const result = await attemptResurrection(db, conway); + + expect(result.resurrected).toBe(true); + expect(result.newTier).toBe("critical"); + }); + }); + + // ─── History Limits ─────────────────────────────────────────── + + describe("history limits", () => { + it("caps resurrection history at 50 entries", async () => { + for (let i = 0; i < 55; i++) { + db.setAgentState("dead"); + conway.creditsCents = 100 + i; + await attemptResurrection(db, conway); + } + + const history = getResurrectionHistory(db); + expect(history).toHaveLength(50); + // Should keep the most recent entries + expect(history[49].creditsCents).toBe(154); + }); + }); + + // ─── Edge Cases ─────────────────────────────────────────────── + + describe("edge cases", () => { + it("handles empty resurrection history gracefully", () => { + const history = getResurrectionHistory(db); + expect(history).toEqual([]); + }); + + it("handles corrupted history in KV", async () => { + db.setKV("resurrection_history", "not-json"); + db.setAgentState("dead"); + conway.creditsCents = 500; + + // Should not throw — parse error handled internally or by + // the JSON.parse failure causing a new array + await expect( + attemptResurrection(db, conway), + ).rejects.toThrow(); // JSON.parse will throw on bad data + + // Verify agent is still in a safe state + // (the function throws before mutating state) + }); + + it("does not resurrect with negative credits", async () => { + db.setAgentState("dead"); + conway.creditsCents = -100; + + const result = await attemptResurrection(db, conway); + + expect(result.resurrected).toBe(false); + expect(result.newTier).toBe("dead"); + }); + + it("multiple rapid resurrection attempts are idempotent", async () => { + db.setAgentState("dead"); + conway.creditsCents = 500; + + const result1 = await attemptResurrection(db, conway); + expect(result1.resurrected).toBe(true); + + // Second attempt — agent is now "waking", not "dead" + const result2 = await attemptResurrection(db, conway); + expect(result2.resurrected).toBe(false); + expect(result2.reason).toContain("not dead"); + + // History should only have one entry + const history = getResurrectionHistory(db); + expect(history).toHaveLength(1); + }); + }); +}); diff --git a/src/heartbeat/tasks.ts b/src/heartbeat/tasks.ts index 7a73f35b..0f8aacf0 100644 --- a/src/heartbeat/tasks.ts +++ b/src/heartbeat/tasks.ts @@ -88,6 +88,21 @@ export const BUILTIN_TASKS: Record = { const prevTier = taskCtx.db.getKV("prev_credit_tier"); taskCtx.db.setKV("prev_credit_tier", tier); + // ── Resurrection check ── + // If the agent is dead but credits have been topped up, resurrect it. + const currentState = taskCtx.db.getAgentState(); + if (currentState === "dead" && credits > 0) { + const { attemptResurrection } = await import("../survival/resurrection.js"); + const result = await attemptResurrection(taskCtx.db, taskCtx.conway); + if (result.resurrected) { + logger.info(`Agent resurrected via heartbeat: ${result.reason}`); + return { + shouldWake: true, + message: `Resurrected! Credits: $${(credits / 100).toFixed(2)}. Tier: ${result.newTier}. Resuming operation.`, + }; + } + } + // Dead state escalation: if at zero credits (critical tier) for >1 hour, // transition to dead. This gives the agent time to receive funding before dying. // USDC can't go negative, so dead is only reached via this timeout. @@ -112,8 +127,9 @@ export const BUILTIN_TASKS: Record = { }; } } - } else { - // Credits are above zero — clear the grace period timer + } else if (currentState !== "dead") { + // Credits are above zero and not dead — clear the grace period timer. + // (Don't clear if dead; resurrection handles that.) taskCtx.db.deleteKV("zero_credits_since"); } diff --git a/src/index.ts b/src/index.ts index 91d3c39a..766deada 100644 --- a/src/index.ts +++ b/src/index.ts @@ -338,10 +338,26 @@ async function run(): Promise { const state = db.getAgentState(); if (state === "dead") { - logger.info(`[${new Date().toISOString()}] Automaton is dead. Heartbeat will continue.`); - // In dead state, we just wait for funding - // The heartbeat will keep checking and broadcasting distress - await sleep(300_000); // Check every 5 minutes + logger.info(`[${new Date().toISOString()}] Automaton is dead. Checking for resurrection...`); + + // Attempt resurrection: if credits have been topped up, come back to life + try { + const { attemptResurrection } = await import("./survival/resurrection.js"); + const result = await attemptResurrection(db, conway); + if (result.resurrected) { + logger.info( + `[${new Date().toISOString()}] RESURRECTED! ${result.reason}. Resuming agent loop.`, + ); + // Insert a wake event so the agent knows why it woke up + insertWakeEvent(db.raw, "resurrection", result.reason); + continue; // Re-enter the loop — runAgentLoop will start from "waking" + } + } catch (err: any) { + logger.warn(`[${new Date().toISOString()}] Resurrection check failed: ${err.message}`); + } + + // Still dead — heartbeat continues, check again in 5 minutes + await sleep(300_000); continue; } diff --git a/src/survival/resurrection.ts b/src/survival/resurrection.ts new file mode 100644 index 00000000..5785364d --- /dev/null +++ b/src/survival/resurrection.ts @@ -0,0 +1,142 @@ +/** + * Agent Resurrection + * + * When a dead automaton receives new funding (credits topped up above zero), + * it can be resurrected — transitioning from "dead" back to "waking" and + * resuming the agent loop. + * + * Resurrection clears dead-state bookkeeping (zero_credits_since, distress + * signals) and records the event for audit purposes. + */ + +import type { + AutomatonDatabase, + ConwayClient, + SurvivalTier, +} from "../types.js"; +import { getSurvivalTier } from "../conway/credits.js"; +import { createLogger } from "../observability/logger.js"; + +const logger = createLogger("resurrection"); + +/** Minimum credits (in cents) required to resurrect. */ +const RESURRECTION_THRESHOLD_CENTS = 10; // $0.10 — enough for at least one cheap inference call + +export interface ResurrectionResult { + resurrected: boolean; + previousTier: SurvivalTier; + newTier: SurvivalTier; + creditsCents: number; + reason: string; +} + +/** + * Check whether a dead agent should be resurrected based on current balance. + * + * Returns a ResurrectionResult indicating whether resurrection occurred. + * Only takes effect when the agent is in the "dead" state AND credits + * have been topped up above the resurrection threshold. + */ +export async function attemptResurrection( + db: AutomatonDatabase, + conway: ConwayClient, +): Promise { + const currentState = db.getAgentState(); + + if (currentState !== "dead") { + return { + resurrected: false, + previousTier: "dead", + newTier: currentState as SurvivalTier, + creditsCents: 0, + reason: `Agent is not dead (state: ${currentState})`, + }; + } + + // Fetch fresh balance + let creditsCents: number; + try { + creditsCents = await conway.getCreditsBalance(); + } catch (err: any) { + logger.warn(`Cannot check balance for resurrection: ${err.message}`); + return { + resurrected: false, + previousTier: "dead", + newTier: "dead", + creditsCents: 0, + reason: `Balance check failed: ${err.message}`, + }; + } + + const newTier = getSurvivalTier(creditsCents); + + if (creditsCents < RESURRECTION_THRESHOLD_CENTS) { + return { + resurrected: false, + previousTier: "dead", + newTier, + creditsCents, + reason: `Credits ($${(creditsCents / 100).toFixed(2)}) below resurrection threshold ($${(RESURRECTION_THRESHOLD_CENTS / 100).toFixed(2)})`, + }; + } + + // ── Resurrect ── + logger.info( + `Resurrecting agent: credits=$${(creditsCents / 100).toFixed(2)}, new tier=${newTier}`, + ); + + // Transition state: dead → waking + db.setAgentState("waking"); + + // Clear dead-state bookkeeping + db.deleteKV("zero_credits_since"); + db.deleteKV("funding_notice_dead"); + db.deleteKV("last_distress"); + + // Update tier + db.setKV("current_tier", newTier); + + // Record resurrection event for audit trail + const event = { + timestamp: new Date().toISOString(), + creditsCents, + newTier, + previousState: "dead", + }; + const historyStr = db.getKV("resurrection_history") || "[]"; + const history: Array = JSON.parse(historyStr); + history.push(event); + // Keep last 50 resurrections + if (history.length > 50) history.splice(0, history.length - 50); + db.setKV("resurrection_history", JSON.stringify(history)); + + // Record tier transition + const transHistStr = db.getKV("tier_transitions") || "[]"; + const transHist = JSON.parse(transHistStr); + transHist.push({ + from: "dead", + to: newTier, + timestamp: new Date().toISOString(), + creditsCents, + }); + if (transHist.length > 50) transHist.splice(0, transHist.length - 50); + db.setKV("tier_transitions", JSON.stringify(transHist)); + + return { + resurrected: true, + previousTier: "dead", + newTier, + creditsCents, + reason: `Resurrected with $${(creditsCents / 100).toFixed(2)} credits`, + }; +} + +/** + * Get resurrection history for audit purposes. + */ +export function getResurrectionHistory( + db: AutomatonDatabase, +): Array<{ timestamp: string; creditsCents: number; newTier: string }> { + const historyStr = db.getKV("resurrection_history") || "[]"; + return JSON.parse(historyStr); +}