[codex] Retry Friendbot funding setup

.gitignore

@@ -4,6 +4,7 @@ dist/
 .env.prod
 .env.local
 .dev-seed
+.setup-wallets-checkpoint.json
 *.log
 data/audit.log.jsonl
 data/spending.json

docs/scripts/setup-wallets.md

@@ -9,6 +9,8 @@
 3. Store the `.dev-seed` mnemonic somewhere safe.
 4. Re-run `npm run setup` whenever you need the same test wallets again.
 
+Funding progress is checkpointed in `.setup-wallets-checkpoint.json`. If Friendbot rate-limits or returns a transient 5xx/network error, the script retries the current wallet up to five times. On rerun, wallets already funded in the checkpoint are skipped so the script resumes from the next wallet instead of starting from scratch.
+
 ## Using your own seed
 
 Pass a BIP-39 mnemonic directly:

scripts/__tests__/setup-wallets.test.ts

@@ -1,12 +1,16 @@
-import { mkdtempSync, readFileSync, rmSync } from "fs";
+import { existsSync, mkdtempSync, readFileSync, rmSync } from "fs";
 import { tmpdir } from "os";
 import path from "path";
-import { afterEach, describe, expect, it } from "vitest";
+import { afterEach, describe, expect, it, vi } from "vitest";
 import {
   deriveWalletsFromSeed,
   DEV_SEED_FILE,
+  fundAccount,
+  fundWalletWithCheckpoint,
   isMnemonicSeed,
+  loadSetupWalletCheckpoint,
   resolveSetupSeed,
+  SETUP_WALLET_CHECKPOINT_FILE,
 } from "../setup-wallets.ts";
 
 const tempDirs: string[] = [];
@@ -81,3 +85,79 @@ describe("setup-wallets seed handling", () => {
     ).rejects.toThrow(/Aborted/);
   });
 });
+
+describe("setup-wallets Friendbot funding", () => {
+  const noWait = async () => {};
+
+  it("retries transient Friendbot HTTP failures before succeeding", async () => {
+    const fetchFn = vi
+      .fn()
+      .mockResolvedValueOnce(new Response("rate limited", { status: 429 }))
+      .mockResolvedValueOnce(new Response("temporarily unavailable", { status: 503 }))
+      .mockResolvedValueOnce(new Response("ok", { status: 200 }));
+
+    await expect(
+      fundAccount("GTRANSIENT", { fetchFn, sleepFn: noWait }),
+    ).resolves.toBe("funded");
+    expect(fetchFn).toHaveBeenCalledTimes(3);
+  });
+
+  it("retries transient network failures before succeeding", async () => {
+    const fetchFn = vi
+      .fn()
+      .mockRejectedValueOnce(new TypeError("network down"))
+      .mockResolvedValueOnce(new Response("ok", { status: 200 }));
+
+    await expect(
+      fundAccount("GNETWORK", { fetchFn, sleepFn: noWait }),
+    ).resolves.toBe("funded");
+    expect(fetchFn).toHaveBeenCalledTimes(2);
+  });
+
+  it("fails permanent Friendbot validation errors without retrying", async () => {
+    const fetchFn = vi
+      .fn()
+      .mockResolvedValueOnce(new Response("invalid Stellar address", { status: 400 }));
+
+    await expect(
+      fundAccount("GBAD", { fetchFn, sleepFn: noWait }),
+    ).rejects.toThrow(/permanent HTTP 400/);
+    expect(fetchFn).toHaveBeenCalledTimes(1);
+  });
+
+  it("treats already-funded accounts as success", async () => {
+    const fetchFn = vi
+      .fn()
+      .mockResolvedValueOnce(new Response("createAccountAlreadyExist", { status: 400 }));
+
+    await expect(
+      fundAccount("GEXISTS", { fetchFn, sleepFn: noWait }),
+    ).resolves.toBe("already_exists");
+    expect(fetchFn).toHaveBeenCalledTimes(1);
+  });
+
+  it("writes a checkpoint after funding and skips that wallet on rerun", async () => {
+    const cwd = tempDir();
+    const [wallet] = deriveWalletsFromSeed("checkpoint-seed");
+    const fetchFn = vi.fn().mockResolvedValue(new Response("ok", { status: 200 }));
+    const now = () => new Date("2026-06-27T00:00:00.000Z");
+
+    await expect(
+      fundWalletWithCheckpoint(wallet, { cwd, fetchFn, sleepFn: noWait, now }),
+    ).resolves.toBe("funded");
+
+    const checkpointFile = path.join(cwd, SETUP_WALLET_CHECKPOINT_FILE);
+    expect(existsSync(checkpointFile)).toBe(true);
+    const checkpoint = loadSetupWalletCheckpoint(checkpointFile);
+    expect(checkpoint.funded[wallet.name]).toEqual({
+      publicKey: wallet.publicKey,
+      fundedAt: "2026-06-27T00:00:00.000Z",
+    });
+
+    fetchFn.mockClear();
+    await expect(
+      fundWalletWithCheckpoint(wallet, { cwd, fetchFn, sleepFn: noWait, now }),
+    ).resolves.toBe("skipped");
+    expect(fetchFn).not.toHaveBeenCalled();
+  });
+});

scripts/setup-wallets.ts

@@ -18,6 +18,8 @@ const HORIZON_URL = "https://horizon-testnet.stellar.org";
 const FRIENDBOT_URL = "https://friendbot.stellar.org";
 const USDC_ISSUER = "GBBD47IF6LWK7P7MDEVSCWR7DPUWV3NY3DTQEVFL4NAT4AQH3ZLLFLA5";
 export const DEV_SEED_FILE = ".dev-seed";
+export const SETUP_WALLET_CHECKPOINT_FILE = ".setup-wallets-checkpoint.json";
+export const FRIENDBOT_MAX_ATTEMPTS = 5;
 export const WALLET_NAMES = [
   "AGENT",
   "CAREGIVER",
@@ -36,6 +38,48 @@ export interface WalletInfo {
   secretKey: string;
 }
 
+export interface SetupWalletCheckpoint {
+  version: 1;
+  funded: Record<string, { publicKey: string; fundedAt: string }>;
+}
+
+type FetchFn = typeof fetch;
+type SleepFn = (ms: number) => Promise<void>;
+
+function sleep(ms: number) {
+  return new Promise<void>((resolve) => setTimeout(resolve, ms));
+}
+
+function checkpointPath(cwd = process.cwd()) {
+  return path.join(cwd, SETUP_WALLET_CHECKPOINT_FILE);
+}
+
+export function loadSetupWalletCheckpoint(filePath = checkpointPath()): SetupWalletCheckpoint {
+  if (!existsSync(filePath)) {
+    return { version: 1, funded: {} };
+  }
+
+  const parsed = JSON.parse(readFileSync(filePath, "utf-8")) as Partial<SetupWalletCheckpoint>;
+  return {
+    version: 1,
+    funded: parsed.funded || {},
+  };
+}
+
+export function saveSetupWalletCheckpoint(
+  checkpoint: SetupWalletCheckpoint,
+  filePath = checkpointPath(),
+) {
+  writeFileSync(filePath, `${JSON.stringify(checkpoint, null, 2)}\n`, { mode: 0o600 });
+}
+
+export function isWalletFundedInCheckpoint(
+  checkpoint: SetupWalletCheckpoint,
+  wallet: Pick<WalletInfo, "name" | "publicKey">,
+) {
+  return checkpoint.funded[wallet.name]?.publicKey === wallet.publicKey;
+}
+
 function normalizeSeedMaterial(seedMaterial: string) {
   return seedMaterial.trim().normalize("NFKD").split(/\s+/).join(" ");
 }
@@ -162,15 +206,106 @@ export async function resolveSetupSeed(options: {
   return { seed, source: "generated", path: seedPath };
 }
 
-async function fundAccount(publicKey: string): Promise<void> {
-  const response = await fetch(`${FRIENDBOT_URL}?addr=${publicKey}`);
-  if (!response.ok) {
-    const text = await response.text();
-    // Friendbot returns an error if already funded, which is fine
-    if (!text.includes("createAccountAlreadyExist")) {
-      throw new Error(`Friendbot failed for ${publicKey}: ${text}`);
+function isTransientFriendbotStatus(status: number) {
+  return status === 429 || status === 500 || status === 502 || status === 503 || status === 504;
+}
+
+function retryDelayMs(attempt: number) {
+  return 250 * 2 ** (attempt - 1);
+}
+
+export async function fundAccount(
+  publicKey: string,
+  options: {
+    fetchFn?: FetchFn;
+    sleepFn?: SleepFn;
+    maxAttempts?: number;
+  } = {},
+): Promise<"funded" | "already_exists"> {
+  const fetchFn = options.fetchFn || fetch;
+  const sleepFn = options.sleepFn || sleep;
+  const maxAttempts = options.maxAttempts || FRIENDBOT_MAX_ATTEMPTS;
+  const url = `${FRIENDBOT_URL}?addr=${publicKey}`;
+
+  for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+    try {
+      const response = await fetchFn(url);
+      if (response.ok) {
+        return "funded";
+      }
+
+      const text = await response.text();
+      // Friendbot returns an error if already funded, which is fine.
+      if (text.includes("createAccountAlreadyExist")) {
+        return "already_exists";
+      }
+
+      if (isTransientFriendbotStatus(response.status)) {
+        if (attempt < maxAttempts) {
+          logger.warn(
+            { publicKey: publicKey.slice(0, 8), status: response.status, attempt, maxAttempts },
+            "Friendbot transient failure; retrying",
+          );
+          await sleepFn(retryDelayMs(attempt));
+          continue;
+        }
+        throw new Error(
+          `Friendbot failed for ${publicKey}: transient HTTP ${response.status} after ${maxAttempts} attempts: ${text}`,
+        );
+      }
+
+      throw new Error(`Friendbot failed for ${publicKey}: permanent HTTP ${response.status}: ${text}`);
+    } catch (err) {
+      if (err instanceof Error && err.message.startsWith(`Friendbot failed for ${publicKey}:`)) {
+        throw err;
+      }
+
+      if (attempt < maxAttempts) {
+        logger.warn(
+          { publicKey: publicKey.slice(0, 8), attempt, maxAttempts, err: err instanceof Error ? err.message : String(err) },
+          "Friendbot network failure; retrying",
+        );
+        await sleepFn(retryDelayMs(attempt));
+        continue;
+      }
+
+      throw new Error(
+        `Friendbot failed for ${publicKey}: network error after ${maxAttempts} attempts: ${
+          err instanceof Error ? err.message : String(err)
+        }`,
+      );
     }
   }
+
+  throw new Error(`Friendbot failed for ${publicKey}: exhausted retry attempts`);
+}
+
+export async function fundWalletWithCheckpoint(
+  wallet: WalletInfo,
+  options: {
+    cwd?: string;
+    checkpointFile?: string;
+    fetchFn?: FetchFn;
+    sleepFn?: SleepFn;
+    now?: () => Date;
+  } = {},
+): Promise<"funded" | "already_exists" | "skipped"> {
+  const filePath = options.checkpointFile || checkpointPath(options.cwd);
+  const checkpoint = loadSetupWalletCheckpoint(filePath);
+  if (isWalletFundedInCheckpoint(checkpoint, wallet)) {
+    return "skipped";
+  }
+
+  const result = await fundAccount(wallet.publicKey, {
+    fetchFn: options.fetchFn,
+    sleepFn: options.sleepFn,
+  });
+  checkpoint.funded[wallet.name] = {
+    publicKey: wallet.publicKey,
+    fundedAt: (options.now || (() => new Date()))().toISOString(),
+  };
+  saveSetupWalletCheckpoint(checkpoint, filePath);
+  return result;
 }
 
 async function addUsdcTrustline(keypair: Keypair): Promise<void> {
@@ -228,10 +363,11 @@ async function main() {
   logger.info("step 1: funding accounts via Friendbot");
   for (const wallet of wallets) {
     try {
-      await fundAccount(wallet.publicKey);
-      logger.info({ name: wallet.name, wallet: wallet.publicKey.slice(0, 8) }, "funded");
+      const result = await fundWalletWithCheckpoint(wallet);
+      logger.info({ name: wallet.name, wallet: wallet.publicKey.slice(0, 8), result }, "funded");
     } catch (err: any) {
       logger.error({ name: wallet.name, err: err.message }, "failed to fund wallet");
+      throw err;
     }
   }