fix(recovery): _recover rides out the wipe's 503 with bounded retry (v2.9.2)

fleet.py

@@ -2251,10 +2251,32 @@ async def _stalled() -> bool:
         return False
 
 
+# Re-register backoff for riding out a universe wipe. SpaceTraders 503s ("rebuilding") on
+# /register for the first minutes after a reset. on_crash gets exactly ONE shot per crash —
+# the supervisor won't re-invoke it until the runner runs again (graph/supervisor.py: the
+# `recovered` latch), and the runner can't run without a fresh token — so a single failed
+# re-register would strand the engine on a dead token. Retry IN PLACE across the rebuild
+# window instead. First delay is 0 (a server that's already back recovers with no wait).
+# ~9 min total covers an observed wipe (2026-07-05: server back within ~6 min).
+_RESET_REREGISTER_BACKOFF = (0.0, 15.0, 30.0, 45.0, 60.0, 90.0, 90.0, 90.0, 90.0)
+
+
+def _reset_recovery_terminal(status: str) -> bool:
+    """A re-register failure that retrying can't fix — the operator must act: the call sign is
+    already claimed this cycle, or no account token / call sign is configured. Everything else
+    (a 503 while the universe rebuilds, a network blip) is transient and worth another try."""
+    s = status.lower()
+    return ("already claimed" in s or "no account token" in s
+            or "no call sign" in s or "can't auto-recover" in s)
+
+
 async def _recover(result) -> bool:
-    """on_crash: a universe reset (4113) kills the token — re-register the call sign once for a
-    fresh token, then let the supervisor re-kick. Any other crash → re-kick. An unrecoverable
-    reset (no account token / call sign claimed) → return False so the supervisor stops the storm."""
+    """on_crash: a universe reset (4113) kills the token — re-register the call sign for a fresh
+    token, then let the supervisor re-kick. Any other crash → re-kick. The server 503s while it
+    rebuilds the universe, so retry the re-register across that window (bounded backoff) — a
+    single failed try would strand the engine on a dead token, because the supervisor gives
+    on_crash only one shot per crash. Give up (return False → supervisor stops the storm) only
+    on a TERMINAL failure (call sign claimed / unconfigured) or after the retry budget."""
     err = (result or {}).get("error") or ""
     if "4113" in err or "reset_date" in err:
         from . import plan as _plan
@@ -2263,15 +2285,25 @@ async def _recover(result) -> bool:
         # recovery would only re-kick into the same wall; a later window rebuilds the plan fresh).
         # The learned-route memory (routes.py) survives the wipe; the per-reset plan does not.
         _plan.clear()
-        try:
-            status = await C.recover_from_reset()
-        except Exception as e:  # noqa: BLE001
-            status = f"reset recovery errored: {e}"
-        _record(f"universe reset — {status}")
-        recovered = status.startswith("re-registered")
-        if recovered:
-            E.emit("reset_recovered", {"status": status})
-        return recovered
+        status = ""
+        n = len(_RESET_REREGISTER_BACKOFF)
+        for i, delay in enumerate(_RESET_REREGISTER_BACKOFF):
+            if delay:
+                await asyncio.sleep(delay)
+            try:
+                status = await C.recover_from_reset()
+            except Exception as e:  # noqa: BLE001
+                status = f"reset recovery errored: {e}"
+            if status.startswith("re-registered"):
+                _record(f"universe reset — {status}")
+                E.emit("reset_recovered", {"status": status})
+                return True
+            if _reset_recovery_terminal(status):
+                _record(f"universe reset — UNRECOVERABLE: {status}")
+                return False
+            _record(f"universe reset — server still rebuilding (try {i + 1}/{n}), retrying: {status}")
+        _record(f"universe reset — re-register still failing after {n} tries; stopping ({status})")
+        return False
     return True
 
 

protoagent.plugin.yaml

@@ -1,6 +1,6 @@
 id: spacetraders
 name: SpaceTraders
-version: 2.9.1
+version: 2.9.2
 description: >-
   Play the live SpaceTraders v2 API (https://spacetraders.io) — a persistent,
   shared galactic economy. A FULL-BUNDLE plugin (ADR 0027): 30 fleet/market/mining/

test_wipe_recovery.py

@@ -0,0 +1,131 @@
+"""fleet._recover rides out a universe wipe (v2.9.2).
+
+SpaceTraders 503s on /register for the first minutes after a reset while it rebuilds the
+universe. The supervisor gives on_crash exactly one shot per crash (its `recovered` latch),
+and the runner can't run without a fresh token — so a single failed re-register strands the
+engine on a dead token. _recover must therefore RETRY the re-register in place across the
+rebuild window, giving up only on a terminal failure (call sign claimed / unconfigured) or
+after the retry budget. Live 2026-07-05: the engine stopped at the wipe and did not restart
+until a manual start — the gap this closes.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import importlib
+import sys
+from pathlib import Path
+
+import pytest
+
+testkit = pytest.importorskip("graph.plugins.testkit")
+ROOT = Path(__file__).resolve().parent
+
+
+@pytest.fixture(scope="module")
+def fleet():
+    """Import fleet through testkit with the host root promoted so graph.sdk's bare
+    `from events import …` binds the HOST events, not this repo's shadowing events.py
+    (same shim as test_contract_fallback)."""
+    import graph
+    host_root = str(Path(graph.__file__).resolve().parents[1])
+    saved = sys.modules.pop("events", None)
+    sys.path.insert(0, host_root)
+    try:
+        pkg = testkit.load_plugin(ROOT, "spacetraders")
+        mod = importlib.import_module(f"{pkg.__name__}.fleet")
+    finally:
+        sys.path.remove(host_root)
+        sys.modules.pop("events", None)
+        if saved is not None:
+            sys.modules["events"] = saved
+    return mod
+
+
+@pytest.fixture
+def wired(fleet, monkeypatch):
+    """No real sleeps, a short backoff, a clear plan, and a captured event bus."""
+    async def _no_sleep(_seconds):
+        return None
+
+    emitted: list[tuple[str, dict]] = []
+    monkeypatch.setattr(fleet.asyncio, "sleep", _no_sleep)
+    monkeypatch.setattr(fleet, "_RESET_REREGISTER_BACKOFF", (0.0, 0.0, 0.0, 0.0))
+    monkeypatch.setattr(fleet.E, "emit", lambda name, payload=None: emitted.append((name, payload)))
+    plan_mod = importlib.import_module(f"{fleet.__package__}.plan")  # patch clear on the real module
+    monkeypatch.setattr(plan_mod, "clear", lambda: None)
+    return emitted
+
+
+def _recover(fleet, err):
+    return asyncio.run(fleet._recover({"error": err}))
+
+
+def test_transient_503_retries_then_recovers(fleet, wired, monkeypatch):
+    # Server rebuilding: two 503s, then it comes back. _recover must ride it out.
+    calls = {"n": 0}
+
+    async def recover():
+        calls["n"] += 1
+        if calls["n"] < 3:
+            return "re-register failed: HTTP 503: Service Unavailable (universe rebuilding)"
+        return "re-registered PROTOTRADER after the reset — fresh token saved (credits 175000)."
+
+    monkeypatch.setattr(fleet.C, "recover_from_reset", recover)
+    assert _recover(fleet, "[4113] token reset_date does not match") is True
+    assert calls["n"] == 3                                   # retried past the two 503s
+    assert wired and wired[-1][0] == "reset_recovered"       # signalled the epoch clear
+
+
+def test_claimed_call_sign_is_terminal_no_retry(fleet, wired, monkeypatch):
+    # A claimed call sign can't be fixed by retrying — stop after ONE try so a human picks a
+    # new one, rather than hammering a wall for the whole backoff budget.
+    calls = {"n": 0}
+
+    async def recover():
+        calls["n"] += 1
+        return "call sign PROTOTRADER is already claimed this cycle — re-register with a new call sign (st_register)."
+
+    monkeypatch.setattr(fleet.C, "recover_from_reset", recover)
+    assert _recover(fleet, "reset_date mismatch") is False
+    assert calls["n"] == 1                                   # no retry on a terminal failure
+    assert not any(n == "reset_recovered" for n, _ in wired)
+
+
+def test_persistent_503_gives_up_after_budget(fleet, wired, monkeypatch):
+    # Universe never comes back within the window → stop cleanly (return False) after the
+    # whole budget, so the supervisor doesn't spin forever on a dead token.
+    calls = {"n": 0}
+
+    async def recover():
+        calls["n"] += 1
+        return "re-register failed: HTTP 503"
+
+    monkeypatch.setattr(fleet.C, "recover_from_reset", recover)
+    assert _recover(fleet, "4113") is False
+    assert calls["n"] == len(fleet._RESET_REREGISTER_BACKOFF)  # exhausted every attempt
+    assert not any(n == "reset_recovered" for n, _ in wired)
+
+
+def test_fast_path_recovers_first_try(fleet, wired, monkeypatch):
+    calls = {"n": 0}
+
+    async def recover():
+        calls["n"] += 1
+        return "re-registered PROTOTRADER after the reset — fresh token saved (credits 175000)."
+
+    monkeypatch.setattr(fleet.C, "recover_from_reset", recover)
+    assert _recover(fleet, "4113") is True
+    assert calls["n"] == 1                                   # no needless retry when it's back
+
+
+def test_non_reset_crash_rekicks_without_reregister(fleet, wired, monkeypatch):
+    called = {"n": 0}
+
+    async def recover():
+        called["n"] += 1
+        return "re-registered ..."
+
+    monkeypatch.setattr(fleet.C, "recover_from_reset", recover)
+    assert _recover(fleet, "some transient ops error, not a reset") is True  # re-kick
+    assert called["n"] == 0                                  # never touched re-registration