From 6beca10ffca3319f80c3a33b12858172e2a88c3d Mon Sep 17 00:00:00 2001 From: Josh Mabry Date: Sun, 5 Jul 2026 06:35:43 -0700 Subject: [PATCH] fix(recovery): _recover rides out the wipe's 503 with bounded retry (v2.9.2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit At a universe wipe, SpaceTraders 503s on /register for the first minutes while it rebuilds. `_recover` (the supervisor's on_crash) re-registered ONCE, so that 503 made it return False and the supervisor stopped — stranding the fleet on a dead token. The supervisor gives on_crash exactly one shot per crash (its `recovered` latch won't re-invoke it until the runner runs again, and the runner can't run without a fresh token), so a single failed re-register is terminal. Observed live 2026-07-05: engine stopped at the wipe and did not restart until a manual start ~24 min later; registration only recovered via the agent's own out-of-band st_register retry, which does not restart the ops engine. Fix: retry the re-register IN PLACE across the rebuild window (bounded backoff, ~9 min, first try immediate). Give up (return False → supervisor stops the storm) only on a TERMINAL failure — call sign claimed / no account token / no call sign — or after the budget. Cancellation-safe: an operator stop cancels at the sleep checkpoint (CancelledError is BaseException, not caught by the except Exception). Follow-up (filed separately): if recovery lands out-of-band (agent st_register) after the supervisor already stopped, nothing restarts the engine; and the supervisor's one-shot on_crash is a host-level limitation. Co-Authored-By: Claude Fable 5 --- fleet.py | 56 ++++++++++++++---- protoagent.plugin.yaml | 2 +- test_wipe_recovery.py | 131 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 176 insertions(+), 13 deletions(-) create mode 100644 test_wipe_recovery.py diff --git a/fleet.py b/fleet.py index 69fd88c..9b983d1 100644 --- a/fleet.py +++ b/fleet.py @@ -2251,10 +2251,32 @@ async def _stalled() -> bool: return False +# Re-register backoff for riding out a universe wipe. SpaceTraders 503s ("rebuilding") on +# /register for the first minutes after a reset. on_crash gets exactly ONE shot per crash — +# the supervisor won't re-invoke it until the runner runs again (graph/supervisor.py: the +# `recovered` latch), and the runner can't run without a fresh token — so a single failed +# re-register would strand the engine on a dead token. Retry IN PLACE across the rebuild +# window instead. First delay is 0 (a server that's already back recovers with no wait). +# ~9 min total covers an observed wipe (2026-07-05: server back within ~6 min). +_RESET_REREGISTER_BACKOFF = (0.0, 15.0, 30.0, 45.0, 60.0, 90.0, 90.0, 90.0, 90.0) + + +def _reset_recovery_terminal(status: str) -> bool: + """A re-register failure that retrying can't fix — the operator must act: the call sign is + already claimed this cycle, or no account token / call sign is configured. Everything else + (a 503 while the universe rebuilds, a network blip) is transient and worth another try.""" + s = status.lower() + return ("already claimed" in s or "no account token" in s + or "no call sign" in s or "can't auto-recover" in s) + + async def _recover(result) -> bool: - """on_crash: a universe reset (4113) kills the token — re-register the call sign once for a - fresh token, then let the supervisor re-kick. Any other crash → re-kick. An unrecoverable - reset (no account token / call sign claimed) → return False so the supervisor stops the storm.""" + """on_crash: a universe reset (4113) kills the token — re-register the call sign for a fresh + token, then let the supervisor re-kick. Any other crash → re-kick. The server 503s while it + rebuilds the universe, so retry the re-register across that window (bounded backoff) — a + single failed try would strand the engine on a dead token, because the supervisor gives + on_crash only one shot per crash. Give up (return False → supervisor stops the storm) only + on a TERMINAL failure (call sign claimed / unconfigured) or after the retry budget.""" err = (result or {}).get("error") or "" if "4113" in err or "reset_date" in err: from . import plan as _plan @@ -2263,15 +2285,25 @@ async def _recover(result) -> bool: # recovery would only re-kick into the same wall; a later window rebuilds the plan fresh). # The learned-route memory (routes.py) survives the wipe; the per-reset plan does not. _plan.clear() - try: - status = await C.recover_from_reset() - except Exception as e: # noqa: BLE001 - status = f"reset recovery errored: {e}" - _record(f"universe reset — {status}") - recovered = status.startswith("re-registered") - if recovered: - E.emit("reset_recovered", {"status": status}) - return recovered + status = "" + n = len(_RESET_REREGISTER_BACKOFF) + for i, delay in enumerate(_RESET_REREGISTER_BACKOFF): + if delay: + await asyncio.sleep(delay) + try: + status = await C.recover_from_reset() + except Exception as e: # noqa: BLE001 + status = f"reset recovery errored: {e}" + if status.startswith("re-registered"): + _record(f"universe reset — {status}") + E.emit("reset_recovered", {"status": status}) + return True + if _reset_recovery_terminal(status): + _record(f"universe reset — UNRECOVERABLE: {status}") + return False + _record(f"universe reset — server still rebuilding (try {i + 1}/{n}), retrying: {status}") + _record(f"universe reset — re-register still failing after {n} tries; stopping ({status})") + return False return True diff --git a/protoagent.plugin.yaml b/protoagent.plugin.yaml index de15c81..524f83f 100644 --- a/protoagent.plugin.yaml +++ b/protoagent.plugin.yaml @@ -1,6 +1,6 @@ id: spacetraders name: SpaceTraders -version: 2.9.1 +version: 2.9.2 description: >- Play the live SpaceTraders v2 API (https://spacetraders.io) — a persistent, shared galactic economy. A FULL-BUNDLE plugin (ADR 0027): 30 fleet/market/mining/ diff --git a/test_wipe_recovery.py b/test_wipe_recovery.py new file mode 100644 index 0000000..d085910 --- /dev/null +++ b/test_wipe_recovery.py @@ -0,0 +1,131 @@ +"""fleet._recover rides out a universe wipe (v2.9.2). + +SpaceTraders 503s on /register for the first minutes after a reset while it rebuilds the +universe. The supervisor gives on_crash exactly one shot per crash (its `recovered` latch), +and the runner can't run without a fresh token — so a single failed re-register strands the +engine on a dead token. _recover must therefore RETRY the re-register in place across the +rebuild window, giving up only on a terminal failure (call sign claimed / unconfigured) or +after the retry budget. Live 2026-07-05: the engine stopped at the wipe and did not restart +until a manual start — the gap this closes. +""" + +from __future__ import annotations + +import asyncio +import importlib +import sys +from pathlib import Path + +import pytest + +testkit = pytest.importorskip("graph.plugins.testkit") +ROOT = Path(__file__).resolve().parent + + +@pytest.fixture(scope="module") +def fleet(): + """Import fleet through testkit with the host root promoted so graph.sdk's bare + `from events import …` binds the HOST events, not this repo's shadowing events.py + (same shim as test_contract_fallback).""" + import graph + host_root = str(Path(graph.__file__).resolve().parents[1]) + saved = sys.modules.pop("events", None) + sys.path.insert(0, host_root) + try: + pkg = testkit.load_plugin(ROOT, "spacetraders") + mod = importlib.import_module(f"{pkg.__name__}.fleet") + finally: + sys.path.remove(host_root) + sys.modules.pop("events", None) + if saved is not None: + sys.modules["events"] = saved + return mod + + +@pytest.fixture +def wired(fleet, monkeypatch): + """No real sleeps, a short backoff, a clear plan, and a captured event bus.""" + async def _no_sleep(_seconds): + return None + + emitted: list[tuple[str, dict]] = [] + monkeypatch.setattr(fleet.asyncio, "sleep", _no_sleep) + monkeypatch.setattr(fleet, "_RESET_REREGISTER_BACKOFF", (0.0, 0.0, 0.0, 0.0)) + monkeypatch.setattr(fleet.E, "emit", lambda name, payload=None: emitted.append((name, payload))) + plan_mod = importlib.import_module(f"{fleet.__package__}.plan") # patch clear on the real module + monkeypatch.setattr(plan_mod, "clear", lambda: None) + return emitted + + +def _recover(fleet, err): + return asyncio.run(fleet._recover({"error": err})) + + +def test_transient_503_retries_then_recovers(fleet, wired, monkeypatch): + # Server rebuilding: two 503s, then it comes back. _recover must ride it out. + calls = {"n": 0} + + async def recover(): + calls["n"] += 1 + if calls["n"] < 3: + return "re-register failed: HTTP 503: Service Unavailable (universe rebuilding)" + return "re-registered PROTOTRADER after the reset — fresh token saved (credits 175000)." + + monkeypatch.setattr(fleet.C, "recover_from_reset", recover) + assert _recover(fleet, "[4113] token reset_date does not match") is True + assert calls["n"] == 3 # retried past the two 503s + assert wired and wired[-1][0] == "reset_recovered" # signalled the epoch clear + + +def test_claimed_call_sign_is_terminal_no_retry(fleet, wired, monkeypatch): + # A claimed call sign can't be fixed by retrying — stop after ONE try so a human picks a + # new one, rather than hammering a wall for the whole backoff budget. + calls = {"n": 0} + + async def recover(): + calls["n"] += 1 + return "call sign PROTOTRADER is already claimed this cycle — re-register with a new call sign (st_register)." + + monkeypatch.setattr(fleet.C, "recover_from_reset", recover) + assert _recover(fleet, "reset_date mismatch") is False + assert calls["n"] == 1 # no retry on a terminal failure + assert not any(n == "reset_recovered" for n, _ in wired) + + +def test_persistent_503_gives_up_after_budget(fleet, wired, monkeypatch): + # Universe never comes back within the window → stop cleanly (return False) after the + # whole budget, so the supervisor doesn't spin forever on a dead token. + calls = {"n": 0} + + async def recover(): + calls["n"] += 1 + return "re-register failed: HTTP 503" + + monkeypatch.setattr(fleet.C, "recover_from_reset", recover) + assert _recover(fleet, "4113") is False + assert calls["n"] == len(fleet._RESET_REREGISTER_BACKOFF) # exhausted every attempt + assert not any(n == "reset_recovered" for n, _ in wired) + + +def test_fast_path_recovers_first_try(fleet, wired, monkeypatch): + calls = {"n": 0} + + async def recover(): + calls["n"] += 1 + return "re-registered PROTOTRADER after the reset — fresh token saved (credits 175000)." + + monkeypatch.setattr(fleet.C, "recover_from_reset", recover) + assert _recover(fleet, "4113") is True + assert calls["n"] == 1 # no needless retry when it's back + + +def test_non_reset_crash_rekicks_without_reregister(fleet, wired, monkeypatch): + called = {"n": 0} + + async def recover(): + called["n"] += 1 + return "re-registered ..." + + monkeypatch.setattr(fleet.C, "recover_from_reset", recover) + assert _recover(fleet, "some transient ops error, not a reset") is True # re-kick + assert called["n"] == 0 # never touched re-registration