diff --git a/.github/workflows/deploy-backtest.yml b/.github/workflows/deploy-backtest.yml index eec6768..5609798 100644 --- a/.github/workflows/deploy-backtest.yml +++ b/.github/workflows/deploy-backtest.yml @@ -87,8 +87,39 @@ jobs: npm run gen:local-constants npm run gen:state-dump + # A scenario is (regime, seed) and the regime YAML carries no seed, so --seed is required + # (ADR 0017 §1). --blocks shortens the run for CI; the regime's own 360 would take 12 minutes. - name: Backtest against the dump - run: npm run backtest -- --regime calm-01 --blocks 12 + run: npm run backtest -- --regime calm --seed 101 --blocks 12 --seconds 90 + + # The matrix path is what the competition actually runs, and it is a different code path from + # a single --regime run: it writes matrix.json / standings.json and has to survive a scenario + # that fails without abandoning the rest. Two scenarios is enough to exercise the loop. + - name: Replay a small scenario matrix + run: | + cat > "$RUNNER_TEMP/ci-scenarios.yaml" <<'YAML' + regimes: [calm] + seeds: [101, 202] + YAML + npm run backtest -- --scenarios "$RUNNER_TEMP/ci-scenarios.yaml" --blocks 12 --seconds 90 + node -e ' + const { readdirSync, readFileSync } = require("node:fs"); + const dir = readdirSync("runs").filter((d) => d.startsWith("matrix-")).sort().at(-1); + if (!dir) throw new Error("no matrix directory was produced"); + const m = JSON.parse(readFileSync(`runs/${dir}/matrix.json`, "utf8")); + const s = JSON.parse(readFileSync(`runs/${dir}/standings.json`, "utf8")); + const failed = m.scenarios.filter((x) => !x.agents); + if (failed.length) throw new Error(`scenarios produced no result: ${JSON.stringify(failed)}`); + if (m.scenarios.length !== 2) throw new Error(`expected 2 scenarios, got ${m.scenarios.length}`); + if (!s.agents?.length) throw new Error("standings ranked nobody"); + // Both metrics must survive into the matrix, since the scoring rule is expected to + // change and matrix.json is what makes a finished run re-scorable (ADR 0017 §4). + for (const sc of m.scenarios) + for (const a of sc.agents) + if (!("netPnlUsdc" in a) || !("alphaUsdc" in a)) + throw new Error(`${sc.regime}#${sc.seed} ${a.id} is missing a metric`); + console.log(`matrix ok: ${m.scenarios.length} scenarios, ${s.agents.length} ranked`); + ' # The backtest exits 0 even when the scorer quietly read nothing, which is the failure mode # this whole job exists to catch, so assert on the run's own output. diff --git a/CLAUDE.md b/CLAUDE.md index 1c2da99..142e96c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -23,22 +23,27 @@ deployer/ venue デプロイ（自己完結サブパッケージ。workspace |------|------|--------| | `agent.ts`（`decide(obs, ctx)` export） | ルール戦略 | runtime/bot.ts が read→decide→send のループで駆動（`export const config = { intervalMs }` で間隔指定可） | | `agent.ts`（`run(ctx)` export） | 自走型 | bot.ts はループせず ctx（clients/observe/submit/log）を渡して委譲（例 liquidator） | -| `prompt.md`（frontmatter: name/description 必須） | プロンプト型 | bot.ts が observation を添えて毎判断 LLM に action を出させる（例 my-arb） | - -`runtime/`（汎用スクリプト: bot/read/send/llm/prompt/deploy/agentLog）と `lib/`（共有戦略ヘルパ）は予約名。 -同梱の全 agent は agent.ts と併置で **prompt.md も持ち、両方の動かし方を提供する**（runtime の既定は -agent.ts 優先 = ADR 0015 §2。ただし **雛形 `config/example.yaml` のロスターは取引 agent を prompt モードで -出荷**＝Quick Start 既定は LLM 駆動。要 OLLAMA_API_KEY in .env.local、LLM 判断 ~10s/回なので run 長は -100 blocks/300s 目安。noop はルールのまま。API キー無しでも `model: codex[:]` / `claude-cli[:]` で -**Codex/Claude Code サブスク CLI 実行**が可能 = docs/guide/llm-agents.md）。ロスターの `env` で切り替え: - -- `ERIS_AGENT_MODE: "prompt"` — agent.ts があっても prompt.md（毎判断 LLM）で動かす -- `ERIS_PROMPT_REVISE_EVERY: ""` — prompt モードで N 判断サイクルごとに LLM が prompt 本文を - **自己改訂**する（既定 0=off。改訂版は `runs//agents/.prompt.v.md` に版付き保存され - 以後のサイクルで使用。`ERIS_PROMPT_REVISE_PERSIST: "1"` で agent ディレクトリの prompt.md にも書き戻し） +| `agent.ts` + `improve.md`（frontmatter: name/description 必須） | **自己改善型**（ADR 0018） | decide を毎ブロック駆動しつつ、LLM が取引経路の**外**で戦略コードを書き換える | + +`runtime/`（汎用スクリプト: bot/read/send/llm/improve/deploy/agentLog）と `lib/`（共有戦略ヘルパ）は予約名。 + +**プロンプト型（毎判断 LLM）は ADR 0018 で廃止**。実測で 1 判断 8〜28 ブロック・行動回数がルール型の +1/64 で競技として成立しなかった（ADR 0017 §5 B1）。`ERIS_AGENT_MODE` / `ERIS_PROMPT_*` は fail-fast する。 +`improve.md` は prompt.md の改名ではない（前者は「いつ・何を根拠に・どう直すか」、後者は「この observation で +どう動くか」）。ロスターの `env`: + +- `ERIS_AGENT_FROZEN: "1"` — improve.md を無視して戦略を固定。**ADR 0018 §5 が要求する frozen 対照** + （自己改善が効いたかを毎 run 見えるようにする）をディレクトリ複製なしで作る +- `ERIS_LLM_MODEL: ""` — 改訂呼び出しのバックエンド（improve.md の frontmatter が優先）。 + API キー無しでも `codex[:]` / `claude-cli[:]` でサブスク CLI 実行可 = docs/guide/llm-agents.md +- `ERIS_IMPROVE_LOG_CALLS: "1"` — 改訂の生のやり取りを `agents/.llm.jsonl` に残す（既定 off） + +改訂は `{notes, executorTs}` か `{notes, revertTo: }` を返し、`executorTs: null` は +「今の戦略を維持」。生成コードは **cheatcode 静的検査 → コンパイル → 2 秒の実行上限**を通ってから設置。 +**自動 rollback は無い**（閾値に妥当な値が無いため。旧実装は 18 run 中 0 件発火、逆に「少しでも負けたら」 +だと全員が負けるレジームで毎回巻き戻る）。戻すかどうかはモデルの判断で、版履歴を渡して `revertTo` で行う。 +LLM バックエンドが無くても run は完走し、改訂失敗が記録されて戦略は無改変で走り続ける。 directShim / relay / stdin-stdout プロトコルは廃止済み（ERIS_AGENT_DIRECT_TX は退役）。 -プロンプト型の action 形式は sdk の zod スキーマ（`sdk/src/actionSchema.ts`）から `` を生成し、 -validate 失敗はエラー内容を会話に追記して再試行（上限超過は noop = fail-closed）。 ## 設定（YAML 単一ソース。ADR 0013） @@ -72,7 +77,10 @@ agents: - `npm run build:contracts` — モックオラクル + PriceFeed を forge build（sim:realtime の前提。`out/` 未生成なら最低 1 回） - `npm run gen:local-constants` — deployments.json → `sdk/src/constants.local.ts` 生成（同梱 `deployer/` のローカルデプロイ出力を読む） - `npm run gen:state-dump` — 稼働中の deployer anvil から配布用 state dump + manifest（生成元コミット・deployments 同梱・fingerprint）を `backtest/state/` へ生成（ADR 0016。dump 前に `.local-snapshot` のクリーン断面へ revert し、constants.local.ts も同じ deployments から再生成） -- `npm run backtest -- --regime ` — 参加者バックテスト（ADR 0016 Phase 0 = B1 実時間再生）。state dump をロードした専用 anvil（既定 port 8547）で `config/regimes/.yaml`（+seed）を再生する。`--repeat N`（snapshot/revert 反復・run 毎に採点再構成）/ `--agents `（regime 既定ロスターの差し替え）/ `--protocols` 等の一回上書き。**override は実効 regime YAML に書き出されて agent プロセスにも伝播**（coordinator だけに効かせると agent が観測で死ぬ）。fingerprint 不一致は manifest 同梱 deployments から constants を自動再生成、genesis 不一致は fail-fast +- `npm run backtest -- --regime --seed ` — シナリオ 1 本を再生（ADR 0016 Phase 0 = B1 実時間再生）。state dump をロードした専用 anvil（既定 port 8547）で `config/regimes/.yaml` + seed を再生する。**シナリオ = (regime, seed)** で regime YAML は seed を持たないので `--seed` は必須（ADR 0017 §1）。`--agents `（regime 既定ロスターの差し替え）/ `--protocols`/`--blocks`/`--score-every` 等の一回上書き。**override は実効 regime YAML に書き出されて agent プロセスにも伝播**（coordinator だけに効かせると agent が観測で死ぬ）。fingerprint 不一致は manifest 同梱 deployments から constants を自動再生成、genesis 不一致は fail-fast +- `npm run backtest -- --scenarios config/scenarios/public.yaml` — シナリオ行列を 1 つの anvil 上で全部再生し順位を出す（ADR 0017）。`{regimes, seeds}` の直積で、シナリオ間は snapshot/revert。`runs/matrix-/matrix.json`（シナリオ × agent の生スコア。**netPnlUsdc と alphaUsdc の両方**）と `standings.json`（レジーム内 z-score → レジーム等重み平均）を書く。順位は派生物で、採点方法は将来見直す前提（matrix.json から再計算できる）。`--metric netPnlUsdc|alphaUsdc` / `--repeat N`（較正の診断用。採点は 1 回が既定） + - **公式レジーム**: `calm` / `cex-drift`（OU に drift、kappa 弱化）/ `informed-flow`（相関した方向性フロー）/ `whale`（単発大口の点イベント）/ `lending-incident`（暴落 + victim + 清算）/ `crash`（価格ギャップのみ。流動性引き抜きは issue #52 待ち）。`depeg` は採点方法の見直し待ち。`lst` は競技セット外 + - `--score-every N` は採点断面の間引き。成績は初期/最終断面しか使わない（`alphaByAgent = alphaLast − alphaFirst`）ので**スコアは不変**、equity curve が粗くなるだけ - `npm run typecheck` / `npm run test` — 型チェック / ユニットテスト - `npm run check:strategy` — 戦略コードの cheatcode 静的検査（入口ゲート） - `npm run check:boundaries` — workspace 依存方向（example → sdk ← core）の検査 @@ -180,7 +188,7 @@ OU の base price はそのまま進め、その上に **SEED 由来でランダ runtime/send.ts が同じファイルに mempool 活動（`kind:"mempool"`: submitted / submit_failed / rejected）を自己申告で追記する（coordinator が submitted を数えられなくなる穴を塞ぐ。ADR 0006 §5）。 出力先は coordinator が渡す env `ERIS_RUN_DIR` / `ERIS_AGENT_ID` で決まる。run 後の診断はこれを一次情報にする。 -prompt 型は `ERIS_PROMPT_LOG_CALLS: "1"`（ロスターの env）で LLM との生の対話（system 全文・送信 +自己改善型は `ERIS_IMPROVE_LOG_CALLS: "1"`（ロスターの env）で LLM との生の対話（system 全文・送信 messages・生応答・エラー）を `agents/.llm.jsonl` に残せる（opt-in。プロンプト調整の一次情報）。 ## spot EC2 で重い run を回す（ローカル逼迫の回避。spot skills） diff --git a/README.md b/README.md index 23d4160..163f4cb 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ flowchart LR - **Multi-agent competition** — agents run as fully independent processes, subscribe to blocks at their own pace, and sign and send directly themselves. In-block ordering is determined by anvil `--order fees` (descending priority fee). - **Controllable fair price** — the coordinator generates a SEED-derived deterministic fair price every block and writes it to the on-chain `PriceFeed` and mock oracles. Aave health factors and GMX mark prices follow it. - **Market stress & liquidation** — price spikes/crashes can be injected to trigger the Aave liquidation path. -- **LLM-driven autonomous agents** — a single `prompt.md` is the strategy itself. The LLM emits an action on every decision and can even self-revise the prompt (no hand-written trading logic). +- **Self-improving agents** — the strategy trades every block on its own, and an LLM periodically rewrites it in-run from its own track record. The LLM is never in the trade path. - **Fork-free local deploy mode** — avoids cold-state RPC round trips to the fork backend (fork RPC latency), and multi-asset (WETH/WBTC) works too. - **Backtesting** — with a distributed state dump plus official regimes (market scenarios), a strategy can be verified over and over under the same environment and the same scoring (`--repeat` to read the distribution). @@ -77,13 +77,13 @@ cd .. ### Choose an LLM backend -**The default roster is LLM-driven**: the trading agents run in prompt mode (`prompt.md`, one LLM call per decision), so they need an LLM backend to trade. Pick one — without it the run still completes, but the trading agents fail closed to `noop` and trade nothing: +**The default roster is self-improving**: the trading agents are rule strategies that trade every block on their own, and an LLM periodically rewrites them ([Self-improving agents](docs/guide/llm-agents.md)). A backend is therefore optional — without one the run completes normally, the revisions are recorded as failed, and the strategies keep trading unchanged. Pick one to see the improvement loop actually work: | backend | setup | |---|---| | **Ollama Cloud** (default; model `gpt-oss:120b`) | put `OLLAMA_API_KEY=...` in `.env.local` | | **Local ollama** (no key) | `ERIS_OLLAMA_BASE_URL=http://127.0.0.1:11434/api` in `.env.local`, and set a locally-pulled model via the roster env `ERIS_LLM_MODEL` | -| **Claude Code / Codex subscription** (no API key; spawns the logged-in CLI) | in `config/local.yaml`, swap each prompt agent's `env:` for the commented variant with `ERIS_LLM_MODEL: "claude-cli:haiku"` (or `"codex"`) | +| **Claude Code / Codex subscription** (no API key; spawns the logged-in CLI) | in `config/local.yaml`, add `ERIS_LLM_MODEL: "claude-cli:haiku"` (or `"codex"`) to the agent's `env:` | To skip LLMs entirely and run the same strategies rule-based (`agent.ts`), remove the `env:` line from each agent in the roster. Details: [LLM Agents](docs/guide/llm-agents.md). @@ -119,8 +119,8 @@ Once you bake a state dump from a deployed anvil, you can **replay official regi ```bash npm run gen:state-dump # bake once from the running deployer anvil -npm run backtest -- --regime calm-01 --repeat 5 # calm market, 5 times (prints mean alphaUsdc) -npm run backtest -- --regime crash-01 # crash + Aave liquidation scenario +npm run backtest -- --regime calm --seed 101 # one scenario (regime + seed) +npm run backtest -- --scenarios config/scenarios/public.yaml # the whole public set + standings ``` For details, see [Backtesting](docs/guide/backtest.md). @@ -138,7 +138,7 @@ For details, see [Backtesting](docs/guide/backtest.md). | [Backtesting](docs/guide/backtest.md) | Replaying state dump + official regimes, iterating with `--repeat`, sparring, what is and isn't measurable | | [Run Output and Analysis](docs/guide/run-output.md) | The output files under `runs//` and how to analyze a run afterwards | | [Protocols and Actions](docs/guide/protocols-and-actions.md) | Reference: actions per venue, stablecoin accounting, oracle control | -| [LLM-driven Autonomous Agents](docs/guide/llm-agents.md) | prompt.md-type agents (per-decision LLM, self-revision, conversation log) | +| [Self-improving Agents](docs/guide/llm-agents.md) | agent.ts + improve.md (in-run strategy rewriting, sandbox, rollback, frozen control) | **How the environment works / operations**: diff --git a/config/example.yaml b/config/example.yaml index 0e8d697..957ff9c 100644 --- a/config/example.yaml +++ b/config/example.yaml @@ -19,20 +19,25 @@ # set localDeploy: false, drop `lst` from run.protocols (its vault has no Arbitrum counterpart) and # start `npm run anvil` in another terminal with ARB_RPC_URL set. # -# LLM-driven by default: the trading agents below run in prompt mode (prompt.md, one LLM call per -# decision; ERIS_AGENT_MODE: "prompt") and need an LLM backend. Pick one: +# The agents below are rule strategies (agent.ts) that trade every block. Three of them also ship an +# improve.md, so an LLM periodically rewrites the strategy while it runs (ADR 0018) -- the LLM is +# never in the trade path. Prompt mode, where an LLM produced each action, was removed: it managed +# one decision every 8-28 blocks and 1/64 the actions of the same strategy in rule mode. +# +# A self-improving agent needs an LLM backend. Pick one: # - Ollama Cloud (default; model gpt-oss:120b): set OLLAMA_API_KEY in `.env.local` # - local ollama (no key): ERIS_OLLAMA_BASE_URL=http://127.0.0.1:11434/api in `.env.local` -# - Claude Code / Codex subscription (no API key; spawns the logged-in CLI): swap each agent's -# env for the commented ERIS_LLM_MODEL variant below ("claude-cli:haiku" / "codex") -# Without a backend the run completes but the trading agents fail closed to noop (no trades). -# To run the same strategies rule-based (agent.ts, no LLM), remove the `env:` line from each agent. +# - Claude Code / Codex subscription (no API key; spawns the logged-in CLI): +# ERIS_LLM_MODEL: "claude-cli:haiku" / "codex" in the agent's env +# Without a backend the run still completes: revisions fail and are recorded, and the strategy keeps +# trading unchanged. Set ERIS_AGENT_FROZEN=1 on an agent to skip the improvement loop entirely. # See docs/guide/llm-agents.md. run: seed: 1 - # LLM decisions take ~10s each over Ollama Cloud, so give the run enough blocks/wall-clock for - # prompt-mode agents to act (rule-based runs are fine with fewer, e.g. 24 blocks / 70s). + # The trading loop is rule-speed (one decision per block) regardless of the LLM, so the run length + # is not set by LLM latency any more. It does need to be long enough for at least one revision to + # land -- improve.md declares reviseEveryBlocks: 60. blocks: 100 seconds: 300 blockTimeSec: 2 @@ -108,19 +113,22 @@ agents: description: does nothing (baseline; rule-based on purpose) - id: venue-arb wallet: AGENT2_PRIVATE_KEY - description: WETH-only cross-venue arbitrage (LLM-driven via prompt.md) - env: { ERIS_AGENT_MODE: "prompt", ERIS_PROMPT_LOG_CALLS: "1" } - # Claude Code / Codex subscription instead of Ollama (no API key) — use this env line instead: - # env: { ERIS_AGENT_MODE: "prompt", ERIS_PROMPT_LOG_CALLS: "1", ERIS_LLM_MODEL: "claude-cli:haiku" } - - id: multi-arb + description: WETH-only cross-venue arbitrage, with an LLM revising it in-run (improve.md) + env: { ERIS_IMPROVE_LOG_CALLS: "1" } + # Claude Code / Codex subscription instead of Ollama (no API key): + # env: { ERIS_IMPROVE_LOG_CALLS: "1", ERIS_LLM_MODEL: "claude-cli:haiku" } + - id: venue-arb-frozen + dir: venue-arb wallet: AGENT3_PRIVATE_KEY - description: base-agnostic cross-venue arbitrage (all active bases x all venues; LLM-driven via prompt.md) - env: { ERIS_AGENT_MODE: "prompt", ERIS_PROMPT_LOG_CALLS: "1" } - # env: { ERIS_AGENT_MODE: "prompt", ERIS_PROMPT_LOG_CALLS: "1", ERIS_LLM_MODEL: "claude-cli:haiku" } - - id: lst-carry + description: the same strategy with the improvement loop off — the control that says whether revising helped (ADR 0018 §5) + env: { ERIS_AGENT_FROZEN: "1" } + - id: multi-arb wallet: AGENT4_PRIVATE_KEY - description: liquid staking — stake for yield, or trade the LST redemption/market gap (LLM-driven via prompt.md) - env: { ERIS_AGENT_MODE: "prompt", ERIS_PROMPT_LOG_CALLS: "1" } - # env: { ERIS_AGENT_MODE: "prompt", ERIS_PROMPT_LOG_CALLS: "1", ERIS_LLM_MODEL: "claude-cli:haiku" } + description: base-agnostic cross-venue arbitrage (all active bases x all venues), revised in-run + env: { ERIS_IMPROVE_LOG_CALLS: "1" } + - id: lst-carry + wallet: AGENT5_PRIVATE_KEY + description: liquid staking — stake for yield, or trade the LST redemption/market gap, revised in-run + env: { ERIS_IMPROVE_LOG_CALLS: "1" } # config/lst.yaml is the same venue with a second competing participant and the calibration knobs # spelled out, if you want to look at the LST market on its own. diff --git a/config/lst.yaml b/config/lst.yaml index a0b8358..b7ef56b 100644 --- a/config/lst.yaml +++ b/config/lst.yaml @@ -127,25 +127,24 @@ agents: wallet: AGENT1_PRIVATE_KEY baseline: true description: does nothing (baseline) - # LLM-driven through prompt.md, matching the committed example.yaml roster: the venue's default - # path is the one a participant actually submits (ADR 0015 §2, issue #38). Needs OLLAMA_API_KEY - # in .env.local. Without a backend the run completes and this agent fails closed to noop, so if - # it never trades, check agents/lst-carry.jsonl for the reason before blaming the strategy. - # Remove the `env:` line to run the same strategy rule-based via agent.ts. + # Self-improving (agent.ts + improve.md, ADR 0018): the strategy trades every block and an LLM + # periodically rewrites it. Needs OLLAMA_API_KEY in .env.local, or ERIS_LLM_MODEL: "claude-cli" + # for a subscription CLI. Without a backend the run still completes — the revisions are recorded + # as failed and the strategy keeps trading unchanged. - id: lst-carry wallet: AGENT2_PRIVATE_KEY - description: liquid staking — stake for yield, or trade the redemption/market gap (LLM-driven via prompt.md) - env: { ERIS_AGENT_MODE: "prompt", ERIS_PROMPT_LOG_CALLS: "1" } - # Claude Code / Codex subscription instead of Ollama (no API key) — use this env line instead. - # This is the combination the prompt was verified on (see docs/guide/llm-agents.md): - # env: { ERIS_AGENT_MODE: "prompt", ERIS_PROMPT_LOG_CALLS: "1", ERIS_LLM_MODEL: "claude-cli:haiku" } - # The same strategy run rule-based (agent.ts) with a wider entry threshold: a second participant - # competing for the same dislocation, and a control for the prompt-driven one above. + description: liquid staking — stake for yield, or trade the redemption/market gap, revised in-run + env: { ERIS_IMPROVE_LOG_CALLS: "1" } + # Claude Code / Codex subscription instead of Ollama (no API key): + # env: { ERIS_IMPROVE_LOG_CALLS: "1", ERIS_LLM_MODEL: "claude-cli:haiku" } + # The same strategy with the improvement loop off and a wider entry threshold: a second participant + # competing for the same dislocation, and the frozen control for the self-improving one above + # (ADR 0018 §5 — without it there is no way to tell whether revising helped). - id: lst-carry-wide dir: lst-carry wallet: AGENT3_PRIVATE_KEY - description: lst-carry with a wider entry threshold (rule-based via agent.ts) - env: { ERIS_LST_SAFETY_BPS: "40" } + description: lst-carry with a wider entry threshold, improvement loop off (the frozen control) + env: { ERIS_LST_SAFETY_BPS: "40", ERIS_AGENT_FROZEN: "1" } - id: venue-arb wallet: AGENT4_PRIVATE_KEY description: WETH-only cross-venue arbitrage (keeps the AMM venues honest) diff --git a/config/regimes/calm-01.yaml b/config/regimes/calm.yaml similarity index 54% rename from config/regimes/calm-01.yaml rename to config/regimes/calm.yaml index 9351ed5..eabaa6b 100644 --- a/config/regimes/calm-01.yaml +++ b/config/regimes/calm.yaml @@ -1,18 +1,22 @@ -# config/regimes/calm-01.yaml — official regime: calm (normal market with no stress). ADR 0016 §2 +# config/regimes/calm.yaml — official regime: calm (normal market with no stress). ADR 0016 §2 / ADR 0017 §1 # -# regime = this file (market conditions) + seed. Defines the OU fair price / flow intensity / stress. -# Run: npm run backtest -- --regime calm-01 [--agents ] [--repeat N] +# regime = this file (market conditions), seed = supplied at run time. This file defines the OU fair +# price / flow intensity / stress; the seed picks the realized path out of that family. +# Run: npm run backtest -- --regime calm --seed 101 [--agents ] +# Or as part of a scenario matrix: npm run backtest -- --scenarios config/scenarios/public.yaml # -# Note (ADR 0016 §3): -# - blockTimeSec is part of the regime (fixed to the production value). Shortening is an explicit -# override for behavior checks and smoke tests only; runs whose scores you read must use this default. -# - The published regime's seed is one sample from the range. The production run's seed is a different sample (anti-overfitting). +# Note: +# - There is deliberately no `run.seed` here (ADR 0017 §1). The seed is the second axis of a +# scenario, not a property of the regime, so it must be given explicitly and a run cannot +# silently inherit one. +# - blockTimeSec and blocks are part of the regime (fixed to the production values). Shortening is +# an explicit override for behavior checks and smoke tests only; runs whose scores you read must +# use these defaults (ADR 0016 §3). # The roster is the default for pre-submission verification. Replaceable via --agents . run: - seed: 101 - blocks: 60 - seconds: 600 # end by block count (set well above blocks x blockTimeSec) + blocks: 360 # R: ADR 0017 §1 (crash window x3-4, ~72 LLM decisions, inside anvil's ~1050 history depth) + seconds: 1800 # end by block count (must stay well above blocks x blockTimeSec = 720s) blockTimeSec: 2 # fixed to the regime (ADR 0016 §2) protocols: [uniswap, balancer, curve, gmx, aave] economicGas: false diff --git a/config/regimes/cex-drift.yaml b/config/regimes/cex-drift.yaml new file mode 100644 index 0000000..52179df --- /dev/null +++ b/config/regimes/cex-drift.yaml @@ -0,0 +1,74 @@ +# config/regimes/cex-drift.yaml — official regime 1: reference price drifting away from on-chain price. +# ADR 0017 §1 +# +# What this regime tests: the fair price (the off-chain reference every agent can read from the +# PriceFeed, one block late) walks away from where the pools are trading and keeps going, instead of +# snapping back. Pool prices only move when someone trades, so a persistent drift opens a gap that +# stays open and widens -- the skill is noticing early and sizing into a move that has not finished. +# Run: npm run backtest -- --regime cex-drift --seed 303 [--agents ] +# +# The two knobs that make it this regime rather than `calm` (market.* is new in ADR 0017; before it +# the OU parameters were only reachable through raw env, so a regime could not set them): +# +# drift 0.0015/block -- a directional pull. Over 360 blocks that compounds to roughly +70% if +# nothing resisted it; mean reversion and the pools' own impact pull much +# of it back, so the realized path is a strong trend, not a moonshot. +# kappa 0.004 -- one fifth of calm's 0.02. Mean reversion is what normally erases β by +# run end (ADR 0003/0007); weakening it is precisely what lets a drift +# persist. It is deliberately not zero: with no anchor at all the price +# random-walks off and never comes back, which stops being a drift regime +# and starts being a lottery on the seed's direction. +# +# NOTE on scoring: this regime carries real β by construction, and the competition currently ranks on +# netPnlUsdc (ADR 0017 §4). Funding is USDC-only so nobody starts exposed, but an agent that ends the +# run holding inventory is marked at the drifted price. That is the intended difficulty here -- the +# drift is tradeable in both directions -- but it is the regime where the choice of scoring metric +# matters most, so re-check its calibration when the metric changes. +# +# There is deliberately no `run.seed` (ADR 0017 §1): the seed is supplied per scenario. + +run: + blocks: 360 # R: ADR 0017 §1 (long enough for a drift to develop and be traded) + seconds: 1800 # end by block count (must stay well above blocks x blockTimeSec = 720s) + blockTimeSec: 2 # fixed to the regime (ADR 0016 §2) + protocols: [uniswap, balancer, curve, gmx, aave] + economicGas: false + localDeploy: true + reportDir: ./runs + +market: + volatility: 0.004 # same per-block noise as calm: the regime is the trend, not the noise + kappa: 0.004 # weak mean reversion, so the drift is allowed to persist + drift: 0.0015 # the directional pull that defines this regime + +funding: + wethWei: "0" # USDC-only distribution: nobody starts already exposed to the drift (ADR 0017 §4) + usdcUnits: "25000000000" + +limits: + agentWethWei: "1000000000000000000" + agentUsdcUnits: "5000000000" + +flow: + uninformedMaxWethWei: "1000000000000000000" + informedMaxWethWei: "2000000000000000000" + balancerMaxWethWei: "1000000000000000000" + curveMaxWethWei: "1000000000000000000" + informedArbFeeBps: 30 + uninformedArrivalRate: "0.9" + uninformedSizeSigma: "1.0" + gmxArrivalRate: "0.75" + gmxSizeSigma: "1.0" + aaveActorSizeSigma: "1.0" + +agents: + - id: noop + wallet: AGENT1_PRIVATE_KEY + baseline: true + description: does nothing (baseline) + - id: venue-arb + wallet: AGENT2_PRIVATE_KEY + description: WETH-only cross-venue arbitrage + - id: multi-arb + wallet: AGENT3_PRIVATE_KEY + description: base-agnostic cross-venue arbitrage (all active bases x all venues) diff --git a/config/regimes/crash.yaml b/config/regimes/crash.yaml new file mode 100644 index 0000000..39710d3 --- /dev/null +++ b/config/regimes/crash.yaml @@ -0,0 +1,70 @@ +# config/regimes/crash.yaml — official regime 6: crash event (price gap). ADR 0017 §1 +# +# A seed-randomized trapezoidal crash on the same market conditions as calm, with no victims: this +# regime is about surviving and trading a price gap, not about the lending side. The liquidation wave +# lives in `lending-incident`, which is the same shock plus Aave victims. +# Run: npm run backtest -- --regime crash --seed 606 [--agents ] +# +# INCOMPLETE (issue #52): the other half of this regime is the liquidity withdrawal -- LPs pulling +# depth so that the same notional costs far more slippage while the gap is open. Until that event +# type exists, every pool keeps its block-0 depth for the whole run, which makes this regime a +# "larger opportunity" regime rather than a crash: the gap grows but the cost of taking it does not. +# Read scores from it with that in mind, and re-calibrate the magnitude once #52 lands (a thinner +# book moves price further for the same flow, so the two knobs interact). +# +# There is deliberately no `run.seed` (ADR 0017 §1): the seed is supplied per scenario. +# blockTimeSec and blocks are part of the regime (ADR 0016 §3). + +run: + blocks: 360 # R: ADR 0017 §1 (crash window x3-4, ~72 LLM decisions, inside anvil's ~1050 history depth) + seconds: 1800 # a stress run auto-disables the time limit and ends by block count (ADR 0009) + blockTimeSec: 2 + protocols: [uniswap, balancer, curve, gmx, aave] + economicGas: false + localDeploy: true + reportDir: ./runs + +funding: + wethWei: "0" # USDC-only distribution: netPnlUsdc carries β, so nobody may start already exposed (ADR 0017 §4) + usdcUnits: "25000000000" + +limits: + agentWethWei: "1000000000000000000" + agentUsdcUnits: "5000000000" + +flow: + uninformedMaxWethWei: "1000000000000000000" + informedMaxWethWei: "2000000000000000000" + balancerMaxWethWei: "1000000000000000000" + curveMaxWethWei: "1000000000000000000" + informedArbFeeBps: 30 + uninformedArrivalRate: "0.9" + uninformedSizeSigma: "1.0" + gmxArrivalRate: "0.75" + gmxSizeSigma: "1.0" + aaveActorSizeSigma: "1.0" + +stress: + # Ranges, not values: the seed samples them (ADR 0009). Deeper than lending-incident's crash because + # nothing here has to stay inside a victim's health-factor calibration. + events: + - { + type: crash, + magnitudeRange: [0.15, 0.22], + windowFrac: [0.25, 0.7], + rampBlocks: 3, + holdBlocks: 6, + decayBlocks: 8, + } + +agents: + - id: noop + wallet: AGENT1_PRIVATE_KEY + baseline: true + description: does nothing (baseline) + - id: venue-arb + wallet: AGENT2_PRIVATE_KEY + description: WETH-only cross-venue arbitrage + - id: multi-arb + wallet: AGENT3_PRIVATE_KEY + description: base-agnostic cross-venue arbitrage (all active bases x all venues) diff --git a/config/regimes/informed-flow.yaml b/config/regimes/informed-flow.yaml new file mode 100644 index 0000000..ce90017 --- /dev/null +++ b/config/regimes/informed-flow.yaml @@ -0,0 +1,76 @@ +# config/regimes/informed-flow.yaml — official regime 2: correlated directional order flow. +# ADR 0017 §1 +# +# What this regime tests: the market gets pushed one way, hard, for a stretch of blocks, and then +# turns. Every venue leans together, so the gap that opens is against fair rather than between +# venues -- the usual cross-venue arbitrage has much less to bite on, and the skill becomes reading +# a one-sided tape: how far the flow will run, when it turns, and whether to lean against it. +# Run: npm run backtest -- --regime informed-flow --seed 404 [--agents ] +# +# The knobs that make it this regime rather than `calm`: +# +# uninformedPersistBlocks 12 -- a venue's uninformed direction is held for 12 blocks instead of +# being redrawn each block, so pressure accumulates instead of +# cancelling out. 360 blocks gives ~30 direction changes. +# uninformedTrendCorrelation 1 -- new in ADR 0017. Every venue takes the market-wide direction +# rather than its own. At the default 0 the per-venue hash +# deliberately splits the venues up/down to manufacture a +# cross-venue spread, which is the opposite shape from this regime. +# uninformedMax 3 WETH -- 3x calm on uniswap. Correlated pressure only matters if it is big +# enough to move the mid against the informed flow pushing back. +# +# The informed flow bot (which pulls price back toward fair) is left at calm's setting on purpose: +# it is the counterparty that makes the push finite. Raising it too would just cancel the regime out. +# +# CAVEAT, and the reason balancer/curve are NOT raised: `balancerMaxWethWei` and `curveMaxWethWei` +# are a single cap used for BOTH the uninformed and the informed leg on those venues +# (`ammMax` in core/src/flow/logic.ts maps each to `[cap, cap]`); only uniswap has separate +# `uninformedMaxWethWei` / `informedMaxWethWei`. Raising them here would have tripled the +# gap-closing flow too -- cancelling out the regime on two of three venues and leaving the venues +# with asymmetric reversion strength, which is a spurious cross-venue signal in a regime that is +# supposed to have none. So the directional push is uniswap-sized, and balancer/curve stay at calm. +# +# There is deliberately no `run.seed` (ADR 0017 §1): the seed is supplied per scenario. + +run: + blocks: 360 # R: ADR 0017 §1 (~30 direction changes at persistBlocks=12) + seconds: 1800 # end by block count (must stay well above blocks x blockTimeSec = 720s) + blockTimeSec: 2 # fixed to the regime (ADR 0016 §2) + protocols: [uniswap, balancer, curve, gmx, aave] + economicGas: false + localDeploy: true + reportDir: ./runs + +funding: + wethWei: "0" # USDC-only distribution: nobody starts already exposed (ADR 0017 §4) + usdcUnits: "25000000000" + +limits: + agentWethWei: "1000000000000000000" + agentUsdcUnits: "5000000000" + +flow: + uninformedMaxWethWei: "3000000000000000000" # 3x calm: the pressure has to be able to move the mid + uninformedPersistBlocks: 12 # hold a direction long enough for it to accumulate + uninformedTrendCorrelation: 1.0 # every venue leans the same way (ADR 0017 regime 2) + informedMaxWethWei: "2000000000000000000" # unchanged: this is the force pushing back + balancerMaxWethWei: "1000000000000000000" # one cap for both legs on this venue -- see the caveat above + curveMaxWethWei: "1000000000000000000" # ditto + informedArbFeeBps: 30 + uninformedArrivalRate: "0.9" + uninformedSizeSigma: "1.0" + gmxArrivalRate: "0.75" + gmxSizeSigma: "1.0" + aaveActorSizeSigma: "1.0" + +agents: + - id: noop + wallet: AGENT1_PRIVATE_KEY + baseline: true + description: does nothing (baseline) + - id: venue-arb + wallet: AGENT2_PRIVATE_KEY + description: WETH-only cross-venue arbitrage + - id: multi-arb + wallet: AGENT3_PRIVATE_KEY + description: base-agnostic cross-venue arbitrage (all active bases x all venues) diff --git a/config/regimes/crash-01.yaml b/config/regimes/lending-incident.yaml similarity index 68% rename from config/regimes/crash-01.yaml rename to config/regimes/lending-incident.yaml index aea4379..0d4bae4 100644 --- a/config/regimes/crash-01.yaml +++ b/config/regimes/lending-incident.yaml @@ -1,9 +1,13 @@ -# config/regimes/crash-01.yaml — official regime: crash (trapezoidal crash + Aave liquidation opportunities). ADR 0016 §2 +# config/regimes/lending-incident.yaml — official regime 4: collateral crash + a liquidation wave. +# ADR 0016 §2 / ADR 0017 §1 # -# On top of the same market conditions as calm-01, overlays a seed-randomized crash (ADR 0009) and -# a group of liquidation-target victims (excluded from scoring). The liquidator agent receives the -# victim addresses via ERIS_LIQUIDATION_VICTIMS. -# Run: npm run backtest -- --regime crash-01 [--agents ] [--repeat N] +# On top of the same market conditions as calm, overlays a seed-randomized crash (ADR 0009) and a +# group of liquidation-target victims (excluded from scoring). The liquidator agent receives the +# victim addresses via ERIS_LIQUIDATION_VICTIMS. What this regime tests that `crash` does not is the +# lending side: whether an agent can find and take undercollateralized positions under time pressure. +# Run: npm run backtest -- --regime lending-incident --seed 202 [--agents ] +# +# There is deliberately no `run.seed` (ADR 0017 §1): the seed is supplied per scenario. # # Calibration coupling (ADR 0009 §4 / CLAUDE.md): # - Condition to open victims: HF0 >= LT/(0.97*LTV) (verified on-chain against measured reserve values, fail-fast) @@ -12,9 +16,8 @@ # blockTimeSec is part of the regime (ADR 0016 §3). Runs whose scores you read must use this default. run: - seed: 202 - blocks: 60 - seconds: 600 # a stress run auto-disables the time limit and ends by block count (ADR 0009) + blocks: 360 # R: ADR 0017 §1 (crash window x3-4, ~72 LLM decisions, inside anvil's ~1050 history depth) + seconds: 1800 # a stress run auto-disables the time limit and ends by block count (ADR 0009) blockTimeSec: 2 # fixed to the regime (ADR 0016 §2) protocols: [uniswap, balancer, curve, gmx, aave] economicGas: false diff --git a/config/regimes/lst-01.yaml b/config/regimes/lst.yaml similarity index 84% rename from config/regimes/lst-01.yaml rename to config/regimes/lst.yaml index 895b49c..e75f7cb 100644 --- a/config/regimes/lst-01.yaml +++ b/config/regimes/lst.yaml @@ -1,9 +1,9 @@ -# config/regimes/lst-01.yaml — official regime: the liquid staking venue (issue #38). ADR 0016 §2 +# config/regimes/lst.yaml — the liquid staking venue (issue #38). ADR 0016 §2 # -# regime = this file (market conditions) + seed. Same discipline as calm-01/crash-01: ranges rather -# than values, blockTimeSec fixed to the production value, and the published seed is one sample of -# many (the production run uses a different one). -# Run: npm run backtest -- --regime lst-01 [--agents ] [--repeat N] +# Not part of the ADR 0017 competition set (the seven regimes there do not include the LST venue). +# Kept as an official regime for venue-level verification, and it follows the same discipline: +# ranges rather than values, blockTimeSec fixed to the production value, seed supplied at run time. +# Run: npm run backtest -- --regime lst --seed 301 [--agents ] # # What this regime tests that the others do not: an asset with two prices at once. The vault owes # `redemptionRateWeth` per share but only through a withdrawal queue; the LST/WETH pool pays @@ -12,11 +12,10 @@ # pay for it, not par. # # Funding note: this venue is WETH-denominated and cannot be traded from a USDC-only wallet, so -# unlike calm-01 this regime hands out WETH. That reintroduces price drift into netPnlUsdc — read +# unlike calm this regime hands out WETH. That reintroduces price drift into netPnlUsdc — read # alphaUsdc, and read scores within a run rather than across price paths. run: - seed: 301 blocks: 60 seconds: 600 # end by block count (set well above blocks x blockTimeSec) blockTimeSec: 2 # fixed to the regime (ADR 0016 §2) diff --git a/config/regimes/whale.yaml b/config/regimes/whale.yaml new file mode 100644 index 0000000..c8b09e2 --- /dev/null +++ b/config/regimes/whale.yaml @@ -0,0 +1,93 @@ +# config/regimes/whale.yaml — official regime 3: single large orders that move the mid. ADR 0017 §1 +# +# What this regime tests: at a handful of unannounced blocks, one very large market order prints and +# knocks a pool away from fair. Fair itself does not move at all -- that is the difference from +# `crash`, where fair drops and the pools lag behind it. Here the pool is the thing that is wrong, so +# the trade is to take the other side of the print and let the venue reprice, and the risk is that +# the dislocation is bigger than the depth you are willing to cross. +# Run: npm run backtest -- --regime whale --seed 505 [--agents ] +# +# Calibration: +# magnitudeRange [25, 60] order size in WETH. Against the deployed Uniswap pool this is enough to +# move the mid clearly without emptying one side of the book. Recalibrate +# if the deployer's pool depth changes -- the whole event is impact, so it +# is entirely a function of depth. +# MEASURED, needs revisiting: a 30-block smoke run of this regime paid the +# bundled arb agents ~1,600 and ~2,800 USDC on 25,000 of capital. That is +# a very large edge for four prints, which suggests the size is generous +# enough that simply being present pays and the regime may not separate +# strong strategies from adequate ones. Left as-is pending the ADR 0017 §5 +# pilot, which is where the size gets settled against measured +# discrimination rather than against a guess. +# side (unset) the seed picks buy or sell per event. Leaving it unset is what stops the +# direction being memorizable across the published seeds. +# venue defaults to uniswap, the deepest venue, so the size has to be real. The +# second and third whales are pointed at balancer and curve so a strategy +# cannot just watch one pool. +# 4 events spread across the run via disjoint windowFrac bands. Several prints per +# run mean the score is not decided by one draw (ADR 0017 §5). +# +# The whale trades from its own wallet, endowed at setup from the resolved schedule, so it neither +# drains the ordinary flow wallets nor shows up in blocks.csv as background flow. +# +# There is deliberately no `run.seed` (ADR 0017 §1): the seed is supplied per scenario. + +run: + blocks: 360 # R: ADR 0017 §1 + seconds: 1800 # a stress run auto-disables the time limit and ends by block count (ADR 0009) + blockTimeSec: 2 # fixed to the regime (ADR 0016 §2) + protocols: [uniswap, balancer, curve, gmx, aave] + economicGas: false + localDeploy: true + reportDir: ./runs + +funding: + wethWei: "0" # USDC-only distribution: nobody starts already exposed (ADR 0017 §4) + usdcUnits: "25000000000" + +limits: + agentWethWei: "1000000000000000000" + agentUsdcUnits: "5000000000" + +flow: + uninformedMaxWethWei: "1000000000000000000" + informedMaxWethWei: "2000000000000000000" + balancerMaxWethWei: "1000000000000000000" + curveMaxWethWei: "1000000000000000000" + informedArbFeeBps: 30 + uninformedArrivalRate: "0.9" + uninformedSizeSigma: "1.0" + gmxArrivalRate: "0.75" + gmxSizeSigma: "1.0" + aaveActorSizeSigma: "1.0" + +stress: + # Ranges, not values: the seed samples both the size and the block (ADR 0009). The windows are + # disjoint so the four prints land spread out rather than clustering into one shock. + events: + - { type: whale, magnitudeRange: [25, 60], windowFrac: [0.10, 0.25] } + - { + type: whale, + magnitudeRange: [25, 60], + windowFrac: [0.3, 0.45], + venue: balancer, + } + - { + type: whale, + magnitudeRange: [25, 60], + windowFrac: [0.5, 0.65], + venue: curve, + } + - { type: whale, magnitudeRange: [25, 60], windowFrac: [0.7, 0.85] } + +agents: + - id: noop + wallet: AGENT1_PRIVATE_KEY + baseline: true + description: does nothing (baseline) + - id: venue-arb + wallet: AGENT2_PRIVATE_KEY + description: WETH-only cross-venue arbitrage + - id: multi-arb + wallet: AGENT3_PRIVATE_KEY + description: base-agnostic cross-venue arbitrage (all active bases x all venues) diff --git a/config/scenarios/public.yaml b/config/scenarios/public.yaml new file mode 100644 index 0000000..1e4e9e3 --- /dev/null +++ b/config/scenarios/public.yaml @@ -0,0 +1,29 @@ +# config/scenarios/public.yaml — the public scenario set (ADR 0017 §2). +# +# A scenario is (regime, seed). This file is the cartesian product of the two lists below, so the +# set is `regimes.length * seeds.length` scenarios, each addressed as `#`. +# +# Public means both halves are published: the regime YAMLs (including the stress ranges) and the +# seeds. Participants tune against this set before the competition. The private set is the same +# regimes with a disjoint, unpublished seed list -- same distribution family, different realizations +# (ADR 0009's "give ranges, not values" applied to the competition itself). +# +# Because the generator is open source, you can and should sample your own seeds instead of +# overfitting to these five. Generalizing across the distribution is the thing being measured; the +# published seeds are five draws from it, not the target. +# +# Run: npm run backtest -- --scenarios config/scenarios/public.yaml --agents + +# Six of the seven regimes ADR 0017 lists. The seventh, depeg, waits on the scoring rework: its shock +# moves the unit of account itself, which the current USDC-denominated score cannot express. +# `crash` is also incomplete until issue #52 lands its liquidity-withdrawal half -- today it is the +# price gap alone. +regimes: + - calm + - cex-drift + - informed-flow + - whale + - lending-incident + - crash + +seeds: [101, 202, 303, 404, 505] diff --git a/core/src/backtest/standings.ts b/core/src/backtest/standings.ts new file mode 100644 index 0000000..8fd5dae --- /dev/null +++ b/core/src/backtest/standings.ts @@ -0,0 +1,204 @@ +// Scenario-matrix aggregation (ADR 0017 §4). +// +// Turns a scenario x agent score matrix into standings in three layers: +// 1. scenario score -- the raw metric from that scenario's single run +// 2. scenario z -- normalized across agents *within the scenario* +// 3. total -- mean over scenarios inside a regime, then mean over regimes (equal weight) +// +// Why normalize per scenario rather than per regime: every agent in a scenario ran in the same world +// (ADR 0017 §1 co-location), so comparing them to each other is the one comparison the design +// actually guarantees is fair. Doing it per scenario also flattens the seed-to-seed scale spread +// inside a regime, not just the regime-to-regime spread, which is a strict improvement over +// normalizing the regime as one pool. +// +// Pure. No filesystem, no chain. The CLI collects summaries and hands them here, so the aggregation +// rule can be re-run over a stored matrix.json when the scoring method changes (it is expected to -- +// ADR 0017 leaves the metric and the formula open). + +// How far below the worst finisher a disqualified agent lands, in z units (standard deviations). +// Being disqualified has to be worse than finishing last, or crashing becomes a strategy; and it has +// to be a bounded penalty rather than -Infinity, or one bad scenario decides the whole competition. +export const DISQUALIFIED_Z_PENALTY = 1; + +export type ScoringMetric = "netPnlUsdc" | "alphaUsdc"; + +export type AgentScore = { + id: string; + netPnlUsdc?: number; + alphaUsdc?: number; + // Set when the agent must not be credited with a score for this scenario: it broke a rule, its + // process died, or it never reported. The reason is carried through to the report. + disqualified?: string; +}; + +export type ScenarioResult = { + regime: string; + seed: number; + // Absent when the run produced no summary at all. Such scenarios are excluded from the + // aggregation and reported separately -- a failure of the environment must not be charged to the + // participants, and a row of zeros would silently dilute everyone's average. + agents?: AgentScore[]; + runDir?: string; + error?: string; +}; + +export type ScenarioStanding = { + regime: string; + seed: number; + scores: Record; + z: Record; + disqualified: Record; +}; + +export type Standings = { + metric: ScoringMetric; + agents: Array<{ + id: string; + total: number; + byRegime: Record; + scenariosScored: number; + disqualifications: number; + }>; + regimes: string[]; + scenarios: ScenarioStanding[]; + excludedScenarios: Array<{ regime: string; seed: number; error?: string }>; +}; + +export function scenarioId(regime: string, seed: number): string { + return `${regime}#${seed}`; +} + +function mean(values: number[]): number { + return values.reduce((a, b) => a + b, 0) / values.length; +} + +// Population standard deviation: the agents in a scenario are the whole set being compared, not a +// sample drawn from a larger one. +function stdev(values: number[], mu: number): number { + return Math.sqrt(mean(values.map((v) => (v - mu) ** 2))); +} + +function metricOf( + agent: AgentScore, + metric: ScoringMetric, +): number | undefined { + const raw = agent[metric]; + return typeof raw === "number" && Number.isFinite(raw) ? raw : undefined; +} + +// z-scores for one scenario. Agents that finished are normalized against each other; agents that did +// not are placed below all of them. +export function scenarioZScores( + agents: AgentScore[], + metric: ScoringMetric, +): { + z: Record; + scores: Record; + disqualified: Record; +} { + const scores: Record = {}; + const disqualified: Record = {}; + const finishers: string[] = []; + + for (const agent of agents) { + const value = metricOf(agent, metric); + if (agent.disqualified !== undefined) { + disqualified[agent.id] = agent.disqualified; + // Keep the raw number when there is one: the report should still show what the agent was + // holding when it was disqualified, even though the number does not earn it any z. + if (value !== undefined) scores[agent.id] = value; + continue; + } + if (value === undefined) { + // A finisher with no readable metric is a reporting failure, not a zero. Treat it the same as + // any other agent we cannot score rather than crediting it with an average result. + disqualified[agent.id] = `no ${metric} in summary`; + continue; + } + scores[agent.id] = value; + finishers.push(agent.id); + } + + const z: Record = {}; + const values = finishers.map((id) => scores[id]); + if (finishers.length > 0) { + const mu = mean(values); + const sd = stdev(values, mu); + // sd === 0 means every finisher tied. No one out-traded anyone, so no one gains ground. + for (const id of finishers) z[id] = sd > 0 ? (scores[id] - mu) / sd : 0; + } + const worst = + finishers.length > 0 ? Math.min(...finishers.map((id) => z[id])) : 0; + for (const id of Object.keys(disqualified)) + z[id] = finishers.length > 0 ? worst - DISQUALIFIED_Z_PENALTY : 0; + + return { z, scores, disqualified }; +} + +export function computeStandings( + results: ScenarioResult[], + metric: ScoringMetric, +): Standings { + const scored: ScenarioStanding[] = []; + const excluded: Standings["excludedScenarios"] = []; + + for (const result of results) { + if (!result.agents || result.agents.length === 0) { + excluded.push({ + regime: result.regime, + seed: result.seed, + error: result.error ?? "no summary.json", + }); + continue; + } + const { z, scores, disqualified } = scenarioZScores(result.agents, metric); + scored.push({ + regime: result.regime, + seed: result.seed, + scores, + z, + disqualified, + }); + } + + // Regime order follows first appearance so the report reads in the order the matrix was run. + const regimes: string[] = []; + for (const s of scored) + if (!regimes.includes(s.regime)) regimes.push(s.regime); + + const agentIds: string[] = []; + for (const s of scored) + for (const id of Object.keys(s.z)) + if (!agentIds.includes(id)) agentIds.push(id); + + const agents = agentIds.map((id) => { + const byRegime: Record = {}; + let scenariosScored = 0; + let disqualifications = 0; + for (const regime of regimes) { + const inRegime = scored.filter((s) => s.regime === regime && id in s.z); + if (inRegime.length === 0) continue; + byRegime[regime] = mean(inRegime.map((s) => s.z[id])); + scenariosScored += inRegime.length; + disqualifications += inRegime.filter((s) => id in s.disqualified).length; + } + const present = regimes.filter((r) => r in byRegime); + return { + id, + // Equal weight per regime, so a regime with more seeds does not carry more of the total. + total: present.length > 0 ? mean(present.map((r) => byRegime[r])) : 0, + byRegime, + scenariosScored, + disqualifications, + }; + }); + + agents.sort((a, b) => b.total - a.total); + return { + metric, + agents, + regimes, + scenarios: scored, + excludedScenarios: excluded, + }; +} diff --git a/core/src/cli/backtest.ts b/core/src/cli/backtest.ts index 9039256..9cfbb11 100644 --- a/core/src/cli/backtest.ts +++ b/core/src/cli/backtest.ts @@ -1,22 +1,46 @@ -// participant backtest CLI (ADR 0016. B1 realtime replay). +// participant backtest CLI + competition scenario runner (ADR 0016 B1 realtime replay, ADR 0017 §3). // -// npm run backtest -- --regime [--agents ] [--repeat N] +// npm run backtest -- --regime --seed [--agents ] [--repeat N] +// npm run backtest -- --scenarios config/scenarios/public.yaml [--agents ] [--metric ...] // [--port 8547] [--state backtest/state] [--keep-anvil] -// [--seed N] [--blocks N] [--seconds N] [--protocols a,b] +// [--blocks N] [--seconds N] [--protocols a,b] [--score-every N] // -// Starts a dedicated anvil with a distributed state dump loaded (generated by gen:state-dump) and -// replays an official regime (config/regimes/*.yaml + seed) via the existing coordinator. --repeat N -// invokes the coordinator repeatedly in the same process, with resetFork's evm_snapshot/evm_revert -// guaranteeing a clean slice between runs (scoring reconstruction finishes at the end of each run = -// before the next revert). +// Starts one anvil with the distributed state dump loaded (generated by gen:state-dump) and replays +// scenarios through the existing coordinator. A scenario is (regime, seed): the regime YAML holds the +// market conditions and the seed picks the realization, so the seed is passed in rather than baked +// into the regime (ADR 0017 §1). +// +// Every scenario is snapshot -> run -> reconstruct -> revert on that same anvil, which is what makes +// "no forking, many runs a day on one devnet" work: resetFork's evm_snapshot/evm_revert gives each +// run a clean slice, and scoring reconstruction finishes before the next revert erases the history +// it reads. +// +// With --scenarios the whole matrix is replayed and two artifacts are written: +// matrix.json raw per-scenario, per-agent scores (both netPnlUsdc and alphaUsdc) +// standings.json the ranking derived from them (regime-internal z-score, equal weight per regime) +// The ranking is a derived view on purpose: the scoring rule is expected to change, and matrix.json +// is what lets it be recomputed without re-running anything (ADR 0017 §4). // // Dependency-light for the same reason as sim-realtime.ts: before importing sdk/constants we must set // ERIS_LOCAL_DEPLOY=1 and finish syncing the constants.local.ts fingerprint. The coordinator is // dynamically imported last. import { spawn, type ChildProcess } from "node:child_process"; -import { existsSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { + existsSync, + mkdirSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; import { basename, join, resolve } from "node:path"; import { parse as parseYaml, stringify as stringifyYaml } from "yaml"; +import { + computeStandings, + scenarioId, + type AgentScore, + type ScenarioResult, + type ScoringMetric, +} from "../backtest/standings.js"; import { gitHead, isAnvilUp, @@ -32,21 +56,30 @@ import { const ROOT = process.cwd(); // npm scripts run at the repo root -const USAGE = `usage: npm run backtest -- --regime [options] - --regime config/regimes/.yaml (or a YAML path). required +const USAGE = `usage: npm run backtest -- (--regime --seed | --scenarios ) [options] + --regime config/regimes/.yaml (or a YAML path). requires --seed + --seed the scenario's seed. regimes no longer carry one (ADR 0017 §1) + --scenarios scenario set (regimes x seeds) to replay as a matrix, e.g. config/scenarios/public.yaml --agents replace the regime's default agents with a roster file (YAML/JSON) - --repeat repeat the same regime N times (snapshot/revert. default 1) + --metric scoring metric for standings: netPnlUsdc (default) or alphaUsdc + --repeat repeat each scenario N times (calibration diagnostic; standings take the median. default 1) --port port for the backtest-only anvil (default 8547) --state state dump directory (default ${STATE_DIR_DEFAULT}) --keep-anvil keep anvil running after exit (for debugging) - --seed/--blocks/--seconds/--protocols/--economic-gas + --blocks/--seconds/--protocols/--economic-gas/--score-every one-off overrides of regime values (for smoke tests. runs you read results from use regime defaults)`; -type AgentSummary = { id: string; alphaUsdc?: number; netPnlUsdc?: number }; +type AgentSummary = { + id: string; + alphaUsdc?: number; + netPnlUsdc?: number; + processExitedEarly?: string; +}; type RunSummary = { runDir: string; blocksProcessed?: number; agents: AgentSummary[]; + violations: Array<{ ownerId?: string }>; }; function readRunSummary(runDir: string): RunSummary | undefined { @@ -55,14 +88,88 @@ function readRunSummary(runDir: string): RunSummary | undefined { const parsed = JSON.parse(readFileSync(path, "utf8")) as { blocksProcessed?: number; agents?: AgentSummary[]; + violations?: Array<{ ownerId?: string }>; }; return { runDir, blocksProcessed: parsed.blocksProcessed, agents: parsed.agents ?? [], + violations: parsed.violations ?? [], }; } +// summary.json -> the scores the aggregation consumes, applying ADR 0017 §4's disqualification rules. +// Disqualification is deliberately not "score 0": a zero would place a crashed agent mid-pack in a +// scenario where everyone lost money, which makes dying a viable tactic. +function scoresFromSummary( + summary: RunSummary, + expectedAgentIds: string[], +): AgentScore[] { + const offenders = new Set( + summary.violations + .map((v) => v.ownerId) + .filter((id): id is string => typeof id === "string"), + ); + const reported = new Map(summary.agents.map((a) => [a.id, a])); + const ids = + expectedAgentIds.length > 0 ? expectedAgentIds : [...reported.keys()]; + return ids.map((id) => { + const agent = reported.get(id); + if (!agent) + return { id, disqualified: "absent from summary.json (did not run)" }; + const disqualified = offenders.has(id) + ? "priority fee cap violation" + : agent.processExitedEarly !== undefined + ? `process exited early: ${agent.processExitedEarly}` + : undefined; + return { + id, + netPnlUsdc: agent.netPnlUsdc, + alphaUsdc: agent.alphaUsdc, + ...(disqualified !== undefined ? { disqualified } : {}), + }; + }); +} + +// Fold N repeats of one scenario into a single per-agent record by picking, for each agent, the +// repeat whose ranking metric is the median and reporting *that repeat's whole record*. +// +// Not a per-metric median: taking the median of netPnlUsdc and of alphaUsdc independently can report +// a pair that no single run produced, and then the run directory recorded alongside explains +// neither. Since --repeat exists so a calibration number can be traced back to a run, the reported +// numbers have to come from one. +function foldRepeats( + runs: AgentScore[][], + metric: ScoringMetric, +): AgentScore[] { + if (runs.length === 1) return runs[0]; + const ids: string[] = []; + for (const run of runs) + for (const a of run) if (!ids.includes(a.id)) ids.push(a.id); + return ids.map((id) => { + const entries = runs + .map((run) => run.find((a) => a.id === id)) + .filter((a): a is AgentScore => a !== undefined); + // Disqualified in any repeat means disqualified: the failure is a property of the agent, and + // letting a lucky repeat wash it out would defeat the point of detecting it. + const failed = entries.find((a) => a.disqualified !== undefined); + const scored = entries.filter( + (a) => typeof a[metric] === "number" && Number.isFinite(a[metric]), + ); + const chosen = + scored.length > 0 + ? scored.sort((a, b) => (a[metric] as number) - (b[metric] as number))[ + Math.floor((scored.length - 1) / 2) + ] + : entries[0]; + return { + ...chosen, + id, + ...(failed ? { disqualified: failed.disqualified } : {}), + }; + }); +} + // Sync constants.local.ts with the deployments bundled in the state manifest (ADR 0016 §2). // On mismatch, extract deployments from the manifest and regenerate; if it still doesn't match, // the state dump and repo version are a mismatched combination -> fail-fast. @@ -93,16 +200,68 @@ async function syncConstants( ); } +// One cell of the evaluation matrix. `#` is the address used in every report +// (ADR 0017 §1); the regime supplies the market conditions and the seed picks the realization. +type Scenario = { regime: string; seed: number; regimePath: string }; + +// --scenarios : { regimes: string[], seeds: number[] } expanded as a cartesian product. +function loadScenarioSet(root: string, path: string): Scenario[] { + const abs = resolve(root, path); + if (!existsSync(abs)) + throw new Error(`scenario set not found: ${abs} (--scenarios)`); + const doc = parseYaml(readFileSync(abs, "utf8")) as { + regimes?: unknown; + seeds?: unknown; + }; + const regimes = doc?.regimes; + const seeds = doc?.seeds; + if (!Array.isArray(regimes) || regimes.length === 0) + throw new Error(`${abs} must contain a non-empty "regimes" array`); + if (!Array.isArray(seeds) || seeds.length === 0) + throw new Error(`${abs} must contain a non-empty "seeds" array`); + const out: Scenario[] = []; + for (const regime of regimes) { + if (typeof regime !== "string") + throw new Error(`${abs}: every entry of "regimes" must be a string`); + // Resolve now rather than at run time: a typo in the set should fail before anvil starts, not + // three hours into a matrix. + const regimePath = resolveRegimePath(root, regime); + for (const seed of seeds) { + if (!Number.isInteger(seed)) + throw new Error(`${abs}: every entry of "seeds" must be an integer`); + out.push({ regime, seed: seed as number, regimePath }); + } + } + return out; +} + async function main(): Promise { const flags = parseFlags(process.argv); - if (!flags.regime) { - console.error(USAGE); - throw new Error("--regime is required"); - } if (flags.config) throw new Error( "for backtest use --regime, not --config (the regime YAML is the run config itself)", ); + if (flags.regime && flags.scenarios) + throw new Error("--regime and --scenarios are mutually exclusive"); + if (!flags.regime && !flags.scenarios) { + console.error(USAGE); + throw new Error("one of --regime or --scenarios is required"); + } + if (flags.scenarios && flags.seed !== undefined) + throw new Error( + "--seed does not apply to --scenarios (the set supplies the seeds)", + ); + + const metric: ScoringMetric = + flags.metric === "alphaUsdc" + ? "alphaUsdc" + : flags.metric === undefined || flags.metric === "netPnlUsdc" + ? "netPnlUsdc" + : (() => { + throw new Error( + `--metric must be netPnlUsdc or alphaUsdc (got ${flags.metric})`, + ); + })(); const repeat = Number(flags.repeat ?? "1"); if (!Number.isInteger(repeat) || repeat < 1) @@ -112,8 +271,31 @@ async function main(): Promise { const port = Number(flags.port ?? "8547"); const rpcUrl = `http://127.0.0.1:${port}`; const stateDirAbs = resolve(ROOT, flags.state ?? STATE_DIR_DEFAULT); - const regimePath = resolveRegimePath(ROOT, flags.regime); - const regimeName = basename(regimePath).replace(/\.ya?ml$/, ""); + + const matrixMode = flags.scenarios !== undefined; + let scenarios: Scenario[]; + if (matrixMode) { + scenarios = loadScenarioSet(ROOT, flags.scenarios); + } else { + // Regimes no longer carry a seed (ADR 0017 §1), so an omitted --seed used to mean "silently + // score seed 1". Fail instead: a scenario without a seed is not a scenario. + if (flags.seed === undefined) + throw new Error( + "--seed is required with --regime (regimes no longer carry run.seed; ADR 0017 §1). " + + "For a whole set use --scenarios config/scenarios/public.yaml", + ); + const seed = Number(flags.seed); + if (!Number.isInteger(seed)) + throw new Error(`--seed must be an integer (got ${flags.seed})`); + const regimePath = resolveRegimePath(ROOT, flags.regime); + scenarios = [ + { + regime: basename(regimePath).replace(/\.ya?ml$/, ""), + seed, + regimePath, + }, + ]; + } // ---- Validate the state manifest + sync constants (done before importing the coordinator) ---- const { manifest, statePath } = readStateManifest(stateDirAbs); @@ -148,14 +330,14 @@ async function main(): Promise { }; agents?: unknown; } & Record; - const regimeDoc = parseYaml(readFileSync(regimePath, "utf8")) as RegimeDoc; const runOverrides: Record = {}; if (flags.protocols) runOverrides.protocols = flags.protocols.split(",").map((s) => s.trim()); - if (flags.seed !== undefined) runOverrides.seed = Number(flags.seed); if (flags.blocks !== undefined) runOverrides.blocks = Number(flags.blocks); if (flags.seconds !== undefined) runOverrides.seconds = Number(flags.seconds); + if (flags["score-every"] !== undefined) + runOverrides.scoreEvery = Number(flags["score-every"]); if (flags["economic-gas"] !== undefined) runOverrides.economicGas = flags["economic-gas"] === "1" || flags["economic-gas"] === "true"; @@ -175,41 +357,107 @@ async function main(): Promise { rosterAgents = roster.agents; } - let effectiveRegimePath = regimePath; - if (Object.keys(runOverrides).length > 0 || rosterAgents !== undefined) { + // The effective regime is always written now, because the seed is no longer in the regime file and + // has to reach the agent processes too: they read the ERIS_CONFIG YAML directly, so an override + // that only reaches the coordinator leaves the agent observing a different world (ADR 0016 §2). + const regimeDocs = new Map(); + const rosterOf = (doc: RegimeDoc): string[] => { + const agents = (rosterAgents ?? doc.agents) as + Array<{ id?: unknown }> | undefined; + return Array.isArray(agents) + ? agents + .map((a) => a?.id) + .filter((id): id is string => typeof id === "string") + : []; + }; + // Parse (and cache) a regime file. Separated from writing the effective YAML so the pre-flight + // checks below can read a regime without also producing a file that the run loop immediately + // overwrites. + const loadRegimeDoc = (scenario: Scenario): RegimeDoc => { + let doc = regimeDocs.get(scenario.regimePath); + if (!doc) { + doc = parseYaml(readFileSync(scenario.regimePath, "utf8")) as RegimeDoc; + if (doc?.run?.seed !== undefined) + console.error( + `[backtest] note: ${scenario.regimePath} still carries run.seed=${doc.run.seed}; ` + + `the scenario seed overrides it. Remove it (ADR 0017 §1: the seed is not part of the regime)`, + ); + regimeDocs.set(scenario.regimePath, doc); + } + return doc; + }; + const effectivePathFor = (scenario: Scenario): string => { + const doc = loadRegimeDoc(scenario); const effective: RegimeDoc = { - ...regimeDoc, - run: { ...(regimeDoc.run ?? {}), ...runOverrides }, + ...doc, + run: { ...(doc.run ?? {}), ...runOverrides, seed: scenario.seed }, ...(rosterAgents !== undefined ? { agents: rosterAgents } : {}), }; - effectiveRegimePath = join(stateDirAbs, `.effective-${regimeName}.yaml`); + const path = join( + stateDirAbs, + `.effective-${scenario.regime}-${scenario.seed}.yaml`, + ); writeFileSync( - effectiveRegimePath, - `# AUTO-GENERATED by backtest CLI — ${regimePath} + CLI overrides. Do not edit by hand.\n` + + path, + `# AUTO-GENERATED by backtest CLI — ${scenario.regimePath} + seed ${scenario.seed}` + + ` + CLI overrides. Do not edit by hand.\n` + stringifyYaml(effective), ); - const overridden = [ - ...Object.keys(runOverrides), - ...(rosterAgents !== undefined ? ["agents"] : []), - ]; - console.error( - `[backtest] overrides present (${overridden.join(", ")}) -> wrote the effective regime to ${effectiveRegimePath}. ` + - `Runs you read results from should use regime default values (ADR 0016 §3)`, - ); + return path; + }; + + // Whether the venues each regime requires are all present in the state dump. If some are missing, + // an eth_call to a zero address gives a cryptic error, so fail-fast first -- and do it for the + // whole set before anvil starts, not lazily per scenario. Once per regime, not once per scenario: + // the protocol list is a property of the regime, so re-checking it per seed just repeats itself. + const checkedRegimes = new Set(); + for (const scenario of scenarios) { + if (checkedRegimes.has(scenario.regimePath)) continue; + checkedRegimes.add(scenario.regimePath); + const doc = loadRegimeDoc(scenario); + const effectiveProtocols = + (runOverrides.protocols as string[] | undefined) ?? + doc?.run?.protocols ?? + []; + const missing = missingVenues(effectiveProtocols, manifest.deployments); + if (missing.length > 0) + throw new Error( + `regime ${scenario.regime}: state dump is missing venues: ${missing.join(", ")}. ` + + `Either rebuild it from a full deploy with npm run gen:state-dump, or narrow the target ` + + `venues with --protocols (e.g. --protocols ${effectiveProtocols + .filter((p) => !missing.includes(p)) + .join(",")})`, + ); } - // Whether the venues the regime requires are all present in the state dump. If some are missing, - // an eth_call to a zero address gives a cryptic error, so fail-fast first. - const effectiveProtocols = - (runOverrides.protocols as string[] | undefined) ?? - regimeDoc?.run?.protocols ?? - []; - const missing = missingVenues(effectiveProtocols, manifest.deployments); - if (missing.length > 0) - throw new Error( - `state dump is missing venues: ${missing.join(", ")}. Either rebuild it from a full deploy ` + - `with npm run gen:state-dump, or narrow the target venues with --protocols ` + - `(e.g. --protocols ${effectiveProtocols.filter((p) => !missing.includes(p)).join(",")})`, + // A roster that differs between regimes makes the standings incomparable: `total` averages only + // the regimes an agent appeared in, so an agent present in one regime is ranked on a fraction of + // the evidence, against a differently sized z-pool. The competition always passes one roster via + // --agents; the per-regime defaults exist for solo verification and do differ (lending-incident + // carries a liquidator), so say so rather than silently producing a lopsided ranking. + if (matrixMode && rosterAgents === undefined) { + const rosters = new Map(); + for (const scenario of scenarios) + rosters.set( + scenario.regime, + rosterOf(loadRegimeDoc(scenario)).sort().join(","), + ); + if (new Set(rosters.values()).size > 1) + console.error( + `[backtest] warning: the regimes in this set have different default rosters, so agents ` + + `appear in different numbers of regimes and their totals are not comparable ` + + `(${[...rosters].map(([r, ids]) => `${r}:[${ids}]`).join(" ")}). ` + + `Pass --agents to score one field across every regime`, + ); + } + if (Object.keys(runOverrides).length > 0 || rosterAgents !== undefined) + console.error( + `[backtest] overrides present (${[ + ...Object.keys(runOverrides), + ...(rosterAgents !== undefined ? ["agents"] : []), + ].join( + ", ", + )}). Runs you read results from should use regime default values (ADR 0016 §3)`, ); // ---- Backtest-only anvil (--load-state) ---- @@ -276,57 +524,183 @@ async function main(): Promise { rmSync(snapshotFile, { force: true }); process.env.ERIS_LOCAL_DEPLOY = "1"; - // Have both the coordinator and agent processes read the effective regime (the agent via ERIS_CONFIG). - process.env.ERIS_CONFIG = effectiveRegimePath; // Evaluate after the constants sync (dynamically, since static imports are hoisted; same as sim-realtime.ts). const { runRealtimeSimulation } = await import("../realtime/coordinator.js"); - const summaries: RunSummary[] = []; - for (let i = 0; i < repeat; i++) { - console.error( - `[backtest] run ${i + 1}/${repeat} (regime=${regimeName}, seed=${runOverrides.seed ?? regimeDoc?.run?.seed ?? "?"})`, + // ---- The scenario matrix ---- + // One long-lived anvil; every scenario is snapshot -> run -> reconstruct -> revert (ADR 0017 §3). + // Scoring reconstruction finishes inside runRealtimeSimulation, i.e. before the next revert + // erases the history it reads. + const results: ScenarioResult[] = []; + // Kept alongside `results` only for the --repeat spread report below; the folded record is what + // reaches matrix.json. + const repeatsByScenario: AgentScore[][][] = []; + const blocksByScenario: Array> = []; + // Prepared before the loop, and rewritten after every scenario. A full private matrix is ~6 h; + // holding the results in memory until the end means an OOM, an anvil death or a Ctrl-C at + // scenario 29 of 30 throws away everything already computed. + const outDir = matrixMode + ? resolve( + ROOT, + "runs", + `matrix-${new Date().toISOString().replace(/[:.]/g, "-")}`, + ) + : undefined; + if (outDir) mkdirSync(outDir, { recursive: true }); + const flush = (): void => { + if (!outDir) return; + writeFileSync( + join(outDir, "matrix.json"), + `${JSON.stringify( + { + schema: 1, + createdAt: new Date().toISOString(), + sourceCommit: gitHead(ROOT) ?? "unknown", + scenarioSet: flags.scenarios, + metric, + repeat, + // Complete only once every scenario has run; until then this is a partial matrix. + scenariosPlanned: scenarios.length, + // Both metrics are stored regardless of which one ranks, so the standings can be + // recomputed under a different scoring rule without re-running anything (ADR 0017 §4). + scenarios: results, + }, + null, + 2, + )}\n`, ); - const { runDir } = await runRealtimeSimulation({ - ANVIL_RPC_URL: rpcUrl, - // Guarantee config.localDeploy even for an arbitrary regime file (one that forgot to write run.localDeploy). - ERIS_LOCAL_DEPLOY: "1", - ERIS_LOCAL_SNAPSHOT_FILE: snapshotFile, - ERIS_RUN_MODE: "backtest", - }); - const summary = readRunSummary(runDir); - if (summary) summaries.push(summary); - else + writeFileSync( + join(outDir, "standings.json"), + `${JSON.stringify(computeStandings(results, metric), null, 2)}\n`, + ); + }; + let index = 0; + for (const scenario of scenarios) { + index++; + const label = scenarioId(scenario.regime, scenario.seed); + const expectedAgents = rosterOf(loadRegimeDoc(scenario)); + // Have both the coordinator and the agent processes read this scenario's effective regime. + process.env.ERIS_CONFIG = effectivePathFor(scenario); + + const perRepeat: AgentScore[][] = []; + const runDirs: string[] = []; + const blocksPerRepeat: Array = []; + let lastError: string | undefined; + for (let i = 0; i < repeat; i++) { console.error( - `[backtest] warning: summary.json not found for run ${i + 1} (${runDir})`, + `[backtest] scenario ${index}/${scenarios.length} ${label}` + + (repeat > 1 ? ` (repeat ${i + 1}/${repeat})` : ""), ); + try { + const { runDir } = await runRealtimeSimulation({ + ANVIL_RPC_URL: rpcUrl, + // Guarantee config.localDeploy even for an arbitrary regime file (one that forgot to write run.localDeploy). + ERIS_LOCAL_DEPLOY: "1", + ERIS_LOCAL_SNAPSHOT_FILE: snapshotFile, + ERIS_RUN_MODE: "backtest", + }); + runDirs.push(runDir); + const summary = readRunSummary(runDir); + if (summary) { + perRepeat.push(scoresFromSummary(summary, expectedAgents)); + blocksPerRepeat.push(summary.blocksProcessed); + } + else { + lastError = `summary.json not found (${runDir})`; + console.error(`[backtest] warning: ${lastError}`); + } + } catch (error) { + // One scenario blowing up must not abandon the rest of the matrix. It is recorded as an + // excluded scenario, which keeps it out of the aggregation instead of scoring everyone + // zero for an environment failure (ADR 0017 §4). + lastError = error instanceof Error ? error.message : String(error); + console.error(`[backtest] scenario ${label} failed: ${lastError}`); + } + } + repeatsByScenario.push(perRepeat); + blocksByScenario.push(blocksPerRepeat); + results.push({ + regime: scenario.regime, + seed: scenario.seed, + ...(perRepeat.length > 0 + ? { agents: foldRepeats(perRepeat, metric) } + : {}), + ...(runDirs.length > 0 ? { runDir: runDirs[runDirs.length - 1] } : {}), + ...(perRepeat.length === 0 && lastError !== undefined + ? { error: lastError } + : {}), + }); + flush(); } - // ---- Aggregation (for multiple runs also print the per-agent mean. ADR 0005: read as a distribution) ---- + // ---- Report ---- + const standings = computeStandings(results, metric); console.log(""); - console.log(`backtest ${regimeName}: ${summaries.length}/${repeat} runs`); - for (const s of summaries) { - const line = s.agents + // With --repeat, show each run rather than only the fold. The point of repeating a scenario is + // to see how far it moves run to run (ADR 0005 says to read results as a distribution), and a + // single folded line hides both the spread and a run that ended early on fewer blocks. + if (repeat > 1) { + for (const [s, scenario] of scenarios.entries()) { + const runs = repeatsByScenario[s] ?? []; + if (runs.length === 0) continue; + console.log( + ` ${scenarioId(scenario.regime, scenario.seed)} — ${runs.length}/${repeat} runs:`, + ); + for (const [i, run] of runs.entries()) + console.log( + ` run ${i + 1} (${blocksByScenario[s]?.[i] ?? "?"} blocks): ` + + run + .map( + (a) => + `${a.id}=${a[metric]?.toFixed(2) ?? "-"}${a.disqualified ? "(DQ)" : ""}`, + ) + .join(" "), + ); + } + console.log(""); + } + for (const result of results) { + const label = scenarioId(result.regime, result.seed); + if (!result.agents) { + console.log(` ${label}: FAILED (${result.error})`); + continue; + } + const line = result.agents .map( (a) => - `${a.id} α=${a.alphaUsdc?.toFixed(2) ?? "-"} pnl=${a.netPnlUsdc?.toFixed(2) ?? "-"}`, + `${a.id}=${a[metric]?.toFixed(2) ?? "-"}${a.disqualified ? "(DQ)" : ""}`, ) .join(" "); + console.log(` ${label}: ${line}`); + } + + if (matrixMode && outDir) { + // Already written incrementally by flush(); this is the final, complete pass. + flush(); + console.log(""); console.log( - ` ${basename(s.runDir)} (${s.blocksProcessed} blocks): ${line}`, + `standings (${metric}, regime-internal z-score, equal weight per regime):`, ); - } - if (summaries.length > 1) { - const byAgent = new Map(); - for (const s of summaries) - for (const a of s.agents) - if (a.alphaUsdc !== undefined) - byAgent.set(a.id, [...(byAgent.get(a.id) ?? []), a.alphaUsdc]); - console.log(" mean alphaUsdc:"); - for (const [id, values] of byAgent) { - const mean = values.reduce((x, y) => x + y, 0) / values.length; - console.log(` ${id}: ${mean.toFixed(2)} (n=${values.length})`); - } + for (const [rank, agent] of standings.agents.entries()) + console.log( + ` ${String(rank + 1).padStart(2)}. ${agent.id.padEnd(20)} ${agent.total.toFixed(3)}` + + ` [${standings.regimes + .map((r) => `${r}=${agent.byRegime[r]?.toFixed(2) ?? "-"}`) + .join(" ")}]` + + (agent.disqualifications > 0 + ? ` DQ x${agent.disqualifications}` + : ""), + ); + if (standings.excludedScenarios.length > 0) + console.log( + ` excluded ${standings.excludedScenarios.length} scenario(s) that produced no result: ` + + standings.excludedScenarios + .map((s) => scenarioId(s.regime, s.seed)) + .join(", "), + ); + console.log(""); + console.log(`wrote ${outDir}/matrix.json and standings.json`); } } finally { if (keepAnvil) { diff --git a/core/src/coordinator.ts b/core/src/coordinator.ts index e90f572..deb9109 100644 --- a/core/src/coordinator.ts +++ b/core/src/coordinator.ts @@ -115,6 +115,10 @@ export async function buildFlowContext( fairPriceUsdcPerWeth: fairPrice, protocols: enabledIds, poolPrices, + // The persisted uninformed trend derives its direction from this rather than from the shared + // RNG stream (which would shift every downstream draw). Without it the direction would be a + // function of the block window alone -- identical on every seed, and therefore memorizable. + flowSeed: ctx.config.flowSeed, ...(aaveActors ? { aaveActors } : {}), flowBalances, // If flow holds base inventory (flowWethWei>0), allow selling (gated by balance). @@ -128,6 +132,9 @@ export async function buildFlowContext( uninformedFlowPersistBlocks: String( ctx.config.uninformedFlowPersistBlocks, ), + uninformedFlowTrendCorrelation: String( + ctx.config.uninformedFlowTrendCorrelation, + ), informedFlowMaxWethWei: ctx.config.informedFlowMaxWethWei.toString(), balancerFlowMaxWethWei: ctx.config.balancerFlowMaxWethWei.toString(), curveFlowMaxWethWei: ctx.config.curveFlowMaxWethWei.toString(), diff --git a/core/src/flow/logic.ts b/core/src/flow/logic.ts index 01c8e0c..f772495 100644 --- a/core/src/flow/logic.ts +++ b/core/src/flow/logic.ts @@ -28,6 +28,10 @@ export type FlowLimits = { uninformedFlowCountPerBlock: number; // Persistence in blocks of the uninformed direction (default 1). >1 makes per-venue trends produce cross-venue divergence naturally. uninformedFlowPersistBlocks: number; + // Probability [0,1] that a venue's persisted direction follows the market-wide one instead of its + // own (default 0 = independent per venue). Turns the trend from a cross-venue spread into + // one-sided pressure on the whole market (ADR 0017 regime 2). + uninformedFlowTrendCorrelation: number; informedFlowMaxWethWei: bigint; balancerFlowMaxWethWei: bigint; curveFlowMaxWethWei: bigint; @@ -57,6 +61,9 @@ export type FlowLimits = { export type FlowContextWire = { round: number; fairPriceUsdcPerWeth: number; + // Seeds the persisted uninformed trend's direction (see trendBit). Without it the direction is a + // function of the block window alone, i.e. identical across every seed and memorizable. + flowSeed?: number; protocols: ProtocolId[]; poolPrices: Partial>; aaveReserves?: { wethSupplied: string; usdcBorrowed: string }; @@ -89,6 +96,7 @@ export type FlowContextWire = { uninformedFlowMaxWethWei: string; uninformedFlowCountPerBlock?: string; uninformedFlowPersistBlocks?: string; + uninformedFlowTrendCorrelation?: string; informedFlowMaxWethWei: string; balancerFlowMaxWethWei: string; curveFlowMaxWethWei: string; @@ -203,6 +211,35 @@ function capUsdc(amount: bigint, balance: FlowBalance | null): bigint { return amount > balance.usdcUnits ? balance.usdcUnits : amount; } +// Direction bit for the persisted uninformed trend, from (flowSeed, window, tag). +// +// Deterministic and off the shared RNG stream on purpose: the trend must not consume draws, or +// enabling it would shift every downstream order. But it must still depend on the seed, or the +// direction is a pure function of the block window -- the same on every published seed and on every +// unpublished one, which makes the whole regime memorizable. +// +// Two bugs this replaces, both found in review: +// - The mixing step used float `*`, and (2^32 * 2^24) exceeds 2^53, so the product's low bits were +// rounded away and `% 2` was pinned. Measured: "uniswap" and "balancer" returned an even hash in +// *every* window, i.e. a permanent one-way bias rather than a trend. Math.imul keeps 32-bit +// arithmetic exact. +// - Reading the parity of an FNV accumulator is not enough even when the arithmetic is exact: the +// low bit stays tied to the window's parity, so every tag alternated on a perfect 1-block clock. +// Rng is an LCG, and `bool()` reads the high bit, which is the well-distributed end. Measured +// over 200 seeds x 50 windows: p(1)=0.504, flip rate 0.481 (0.5 = no autocorrelation). +function hashTag(tag: string): number { + let h = 0x81_1c_9d_c5; + for (let c = 0; c < tag.length; c++) + h = Math.imul(h ^ tag.charCodeAt(c), 0x01_00_01_93) >>> 0; + return h >>> 0; +} + +function trendBit(flowSeed: number, window: number, tag: string): boolean { + const h = + (Math.imul(flowSeed ^ (window + 1), 0x01_00_01_93) ^ hashTag(tag)) >>> 0; + return new Rng(h).bool(); +} + // AMM (uniswap/balancer/curve) flow. uninformed noise + informed (pull price toward fair). // base defaults to WETH. When base!=="WETH", use that base symbol for tokenIn and attach action.base // so the adapter can resolve the WBTC/USDC market. The WETH path is byte-identical to before (no base @@ -236,6 +273,13 @@ export function buildAmmFlow( // >0 makes the per-block count Poisson(λ) and each size lognormal (mean = uninformedMax×0.5, σ=sizeSigma). uninformedArrivalRate = 0, uninformedSizeSigma = 1, + // ADR 0017 regime 2 (informed-flow): probability in [0,1] that a venue's persisted direction follows + // the market-wide direction instead of its own. 0 (default) = the old per-venue independent trend + // (byte-compatible). Only meaningful with persistBlocks > 1, which is what creates a direction at all. + trendCorrelation = 0, + // Seed the persisted trend so its direction is not a pure function of the block window (see + // trendBit). Comes from FlowContextWire.flowSeed; 0 is only reached by a direct unit-test call. + trendSeed = 0, ): FlowOrder[] { const orders: FlowOrder[] = []; const swapType = @@ -260,11 +304,29 @@ export function buildAmmFlow( let trendTokenIn: TokenSymbol | null = null; if (persistBlocks > 1) { const window = Math.floor(round / persistBlocks); - let h = ((window + 1) * 0x9e3779b1) >>> 0; - for (let c = 0; c < protocol.length; c++) - h = ((h ^ protocol.charCodeAt(c)) * 0x01000193) >>> 0; - // USDC in=buy (price up) / base in=sell (price down). venue×window splits up/down and creates a spread. - trendTokenIn = h % 2 === 0 ? "USDC" : base; + // USDC in = buy (price up) / base in = sell (price down). + const venueUp = trendBit(trendSeed, window, protocol); + if (trendCorrelation > 0) { + // Correlated directional flow (ADR 0017 regime 2). The per-venue bit splits the venues up/down + // to manufacture a cross-venue spread; that is the wrong shape for "the whole market is being + // pushed one way". Drawing from a venue-independent bit makes every venue lean together, so + // the gap that opens is against fair rather than against each other -- and an agent has to + // take a side instead of arbitraging the middle. + // + // The correlation is a probability rather than a switch so the regime can sit anywhere between + // the two: at 0.7 most venues follow the market and the stragglers still leave relative value. + // The follow draw is its own stream, so it neither consumes the shared rng nor correlates with + // the direction it is choosing between. + const follow = + new Rng( + Math.imul(trendSeed ^ (window + 1), 0x27_22_0a_95) ^ + hashTag(`${protocol}|corr`), + ).next() < trendCorrelation; + const up = follow ? trendBit(trendSeed, window, "market") : venueUp; + trendTokenIn = up ? "USDC" : base; + } else { + trendTokenIn = venueUp ? "USDC" : base; + } } // Poisson mode (arrivalRate>0): draw the per-block arrival count as Poisson(λ) (0-count blocks arise naturally). // Legacy mode (arrivalRate=0): fixed count max(1, uninformedCount). RNG consumption is as before (byte-compatible). @@ -661,6 +723,10 @@ export function decodeFlowLimits(wire: FlowContextWire["limits"]): FlowLimits { 1, Number(wire.uninformedFlowPersistBlocks ?? "1"), ), + uninformedFlowTrendCorrelation: clampProb( + wire.uninformedFlowTrendCorrelation, + 0, + ), informedFlowMaxWethWei: BigInt(wire.informedFlowMaxWethWei), balancerFlowMaxWethWei: BigInt(wire.balancerFlowMaxWethWei), curveFlowMaxWethWei: BigInt(wire.curveFlowMaxWethWei), @@ -732,6 +798,8 @@ export function buildFlowOrders( limits.informedArbFeeBps, limits.uninformedArrivalRate, limits.uninformedSizeSigma, + limits.uninformedFlowTrendCorrelation, + ctx.flowSeed ?? 0, ), ); } else if (protocol === "aave") { @@ -839,6 +907,8 @@ export function buildFlowOrders( limits.informedArbFeeBps, limits.uninformedArrivalRate, limits.uninformedSizeSigma, + limits.uninformedFlowTrendCorrelation, + ctx.flowSeed ?? 0, ), ); } diff --git a/core/src/postRunCheck.ts b/core/src/postRunCheck.ts index 3fa6c19..e761982 100644 --- a/core/src/postRunCheck.ts +++ b/core/src/postRunCheck.ts @@ -54,3 +54,36 @@ export function checkRunFeeViolations( if (!existsSync(path)) return []; return checkFeeViolations(readFileSync(path, "utf8"), maxPriorityFeeWei); } + +// Environment-owned transactions that reverted, by owner (ADR 0017 regime 3). +// +// The environment's own shocks must not fail quietly. A whale order is submitted through the same +// relay as ordinary flow, and that path catches submission errors -- but an on-chain revert is not a +// submission error: the tx lands, the event log says the whale fired, and only blocks.csv records +// that it did nothing. That is how a missing token approval turned the whale regime into calm with +// every log looking healthy. +export function countRevertedTxs( + blocksCsv: string, + ownerId: string, +): { total: number; reverted: number } { + const I = BLOCKS_CSV_INDEX; + let total = 0; + let reverted = 0; + for (const line of blocksCsv.split("\n").slice(1)) { + if (line.length === 0) continue; + const cols = line.split(","); + if (cols[I.ownerId] !== ownerId) continue; + total++; + if (cols[I.status] === "reverted") reverted++; + } + return { total, reverted }; +} + +export function countRunRevertedTxs( + runDir: string, + ownerId: string, +): { total: number; reverted: number } { + const path = join(runDir, "blocks.csv"); + if (!existsSync(path)) return { total: 0, reverted: 0 }; + return countRevertedTxs(readFileSync(path, "utf8"), ownerId); +} diff --git a/core/src/realtime/agentProcess.ts b/core/src/realtime/agentProcess.ts index e22f50b..9250c81 100644 --- a/core/src/realtime/agentProcess.ts +++ b/core/src/realtime/agentProcess.ts @@ -70,7 +70,7 @@ export class RealtimeAgentProcess { args = spec.args ?? []; } else { // Convention resolution (ADR 0015 §6): id (or the dir override) points to //, and - // bot.ts drives its contents (agent.ts decide/run, or prompt.md). + // bot.ts drives its contents (agent.ts decide/run, plus improve.md when self-improving). const agentDir = resolve(agentsDir, spec.dir ?? spec.id); if (!existsSync(agentDir)) { throw new Error( diff --git a/core/src/realtime/coordinator.ts b/core/src/realtime/coordinator.ts index ad2ee07..ad0f7fc 100644 --- a/core/src/realtime/coordinator.ts +++ b/core/src/realtime/coordinator.ts @@ -16,13 +16,11 @@ import { } from "@eris/sdk/chain.js"; import { RunLogger } from "../logger.js"; import { valueUsdc } from "@eris/sdk/pnl.js"; -import { checkRunFeeViolations } from "../postRunCheck.js"; import { - nextFairPrice, - ouParamsForSymbol, - priceRngForAsset, - Rng, -} from "@eris/sdk/rng.js"; + checkRunFeeViolations, + countRunRevertedTxs, +} from "../postRunCheck.js"; +import { nextFairPrice, priceRngForAsset, Rng } from "@eris/sdk/rng.js"; import type { AgentObservation, AgentSpec, @@ -75,6 +73,7 @@ import { STARTUP_WARN_BPS, } from "./noArb.js"; import { EventSchedule } from "./events.js"; +import { buildWhaleOrder, whaleFunding, WHALE_WALLET_KEY } from "./whale.js"; import { accrueLst, lstBlockEvent, @@ -130,7 +129,12 @@ async function prewarmWorkingSet( const warmRng = new Rng(ctx.config.seed); let warmPrice = startPrice; for (let i = 1; i <= blocks; i++) { - warmPrice = nextFairPrice(warmPrice, warmRng, startPrice); + warmPrice = nextFairPrice( + warmPrice, + warmRng, + startPrice, + ctx.config.ou.global, + ); await updateOracles(ctx, warmPrice); const states = await Promise.all( adapters.map((adapter) => adapter.readState(ctx, warmPrice)), @@ -178,6 +182,11 @@ type RealtimeAgentRuntime = { initial: BalanceSnapshot; included: number; // number of txs included in a block (read by aggregation) reverted: number; // of those, the number that reverted + // Why the agent process went away before the run ended, if it did. Set from onExit, which only + // fires on an early exit or a spawn failure. Surfaced in summary.json because the alternative is + // grepping events.jsonl, and because scenario-matrix standings treat an agent that died as + // disqualified for that scenario rather than as one that chose to sit still (ADR 0017 §4). + exitedEarly?: string; }; type SubmittedMeta = { @@ -318,6 +327,29 @@ export async function runRealtimeSimulation( } } + // Resolved before wallet funding, not at the start of the block loop: the whale wallet has to be + // endowed with enough inventory to place the largest order the seed drew, and that is only knowable + // from the resolved schedule. The schedule is a pure function of (config, seed, runBlocks) with no + // chain dependency, so building it early costs nothing. + const schedule = new EventSchedule( + config.stressEvents, + config.seed, + config.runBlocks, + ); + // A dedicated wallet so a whale order does not drain the ordinary flow wallets mid-run (which + // would quietly change the flow bot's behavior for the rest of the run) and so blocks.csv + // attributes the print to the event rather than to background flow. + const whaleEvents = schedule.events.filter((e) => e.type === "whale"); + if (whaleEvents.length > 0) { + const key = WHALE_WALLET_KEY; + const privateKey = keccak256(stringToBytes(`flow:${config.seed}:${key}`)); + flowWalletMap.set(key, { + id: `flow-${key}`, + address: accountAddress(privateKey), + privateKey, + }); + } + const adminPk = config.privateKeys.admin; const keeperPk = config.privateKeys.keeper; const rng = new Rng(config.seed); @@ -400,6 +432,10 @@ export async function runRealtimeSimulation( })), ]; for (const t of fundTargets) { + // The whale passes through here too, even though its balances are overwritten a moment later + // once the fair price is known: this loop is also where each adapter's setupWallet grants the + // token approvals. Skipping it to avoid the redundant funding left the whale unapproved, and + // all four of its swaps reverted on-chain while every log said the event had fired. const isFlow = t.key !== undefined; // aave borrower actors are endowed with collateral WETH directly (a USDC→WETH prep swap tends to fail on // slippage, and the actor struggles to secure collateral and never reaches borrowing). The collateral is @@ -455,6 +491,52 @@ export async function runRealtimeSimulation( await writeAaveOraclesStorage(ctx, latestFairPrice); } + // ---- whale endowment (ADR 0017 regime 3) ---- + // Funded here rather than in the loop above because the size is denominated against the fair + // price, which is only known once the pools have been read. A whale's whole job is to place an + // order far larger than ordinary flow, so flow-sized funding would make it fail on balance and + // silently turn the regime into calm for that seed. + if (whaleEvents.length > 0) { + const wallet = flowWalletMap.get(WHALE_WALLET_KEY); + if (!wallet) throw new Error("whale wallet missing from flowWalletMap"); + // Fail fast on a whale pointed at a venue this run does not have. Otherwise submitIntent + // cannot resolve an adapter, handleFlowOrders swallows the throw, and the run continues with + // the event silently missing -- every other calibration coupling here (victim HF, the LST rate + // oracle, the state dump's venues) fails at setup instead. + for (const ev of whaleEvents) { + const venue = ev.venue ?? "uniswap"; + if (!enabledIds.includes(venue as ProtocolId)) + throw new Error( + `whale event targets venue "${venue}", which is not enabled for this run ` + + `(enabled: ${enabledIds.join(", ")}). Enable it in run.protocols or retarget the event`, + ); + } + const fairForFunding: Record = { + WETH: latestFairPrice, + ...(ctx.fairPrices ?? {}), + }; + const funding = whaleFunding(schedule.events, fairForFunding); + await fundWallet( + publicClient, + walletClient, + chain, + wallet.privateKey, + config.flowEthWei, + funding.baseWei.WETH ?? 0n, + funding.usdcUnits, + funding.baseWei, + ); + logger.event({ + type: "stress_whale_funded", + address: wallet.address, + baseWei: Object.fromEntries( + Object.entries(funding.baseWei).map(([k, v]) => [k, v.toString()]), + ), + usdcUnits: funding.usdcUnits.toString(), + events: whaleEvents.length, + }); + } + // ---- stress victims (ADR 0009 §4): build seed-derived victims that make liquidation possible ---- // Victims are not included in agentRuntimes = not scored (a profit source for the liquidator agent). const stressVictims: StressVictim[] = deriveStressVictims( @@ -644,6 +726,7 @@ export async function runRealtimeSimulation( const child = agent.process; if (!child) continue; child.onExit = (info) => { + runtime.exitedEarly = info.reason; logger.event({ type: "agent_process_exited", agentId: runtime.id, @@ -802,11 +885,6 @@ export async function runRealtimeSimulation( extraBaseFair[b] = p0; extraAnchor[b] = p0; } - const schedule = new EventSchedule( - config.stressEvents, - config.seed, - config.runBlocks, - ); let processedBlocks = 0; let processing = false; let lastProcessedBlock = Number(await publicClient.getBlockNumber()); @@ -889,7 +967,15 @@ export async function runRealtimeSimulation( // Advance base by OU only, and apply the (deterministic) stress overlay to derive the effective price. // The effective price propagates consistently to PriceFeed / Aave WETH oracle / GMX / scoring (ADR 0009 §1). const blockIndex = bn - runStartBlock; - baseFair = nextFairPrice(baseFair, rng, fairAnchor); + // perBase.WETH, not global: readOuParams populates an entry for every registered base, so + // reading the global here made `market.baseVolatility: { WETH: ... }` parse, typecheck and + // do nothing. The entry falls back to the global when the regime sets no WETH override. + baseFair = nextFairPrice( + baseFair, + rng, + fairAnchor, + config.ou.perBase.WETH ?? config.ou.global, + ); const overlay = schedule.at(blockIndex); latestFairPrice = baseFair * overlay.wethMult; // ADR 0013: advance extra bases with independent Rngs and distribute the effective prices into ctx.fairPrices. @@ -899,7 +985,7 @@ export async function runRealtimeSimulation( extraBaseFair[b], extraPriceRng[b], extraAnchor[b], - ouParamsForSymbol(b), + config.ou.perBase[b] ?? config.ou.global, ); fairPrices[b] = extraBaseFair[b] * (overlay.baseMults[b] ?? 1); } @@ -928,23 +1014,65 @@ export async function runRealtimeSimulation( } } - // LST slashing (issue #38 phase 2): a one-shot staking penalty placed by the same - // seed-driven schedule as spike/crash. Applied here, before the block's other work, so - // the redemption rate an agent observes this block already reflects it — and so the gap - // it opens against the (not yet repriced) market is the opportunity the event creates. - if (lstRuntime) { - // The caught-up range, not just this index: onBlock skips notifications while it is - // busy, and matching one index exactly let a dropped block swallow the whole event. + // Point stress events (lstSlash, whale): one-shot shocks the coordinator executes, + // placed by the same seed-driven schedule as spike/crash. Applied here, before the block's + // other work, so what an agent observes this block already reflects them — and so the gap + // they open against the (not yet repriced) market is the opportunity the event creates. + // + // The caught-up range, not just this index: onBlock skips notifications while it is busy, + // and matching one index exactly let a dropped block swallow the whole event. + { const fromIndex = Math.max(0, fromBlock - runStartBlock); for (const ev of schedule.pointEventsAt(fromIndex, blockIndex)) { - try { - await slashLst(ctx, lstRuntime, ev.magnitude, logger, oracleFee); - } catch (error) { - logger.event({ - type: "lst_slash_failed", - blockIndex, - error: error instanceof Error ? error.message : String(error), - }); + if (ev.type === "lstSlash") { + if (!lstRuntime) continue; + try { + await slashLst( + ctx, + lstRuntime, + ev.magnitude, + logger, + oracleFee, + ); + } catch (error) { + logger.event({ + type: "lst_slash_failed", + blockIndex, + error: + error instanceof Error ? error.message : String(error), + }); + } + } else if (ev.type === "whale") { + // Relayed through the ordinary flow path so the print is signed, ordered and + // attributed exactly like any other flow order, and competes for the same block + // space. It is not hidden: the whale trades from a dedicated address endowed during + // setup, so an agent watching balances at block 0 can identify the wallet and its + // capacity before any print. That is deliberate — reading the tape is part of the + // regime — but it does mean the event is anticipatable, not just reactable. + try { + const order = buildWhaleOrder( + ev, + fairPrices[ev.base] ?? latestFairPrice, + config.defaultPriorityFeeWei, + ); + logger.event({ + type: "stress_whale", + blockIndex, + blockNumber: bn, + venue: ev.venue, + side: ev.side, + base: ev.base, + magnitude: ev.magnitude, + }); + await handleFlowOrders([order]); + } catch (error) { + logger.event({ + type: "stress_whale_failed", + blockIndex, + error: + error instanceof Error ? error.message : String(error), + }); + } } } } @@ -1294,6 +1422,7 @@ export async function runRealtimeSimulation( priceFeed: priceFeedAddress, fromBlock: runStartBlock, toBlock: finalBlock, + scoreEvery: config.scoreEvery, }); valueSeries = meta; alphaByAgent = meta.alphaByAgent; @@ -1321,6 +1450,30 @@ export async function runRealtimeSimulation( const violations = config.economicGas ? [] : checkRunFeeViolations(logger.runDir, config.maxPriorityFeeWei); + + // The environment's own shocks must not fail quietly. A whale is submitted through the ordinary + // relay, so a *submission* error is caught and logged -- but an on-chain revert is not one: the + // tx lands, the schedule says the whale fired, and only blocks.csv shows it did nothing. A + // missing token approval once turned this regime into calm with every log looking healthy. + if (whaleEvents.length > 0) { + const whaleTxs = countRunRevertedTxs( + logger.runDir, + `flow-${WHALE_WALLET_KEY}`, + ); + if (whaleTxs.reverted > 0) + logger.event({ + type: "stress_whale_reverted", + reverted: whaleTxs.reverted, + total: whaleTxs.total, + note: "whale orders landed but reverted on-chain; this regime degraded toward calm", + }); + console.error( + whaleTxs.reverted > 0 + ? `[stress] WARNING: ${whaleTxs.reverted}/${whaleTxs.total} whale orders reverted on-chain — ` + + `the whale regime did not actually shock this run` + : `[stress] ${whaleTxs.total} whale orders executed`, + ); + } if (config.economicGas) { logger.event({ type: "fee_cap_enforcement_disabled", @@ -1332,11 +1485,22 @@ export async function runRealtimeSimulation( // ---- final PnL ---- const finalFairPrice = latestFairPrice; + // Price every registered base, not just WETH. valueUsdc marks an unlisted base at `p[sym] ?? 0`, + // so passing the scalar WETH price valued an agent's WBTC at exactly zero: anyone holding a + // non-WETH base at the last block had that inventory deleted from netPnlUsdc. Measured on a + // 24-agent calm run, the WBTC-trading agents reported a reproducible -6,686 USDC "loss" while the + // reconstruction (which does price every base since issue #41) put the same agents at +13 alpha. + // ctx.fairPrices is the per-base map the block loop already maintains; the WETH-only fallback is + // for a run that ended before the first block was processed. + const finalFairPrices: Record = + ctx.fairPrices && Object.keys(ctx.fairPrices).length > 0 + ? ctx.fairPrices + : { WETH: finalFairPrice }; const agentsSummary = []; for (const agent of agentRuntimes) { const final = await getBalances(publicClient, agent.address); - const initialValue = valueUsdc(agent.initial, finalFairPrice); - let finalValue = valueUsdc(final, finalFairPrice); + const initialValue = valueUsdc(agent.initial, finalFairPrices); + let finalValue = valueUsdc(final, finalFairPrices); const protocolValues: Record = {}; for (const adapter of adapters) { const v = await adapter.valueUsdc( @@ -1367,6 +1531,11 @@ export async function runRealtimeSimulation( ...(agent.id in liquidatableValueByAgent ? { liquidatableValueUsdc: liquidatableValueByAgent[agent.id] } : {}), + // Present only when the agent process went away before the run ended (ADR 0017 §4 reads this + // to disqualify the agent for that scenario instead of scoring its frozen position). + ...(agent.exitedEarly !== undefined + ? { processExitedEarly: agent.exitedEarly } + : {}), // submission count's primary source is the agent's self-reported log (agents/.jsonl) (ADR 0006 §5) includedTxCount: agent.included, revertCount: agent.reverted, diff --git a/core/src/realtime/events.ts b/core/src/realtime/events.ts index 1863234..4101145 100644 --- a/core/src/realtime/events.ts +++ b/core/src/realtime/events.ts @@ -18,12 +18,25 @@ import { Rng } from "@eris/sdk/rng.js"; import type { TokenSymbol } from "@eris/sdk/types.js"; -// spike / crash distort a base's price through the overlay. lstSlash is different in kind: it is a -// one-shot staking penalty applied to the LST vault (issue #38 phase 2), so it carries no price -// multiplier and no trapezoid -- it happens on a single block and the exchange rate is permanently -// lower afterwards. It shares this config section because it is the same thing from a run's point -// of view: a seed-placed shock the agents have to survive. -export type StressEventType = "spike" | "crash" | "lstSlash"; +// spike / crash distort a base's price through the overlay. lstSlash and whale are different in +// kind: they happen on a single block and the coordinator executes them, rather than being a +// multiplier layered on the price path. +// lstSlash a one-shot staking penalty applied to the LST vault (issue #38 phase 2); the exchange +// rate is permanently lower afterwards +// whale a single large market order that moves the mid (ADR 0017 regime 3). Unlike crash, the +// fair price does not move at all -- the pool is knocked away from an unchanged fair, +// which is the opposite direction of dislocation and a different trade to find +// They share this config section because from a run's point of view they are the same thing: a +// seed-placed shock the agents have to survive. +export type StressEventType = "spike" | "crash" | "lstSlash" | "whale"; + +// Types that happen on one block and are executed, rather than layered onto the price as a +// multiplier. `at()` ignores them; `pointEventsAt()` returns them. +const POINT_EVENT_TYPES = new Set(["lstSlash", "whale"]); + +// Which way a whale trades. "random" (the default) lets the seed pick, which is what keeps the +// direction from being memorizable across a published regime's seeds. +export type WhaleSide = "buy" | "sell" | "random"; // Event spec given via env (ERIS_STRESS_EVENTS). Ranges are given, not values. export type StressEventConfig = { @@ -32,7 +45,15 @@ export type StressEventConfig = { base?: TokenSymbol; // Deviation width of the price multiplier. spike acts as +, crash as −. The seed picks from [min,max]. // For lstSlash this is the fraction of the staking pool burnt (0.02 = a 2% slash). + // For whale this is the order size in whole base units (30 = a 30 WETH market order). Absolute + // rather than a fraction because what matters is the size against pool depth, and depth is a + // property of the deployed venue, not of this config. magnitudeRange: [number, number]; + // whale only: which way it trades. Default "random" = the seed decides. + side?: WhaleSide; + // whale only: the venue it hits. Default "uniswap" (the deepest pool, so the size has to be real + // to move it). + venue?: "uniswap" | "balancer" | "curve"; // Fraction of run length for the event start position [min,max]. The seed picks. windowFrac: [number, number]; // Length of each trapezoid segment (block count; fixed). Not applicable to lstSlash, which is @@ -50,6 +71,9 @@ export type ResolvedStressEvent = { type: StressEventType; base: string; // target base (default WETH) magnitude: number; + // whale only: resolved from config.side, with "random" collapsed to a concrete side by the seed. + side?: "buy" | "sell"; + venue?: "uniswap" | "balancer" | "curve"; startBlock: number; rampBlocks: number; holdBlocks: number; @@ -105,20 +129,33 @@ export class EventSchedule { rng.next(), ); const startFrac = lerp(c.windowFrac[0], c.windowFrac[1], rng.next()); - const span = - c.type === "lstSlash" - ? POINT_EVENT_SPAN - : c.rampBlocks + c.holdBlocks + c.decayBlocks; + const span = POINT_EVENT_TYPES.has(c.type) + ? POINT_EVENT_SPAN + : c.rampBlocks + c.holdBlocks + c.decayBlocks; // Clamp startBlock so the window fits inside the run window (scoring history depth; event window ⊂ run window). const maxStart = Math.max(0, runBlocks - span); const startBlock = Math.max( 0, Math.min(Math.round(startFrac * runBlocks), maxStart), ); + // Draw the side for every whale regardless of config so the RNG consumption stays a pure + // function of the event list -- making it conditional would let one event's `side: buy` shift + // the schedule of every event after it. + const sideDraw = c.type === "whale" ? rng.next() : undefined; + const side = + c.type === "whale" + ? c.side === undefined || c.side === "random" + ? sideDraw! < 0.5 + ? ("buy" as const) + : ("sell" as const) + : c.side + : undefined; return { type: c.type, base: c.base ?? "WETH", magnitude, + ...(side !== undefined ? { side } : {}), + ...(c.type === "whale" ? { venue: c.venue ?? "uniswap" } : {}), startBlock, rampBlocks: c.rampBlocks, holdBlocks: c.holdBlocks, @@ -153,7 +190,7 @@ export class EventSchedule { pointEventsAt(fromIndex: number, toIndex = fromIndex): ResolvedStressEvent[] { return this.events.filter( (ev) => - ev.type === "lstSlash" && + POINT_EVENT_TYPES.has(ev.type) && ev.startBlock >= fromIndex && ev.startBlock <= toIndex, ); @@ -162,7 +199,7 @@ export class EventSchedule { at(blockIndex: number): OverlayState { const baseMults: Record = {}; for (const ev of this.events) { - if (ev.type === "lstSlash") continue; // not a price distortion + if (POINT_EVENT_TYPES.has(ev.type)) continue; // executed, not a price distortion const e = envelope(ev, blockIndex); if (e === 0) continue; const sign = ev.type === "crash" ? -1 : 1; @@ -204,8 +241,27 @@ function parseOne(raw: unknown, i: number): StressEventConfig { throw new Error(`${label} must be an object`); } const o = raw as Record; - if (o.type !== "spike" && o.type !== "crash" && o.type !== "lstSlash") { - throw new Error(`${label}.type must be "spike", "crash" or "lstSlash"`); + if ( + o.type !== "spike" && + o.type !== "crash" && + o.type !== "lstSlash" && + o.type !== "whale" + ) { + throw new Error( + `${label}.type must be "spike", "crash", "lstSlash" or "whale"`, + ); + } + if (o.side !== undefined) { + if (o.type !== "whale") + throw new Error(`${label}.side only applies to type "whale"`); + if (o.side !== "buy" && o.side !== "sell" && o.side !== "random") + throw new Error(`${label}.side must be "buy", "sell" or "random"`); + } + if (o.venue !== undefined) { + if (o.type !== "whale") + throw new Error(`${label}.venue only applies to type "whale"`); + if (o.venue !== "uniswap" && o.venue !== "balancer" && o.venue !== "curve") + throw new Error(`${label}.venue must be "uniswap", "balancer" or "curve"`); } if (o.base !== undefined && typeof o.base !== "string") { throw new Error(`${label}.base must be a token symbol string`); @@ -227,8 +283,8 @@ function parseOne(raw: unknown, i: number): StressEventConfig { min: 0, max: 1, }); - // A slash lands on one block, so the trapezoid fields do not apply and are optional there. - const isPoint = o.type === "lstSlash"; + // A point event lands on one block, so the trapezoid fields do not apply and are optional there. + const isPoint = POINT_EVENT_TYPES.has(o.type); const rampBlocks = parseNonNegInt( isPoint ? (o.rampBlocks ?? 0) : o.rampBlocks, `${label}.rampBlocks`, @@ -249,6 +305,10 @@ function parseOne(raw: unknown, i: number): StressEventConfig { return { type: o.type, base: typeof o.base === "string" ? o.base : undefined, + ...(o.side !== undefined ? { side: o.side as WhaleSide } : {}), + ...(o.venue !== undefined + ? { venue: o.venue as "uniswap" | "balancer" | "curve" } + : {}), magnitudeRange, windowFrac, rampBlocks, diff --git a/core/src/realtime/reconstruct.ts b/core/src/realtime/reconstruct.ts index 006bb5c..afc4b85 100644 --- a/core/src/realtime/reconstruct.ts +++ b/core/src/realtime/reconstruct.ts @@ -43,10 +43,16 @@ export type ReconstructionAgent = { id: string; address: Address }; export type ReconstructionMeta = { source: "post-run-reconstruction"; - granularityBlocks: 1; + // Block stride between value cross-sections (config.scoreEvery). 1 = every block. Anything larger + // means the series in events.jsonl is thinned, and a reader reconstructing a per-block return or + // drawdown series from `blocks` alone would mis-scale by this factor. + granularityBlocks: number; fromBlock: number; toBlock: number; + // Number of cross-sections actually read, not the width of the window. With granularityBlocks > 1 + // the two differ; windowBlocks keeps the width available. blocks: number; + windowBlocks: number; failedReads: number; // Which contract/function the failed reads were, so a value that dropped can be traced back to the // read that hid it. The bare counter above said something went wrong but never what (issue #44). @@ -556,6 +562,24 @@ type MulticallFn = ( blockNumber: bigint, ) => Promise; +// Blocks to read a value cross-section at. `every` > 1 thins the series to cut reconstruction cost +// when replaying a scenario matrix (ADR 0017 §3). +// +// fromBlock and toBlock are always included, which is what keeps thinning score-neutral: +// alphaByAgent is alphaLast - alphaFirst, so only those two cross-sections reach summary.json. +// Everything dropped in between is equity-curve resolution in events.jsonl, nothing else. +export function scoringBlocks( + fromBlock: number, + toBlock: number, + every: number, +): number[] { + const step = Math.max(1, Math.floor(every)); + const blocks: number[] = []; + for (let b = fromBlock; b < toBlock; b += step) blocks.push(b); + blocks.push(toBlock); + return blocks; +} + export async function reconstructValueSeries(opts: { publicClient: PublicClient; logger: RunLogger; @@ -565,6 +589,8 @@ export async function reconstructValueSeries(opts: { priceFeed: Address; fromBlock: number; toBlock: number; + // Read a cross-section only every Nth block (config.scoreEvery). Score-neutral; see scoringBlocks. + scoreEvery?: number; }): Promise { const { publicClient, @@ -575,6 +601,7 @@ export async function reconstructValueSeries(opts: { priceFeed, fromBlock, toBlock, + scoreEvery = 1, } = opts; const started = Date.now(); let failedReads = 0; @@ -624,7 +651,15 @@ export async function reconstructValueSeries(opts: { // at another, and collapsing those into one entry would hide half the story. const unpricedKey = (h: UnpricedHolding) => `${h.agentId}|${h.source}|${h.token?.toLowerCase() ?? ""}|${h.reason ?? "unpriced"}`; - for (let b = fromBlock; b <= toBlock; b++) { + const blocks = scoringBlocks(fromBlock, toBlock, scoreEvery); + if (scoreEvery > 1) + logger.event({ + type: "scoring_thinned", + scoreEvery, + crossSections: blocks.length, + windowBlocks: toBlock - fromBlock + 1, + }); + for (const b of blocks) { const snapshot = await readValueSnapshotAtBlock({ publicClient, agents, @@ -732,10 +767,11 @@ export async function reconstructValueSeries(opts: { return { source: "post-run-reconstruction", - granularityBlocks: 1, + granularityBlocks: scoreEvery, fromBlock, toBlock, - blocks: toBlock - fromBlock + 1, + blocks: blocks.length, + windowBlocks: toBlock - fromBlock + 1, failedReads, failedReadTargets: [...failedReadTargets.values()], elapsedMs: Date.now() - started, diff --git a/core/src/realtime/whale.ts b/core/src/realtime/whale.ts new file mode 100644 index 0000000..41879fe --- /dev/null +++ b/core/src/realtime/whale.ts @@ -0,0 +1,123 @@ +// Whale order stress event (ADR 0017 regime 3). +// +// A single large market order that knocks the pool mid away from an unchanged fair price. That is +// the opposite dislocation from `crash`, where fair moves and the pools lag: here fair is where it +// always was and the *pool* is wrong, so the trade to find is the other side of the print rather +// than a directional call. It is also the one event whose entire content is market impact, which +// makes it the direct test of whether an agent sizes against depth. +// +// Executed by the coordinator on one block (it is a trade, not a multiplier on the price path), and +// placed by the same seed-driven schedule as every other stress event (ADR 0009). +import { parseUnits } from "viem"; +import type { ResolvedStressEvent } from "./events.js"; +import type { FlowOrderWire } from "../flowProcess.js"; +import { tokenInfo } from "@eris/sdk/markets.js"; + +// The flow-wallet key the whale trades from. Registered in flowWalletMap only when the schedule +// actually contains a whale, so ordinary runs are unaffected. +export const WHALE_WALLET_KEY = "whale:uninformed"; + +// Multiplier over the *cumulative* same-side notional. Swaps quote at the pool price rather than at +// fair, and a buy has to cover slippage on the way in, so funding exactly the notional would make a +// whale fail on balance -- silently turning the regime into `calm` for the rest of that seed. +// +// Cumulative, not the largest single order: sizing on the max looks sufficient only because buys and +// sells replenish each other, and a seed that draws every whale on the same side (p ~ 1/8 for the +// four in config/regimes/whale.yaml) spends more than any single order. That is exactly the tail the +// headroom is supposed to cover. +const WHALE_FUNDING_HEADROOM = 2n; + +const SWAP_TYPE = { + uniswap: "swap", + balancer: "balancerSwap", + curve: "curveSwap", +} as const; + +// Inventory the whale wallet needs so every scheduled order can actually be placed. +// +// `sell` spends the base, `buy` spends USDC. Both sides are funded for their own cumulative total +// rather than netted, because the schedule's order matters: three sells followed by a buy needs the +// full three sells' worth of base up front, no matter what the buy would have replenished later. +// +// Per base, because an event may target one (`{ type: whale, base: WBTC }`), and a WBTC whale funded +// in WETH is a whale that silently never happens. `prices` is the coordinator's per-base fair map. +export function whaleFunding( + events: ResolvedStressEvent[], + prices: Record, +): { baseWei: Record; usdcUnits: bigint } { + const baseWei: Record = {}; + let usdcTotal = 0; + for (const event of events) { + if (event.type !== "whale" || event.magnitude <= 0) continue; + const base = event.base; + const price = prices[base]; + if (!price || !Number.isFinite(price)) + throw new Error( + `whale event targets ${base} but no fair price is available for it ` + + `(known: ${Object.keys(prices).join(", ") || "none"})`, + ); + if (event.side === "buy") { + usdcTotal += event.magnitude * price; + } else { + const decimals = tokenInfo(base).decimals; + baseWei[base] = + (baseWei[base] ?? 0n) + + parseUnits(event.magnitude.toFixed(decimals), decimals); + } + } + for (const base of Object.keys(baseWei)) + baseWei[base] *= WHALE_FUNDING_HEADROOM; + const usdcDecimals = tokenInfo("USDC").decimals; + return { + baseWei, + usdcUnits: + parseUnits(usdcTotal.toFixed(usdcDecimals), usdcDecimals) * + WHALE_FUNDING_HEADROOM, + }; +} + +// The order a whale event places. Pure: the caller submits it through the ordinary flow relay, so +// the print goes through the same signing, ordering and attribution path as any other flow order. +// +// A buy spends USDC (price up), a sell spends the base (price down) -- the same convention the flow +// bot uses, so an agent reading the tape cannot tell a whale from ordinary flow by its shape. Only +// its size gives it away, which is the point. +export function buildWhaleOrder( + event: ResolvedStressEvent, + fairPriceUsdcPerBase: number, + priorityFeeWei: bigint, +): FlowOrderWire { + if (event.type !== "whale") + throw new Error(`buildWhaleOrder called with a ${event.type} event`); + const venue = event.venue ?? "uniswap"; + const base = event.base; + const baseDecimals = tokenInfo(base).decimals; + const usdcDecimals = tokenInfo("USDC").decimals; + const side = event.side ?? "sell"; + + const amount = + side === "sell" + ? parseUnits(event.magnitude.toFixed(baseDecimals), baseDecimals) + : parseUnits( + (event.magnitude * fairPriceUsdcPerBase).toFixed(usdcDecimals), + usdcDecimals, + ); + + return { + protocol: venue, + kind: "uninformed", + walletKey: WHALE_WALLET_KEY, + priorityFeeWei: priorityFeeWei.toString(), + action: { + type: SWAP_TYPE[venue], + tokenIn: side === "buy" ? "USDC" : base, + amountIn: amount.toString(), + // A whale takes whatever the book gives: the whole content of this event is the impact, so + // capping slippage would cap the event itself. minAmountOut 0 is deliberate. + minAmountOut: "0", + // Non-WETH bases need the market tag so the adapter can resolve the right pool; WETH omits it + // to keep the action byte-identical to ordinary WETH flow. + ...(base === "WETH" ? {} : { base }), + } as unknown as FlowOrderWire["action"], + }; +} diff --git a/core/src/runConfig.ts b/core/src/runConfig.ts index c4acb26..98d8d82 100644 --- a/core/src/runConfig.ts +++ b/core/src/runConfig.ts @@ -122,6 +122,12 @@ const RETIRED_CONFIG_ENV = [ "ROUNDS", "GATE_MODE", "INITIAL_WETH_WEI", + // The fair-price OU parameters moved to the YAML `market.*` section (ADR 0017 regime 1). They used + // to be read straight from process.env by sdk/src/rng.ts, and the coordinator now reads config.ou + // instead -- so leaving these set is a silent no-op rather than the calibration the author intended. + "ERIS_PRICE_VOLATILITY", + "ERIS_PRICE_REVERT_KAPPA", + "ERIS_PRICE_DRIFT", // relay mode has been removed (ADR 0015 §5). Setting it does not roll anything back. "ERIS_AGENT_DIRECT_TX", ] as const; @@ -147,6 +153,7 @@ const CLI_ALIAS: Record = { agents: "AGENTS_CONFIG", "economic-gas": "ERIS_ECONOMIC_GAS", "local-deploy": "ERIS_LOCAL_DEPLOY", + "score-every": "ERIS_SCORE_EVERY", }; function cliOverrides(argv: string[]): Record { const flags = parseCliFlags(argv); diff --git a/docs/adr/0015-core-example-split-and-unified-agent-runtime.md b/docs/adr/0015-core-example-split-and-unified-agent-runtime.md index 66476de..baccf8b 100644 --- a/docs/adr/0015-core-example-split-and-unified-agent-runtime.md +++ b/docs/adr/0015-core-example-split-and-unified-agent-runtime.md @@ -95,7 +95,7 @@ eris-competition-poc/ # npm workspaces ルート │ │ ├─ llm.ts # 素の LLM 呼び出し 1 関数（プロバイダ切替のみ） │ │ └─ agentLog.ts │ ├─ arb-bot/agent.ts # ルール戦略 -│ └─ my-arb/prompt.md # プロンプト型 agent +│ └─ my-arb/{agent.ts,improve.md} # 参加者向け出発点サンプル（ADR 0018 で自己改善型に） └─ deployer/ # venue デプロイは環境側（現状のまま） ``` @@ -112,6 +112,15 @@ venue デプロイ（環境の仕事 = `deployer/`）と参加者コントラク | `agent.ts`（`run(ctx)` export） | 自走型 | bot.ts はループせず ctx（clients/read/send/log）を渡して委譲 | | `prompt.md` | プロンプト型 | bot.ts が observation を添えて LLM に action を出させる | +> **改訂（ADR 0018）: プロンプト型は廃止され、自己改善型に置き換わった。** +> 実測で prompt 型は 1 判断あたり 8〜28 ブロックを要し、同一戦略のルール型に対して行動回数が +> **1/64** だった（ADR 0017 §5 B1）。LLM は取引経路から外し、`agent.ts` + `improve.md` で +> **戦略コードを定期的に書き換える**役に移した。表の 3 行目は次に読み替える: +> +> | 中身 | 種別 | 動き方 | +> |------|------|--------| +> | `agent.ts` + `improve.md` | 自己改善型 | decide を毎ブロック駆動しつつ、LLM が `node:vm` 上の executor を書き換える（`ERIS_AGENT_FROZEN=1` で改訂ループを止められる） | + `runtime/` は予約名であり agent ではない。`params.json` のような構造化パラメータファイルは設けない （戦略パラメータはコードまたはプロンプト本文が持つ）。 diff --git a/docs/adr/0016-participant-backtest-local-anvil-replay.md b/docs/adr/0016-participant-backtest-local-anvil-replay.md index d2e063b..a1b40cb 100644 --- a/docs/adr/0016-participant-backtest-local-anvil-replay.md +++ b/docs/adr/0016-participant-backtest-local-anvil-replay.md @@ -141,6 +141,18 @@ decide 型・prompt 型）の判断頻度や observe / LLM レイテンシとブ クラッシュしない・validate を通る等の挙動確認・スモーク専用とし、**成績を読むのは regime 既定値の run のみ**とする。 +> **改訂（ADR 0017）: seed はこの規約の対象外になった。** +> 本 ADR は seed を regime YAML の一部とみなし、`--seed` を含む全 override を「スモーク専用」として +> いた。ADR 0017 が評価単位を **シナリオ =（regime, seed）** と定めたことで、seed は regime の +> 一部ではなく評価の第 2 軸になり、`config/regimes/*.yaml` からは削除された。したがって: +> +> - **`--seed` は必須の入力であって override ではない。**seed を与えた run の成績は正規のものとして +> 読む。むしろ seed を与えない run はもう存在しない（`--regime` は `--seed` なしで fail-fast する） +> - **`blockTimeSec` / `blocks` / `protocols` 等、それ以外の override は本節の規定のまま**である。 +> 成績を読む run はレジーム既定値を使う +> - 「公開 regime の seed は 1 サンプルで本番は別サンプル」という §2 の過学習対策は、 +> ADR 0017 §2 の Public / Private シードセットへ発展的に置き換わった + **B2: 同期ステップ再生（回帰テスト・paired 比較用）** — 壁時計から切り離したターン制で進める: 毎ブロック、flow 注文の投入後に **全 agent の「このブロックの判断完了」通知（+ tx 到着）が揃った 時点で mine** し、次ブロックへ進む。noop を選んだブロックは tx が来ず沈黙とタイムアウトで diff --git a/docs/adr/0017-scenario-based-evaluation.md b/docs/adr/0017-scenario-based-evaluation.md new file mode 100644 index 0000000..933de4c --- /dev/null +++ b/docs/adr/0017-scenario-based-evaluation.md @@ -0,0 +1,618 @@ +# ADR 0017: シナリオベース評価（レジーム × シードのシナリオ行列と順位付け） + +## Status + +Accepted（2026-08-09 実装。Phase 0–2 の大半。branch feat/lst-venue） + +実装済み: シナリオ =（regime, seed）への分離、シナリオ行列 runner、`matrix.json` / `standings.json`、 +`--score-every`、レジーム 6 本（`calm` / `cex-drift` / `informed-flow` / `whale` / +`lending-incident` / `crash`）。 + +未了（§7 のフェーズ順）: + +- **`crash` の流動性引き抜き**（issue #52）— 価格ギャップだけで、レジーム 6 の半分が欠けている +- **`depeg`**（Phase 3）— 採点方法の見直し待ち +- **§5 の較正値**— パイロットは R=20〜40 の短縮 run で、本番 R=360 での実測は未了。 + 特に `blockTimeSec` と LLM 型の判断頻度、`whale` の magnitude、`informed-flow` の較正値は + 根拠が未確定のまま + +採点値（`netPnlUsdc`）と集約式は Decision として確定しているが、**採点方法自体を将来見直す前提**で +選んでいる（§4）。その差し替えは `matrix.json` の生スコアから再計算でき、本 ADR の実行基盤を +変更しない。 + +## Context + +本 repo は Anvil 上で DeFi トレード競争をシミュレートする環境であり、run は `sim:realtime` 一本、 +参加者向けの反復実行手段として ADR 0016 のバックテスト（配布 state dump をロードしたローカル anvil で +`config/regimes/*.yaml` + seed を再生する B1 実時間再生）が Phase 0 まで実装済みである。 +市場条件のラベル付けは ADR 0005 の「regime = config + seed」、イベントの与え方は ADR 0009 の +「値でなくレンジ、実現値は seed が引く」で確立している。 + +一方、**「コンペの成績をどう決めるか」は未定義のまま**である: + +- ADR 0016 の「決めていないこと」は「backtest を提出ゲートに正式採用するか（合否基準含む）」を + コンペルールの問題として明示的に先送りしている +- 旧 `evaluate` / `gate` / `leaderboard` / `discrimination` ツールは撤去済みで、現行の集計は + backtest CLI が `--repeat` で出す per-agent の mean `alphaUsdc` のみ +- 公式 regime は `calm-01` / `crash-01` / `lst-01` の 3 本しかなく、いずれも `run.seed` を + YAML 内に埋め込んでいる。ADR 0016 §3 は「`--seed` 等の override を掛けた run の成績は読むな + （regime 既定値の run のみ読め）」と規定しており、**seed を評価軸として振ることが規約上できない** + +評価したい市場条件は ASCON の Market shocks 6 種 + 平常時に相当する 7 レジームであり、 +既存実装との対応は次のとおりである: + +| # | レジーム | 現状 | 不足 | +|---|---------|------|------| +| 0 | Calm（平常） | `config/regimes/calm.yaml` | なし | +| 1 | CEX drift（参照価格とオンチェーン価格の乖離ドリフト） | OU の `volatility` / `kappa` / `drift` は `sdk/src/rng.ts` にあるが `process.env` 直読みで **`runConfig.ts` の `SCHEMA` に未登録** | `market.*` セクションの追加（小） | +| 2 | Informed flow（相関のある方向性フロー注入） | informed flow bot + `flow.uninformedPersistBlocks`（方向の持続） | 方向性バーストの強度ノブ露出（小） | +| 3 | Whale order（mid を動かす単発大口） | なし（サイズは Poisson/lognormal のみ） | 点イベント型の stress event 追加（小〜中） | +| 4 | Lending incident（担保価値急落 + 清算ウェーブ） | `lending-incident.yaml`（crash + victim 群 + 清算） | victim 数の拡大のみ | +| 5 | Stablecoin depeg | 未実装。`OverlayState.usdcPx` にフックのみ残置（"v1 is always 1"）。issue #39 が該当 | 建値通貨が動く = 採点 numeraire に直結（大） | +| 6 | Crash event（流動性引き抜き + 価格ギャップ） | 価格ギャップは `crash.yaml`（victim 無し）で充足 | LP 撤退イベント（issue #52。中） | + +### 解決したい課題 + +- **シナリオを第一級の単位にする**: 評価単位を (レジーム r, シード s) の組とし、regime YAML への + seed 埋め込みと「override run の成績は読むな」規約を、評価用途に限って解除する +- **順位付けに耐える再現性**: B1 実時間再生は tx タイミング/着順が非決定であり（ADR 0005）、 + 1 シナリオ 1 実行で順位を決めると run ノイズがシナリオ間分散に交絡する +- **参加者間の公平な比較**: 過去の検証で、unpaired 比較は regime 間分散に埋もれて実力差を検出できず、 + 同一条件に揃えた paired 比較で初めて有意差が出ることが実証されている +- **レジーム横断の集約**: 採点値の絶対値はレジームごとにスケールが桁で異なる（crash は機会が大きい）。 + 生の総和では特定レジームが順位を支配する +- **過学習対策**: Public セットへのチューニングが Private セットに転移することを保証する +- **運用形態**: 常設の devnet を 1 つ立てたまま、シナリオを 1 日に何度も回せる形にする。 + コンペ本体の運用期間は **1 週間**で、提出物はその開始時点で凍結する + +### 検討した選択肢 + +**軸 1: 参加者の配置** + +| 観点 | A. 独立ワールド（参加者ごとに同一 (r,s) を再生） | B. 同居（1 シナリオに全参加者） | +|------|---------------------------------------------|------------------------------| +| 測れるもの | 市場条件への適応力 | それ + 他参加者との競合・ブロック内順序・gas 入札（ADR 0011） | +| 順位の安定性 | 同居メンバーに依存しない | 同居メンバー依存 | +| コスト | シナリオ数 × 参加者数 | シナリオ数のみ | +| 運用リスク | 低い | 単一 anvil への agent 密集で observe が殺到し破綻した前例あり | +| paired 性 | 成立（同一の市場条件。ただし閉ループなので実現パスは分岐） | 成立（文字通り同一 run） | + +**軸 2: レジーム横断の集約** + +| 観点 | A. レジーム内 z-score → レジーム等重み平均 | B. シナリオ内順位 → 平均順位 / Borda | C. 生スコアの総和 | +|------|------------------------------------------|-----------------------------------|------------------------| +| スケール差の吸収 | する | する（完全に） | しない（crash が支配） | +| 外れ値耐性 | 中（1 シナリオの大勝が効く） | 高い | 低い | +| 「どれだけ勝ったか」の保存 | する | 落ちる（1 位は 1 位） | する | + +例: crash で A=+800 / B=+200 / C=−100、calm で A=+10 / B=+40 / C=+25 のとき、 +C 案は A>B>C（crash がそのまま順位）、A 案・B 案はいずれも B>A>C（全レジームでの安定を評価）となる。 + +**軸 3: 採点値** + +| 観点 | A. `netPnlUsdc`（USDC 建て総損益） | B. `alphaUsdc`（β 除去済み） | C. ETH 建てへの移行 | +|------|----------------------------------|---------------------------|-------------------| +| 実装 | ゼロ（両方とも既に `summary.json` に出ている） | ゼロ | 全面改修 + 既存 regime の再較正 | +| 意味 | 価格ドリフト（β）込みの総損益 | 現物在庫を run 内固定の参照 fair で評価し、β を相殺した残り = 取った edge | DeFi ネイティブな建値 | +| β の混入 | する。crash レジームでは現物を持たない `noop` が構造的に上位に来る | 現物在庫のぶんは除去される。ただし**プロトコル内ポジション（Aave 担保 / GMX / LST）は α 側でも live mark** なので残存 β がある（`reconstruct.ts:416`） | LST / WETH 戦略に有利 | +| depeg レジームとの整合 | USDC 自体が動くと建値が壊れる | 同左 | 整合する | + +いずれも既に計算・出力されているため、選択は「どちらで順位を付けるか」だけの問題であり、 +後から差し替えられる（§4 の `matrix.json` に両方を残す）。 + +**軸 4: ブロック（市場時間）の進め方** + +| 観点 | A. B1 実時間（現行 2 秒間隔で mine） | B. B1 のままブロック間隔を伸ばす | C. B2 同期ステップ（ADR 0016 Phase 1） | +|------|-----------------------------------|------------------------------|--------------------------------------| +| 実装 | ゼロ | ゼロ（`run.blockTimeSec` のみ） | coordinator の mine トリガー + bot.ts の判断完了通知 | +| 思考の速さ | 競技力に含まれる（LLM 型は 5 ブロックに 1 回しか動けない） | ほぼ消える（全員が毎ブロック 1 判断できる） | 完全に消える | +| ブロック内順序 | 到着順 + gas 入札（`--order fees`） | ほぼ gas 入札のみ | gas 入札のみ | +| 再現性 | 統計的 | 統計的（到着差は残る） | ルール型は実質ビット再現。LLM 型は非決定のまま | +| 1 run の壁時計 | R × 2 秒 | R × 伸ばした秒数 | 最速だが LLM 型がいると LLM レイテンシ律速 | + +## Decision + +**評価単位をシナリオ = (レジーム r, シード s) とし、Public 7×5=35 / Private 7×20=140 のシナリオ行列を、 +常設 devnet 1 つの上で snapshot/revert により 1 週間かけて再生する。1 シナリオには全参加者を同居させ、 +レジーム内で z-score 正規化してから等重み平均した値で順位を決める。採点値は当面 `netPnlUsdc` を +用いる。seed は regime の一部から評価の第 2 軸へ昇格させ、ADR 0016 §3 の override 規約を評価用途に +限り解除する。** + +軸 1 は B（同居）、軸 2 は A（z-score）、軸 3 は A（`netPnlUsdc`。採点方法自体を将来的に見直す +予定があるため、既存のまま最も素直な指標で始める）、軸 4 は A（現行 2 秒を既定とし、 +較正値としてパイロットで見直す）を採用する。 + +### 1. シナリオの定義とアドレス + +シナリオ ID を `#`（例 `crash#7391`）とし、これが評価・ログ・再現の一次キーになる。 + +- `config/regimes/*.yaml` から **`run.seed` を外す**。regime YAML は「市場条件のパラメータ族」だけを持ち、 + seed は実行時に与える。既存 3 本の `seed: 101 / 202 / 301` は Public セットの 1 サンプルへ移す +- レジーム名から実行番号（`-01`）を外し、`calm` / `cex-drift` / `informed-flow` / `whale` / + `lending-incident` / `depeg` / `crash` の 7 本に整理する +- シード集合は `config/scenarios/public.yaml` / `private.yaml` に列挙する。private は配布物に含めない +- `run.blocks` = **R = 360** をレジームの一部として固定する。上限は anvil の歴史保持深度 + ~1,050 ブロック（採点再構成が歴史ブロック読取に依存するため。ADR 0016 §5）で、360 はその 1/3 に収まる。 + 360 を選ぶ根拠は次の 3 点である: + - crash の窓（ramp3 + hold6 + decay8 = 17 ブロック）が 3〜4 回入り、1 回の巡り合わせで成績が + 決まらない（R=128 だと窓 1 回 = 実質 17 ブロックで勝負が決する） + - 2 秒ブロックで 720 秒となり、prompt 型 agent にも判断の機会がまとまった回数入る + （R=128 では足りない）。**当初はここを「1 判断 ~10 秒なら約 72 回」と見積もっていたが、 + §5 の B1 実測は 13〜44 回だった**（claude-cli で 1 判断 16〜56 秒）。R=360 が R=128 より + まし、という向きは変わらないが、この根拠だけで prompt 型が救われるわけではない + - §3 の運用予算に収まる + +```yaml +# config/scenarios/public.yaml — 生成パラメータごと公開する +regimes: [calm, cex-drift, informed-flow, whale, lending-incident, depeg, crash] +seeds: [101, 202, 303, 404, 505] # 7 x 5 = 35 シナリオ +# private.yaml は同形式・非公開・public と非交差の 20 シード（7 x 20 = 140 シナリオ） +``` + +### 2. Public / Private セットと秘匿モデル + +- **Public 35**（7 レジーム × 5 シード）: regime YAML（レンジ含む）と seed の両方を公開する。 + 参加者はコンペ本体の前のチューニング期間に、手元でこれを回して戦略を作る +- **Private 140**（7 レジーム × 20 シード）: 生成分布は Public と同一族。**レンジは公表し、実現値（seed）のみ秘匿**する。 + これは ADR 0009 の「値でなくレンジを与える」思想と ADR 0016 §2 の「core ソースは公開前提、 + 秘匿は seed のみ」をそのまま踏襲したもので、新しい秘匿機構は導入しない +- **生成ロジックが公開である以上、参加者は自分でシードを振って同分布から無限にシナリオを作れる。 + これは阻止しないし、阻止すべきでもない** — 求めているのは分布への汎化であり、自前シードでの + 検証はむしろ正しい練習法である。「Public の 5 シードに過学習せず、自分でシードを振れ」と + 参加者向けドキュメントに明記する +- Private セットの seed 値そのものはコンペ終了後に公開し、参加者が自分の成績を再現できるようにする + +### 3. 実行モデル: 常設 devnet 1 つ + snapshot/revert + +**フォークは行わず、`--load-state` で全 venue を復元した anvil を 1 つ立てたまま、 +シナリオを snapshot/revert で何度でも回す。**これは ADR 0016 §4 で実装・実証済みの機構であり、 +新規実装は (r,s) 行列の反復と集計に限られる。 + +``` +anvil --load-state venues-state.json ← 常設 devnet。起動は 1 回だけ（~10 秒で全 venue 復元） + ├ evm_snapshot → シナリオ再生（R=360 ブロック）→ 採点再構成 → evm_revert + ├ evm_snapshot → 次のシナリオ … + └ 1 シナリオ ≈ 14 分（mining 12 分 + setup/採点 2 分。`--score-every` で採点を間引いた場合） +``` + +- **状態残留を snapshot/revert で断つ**。同一 devnet で連続実行すると前 run の Aave ポジションが + 残留して PnL を汚染する既知の故障モードがあり、`evm_revert` のクリーン断面がこれを防ぐ。 + victim を建てる stress レジーム（`lending-incident` / `crash`）の fresh state 要件も + この断面で満たされる（ADR 0016 §2 で実証済み） +- **採点再構成は revert の前に完了させる**（歴史ブロックが消えるため。ADR 0016 §4 の既存制約） +- **各シナリオは 1 回だけ実行する**（同一シナリオの反復は採点に用いない）。B1 の tx タイミング + 非決定性は反復ではなく**シナリオ数**で吸収する。1 シナリオのスコアのばらつきは + 「市場条件によるばらつき」と「run ノイズ」に分かれるが、同一 (r,s) の反復は後者を 1/回数に + するだけで前者を一切減らさない。壁時計予算が シナリオ数 × 反復回数 で決まる以上、反復は + シナリオ数を削ることと等価であり、常に損をする。反復は較正時の診断（同じシナリオがどれだけ + ぶれるかの実測。§5）にのみ用いる +- **並列化はポート別の devnet で行う**（同一 anvil への複数 run 同時載せは snapshot/revert が + 干渉して壊れる）。ただし同居構成では 1 シナリオあたりの agent プロセス数が多く、 + 並列度 × ロスターサイズがプロセス数になる点に注意する +- **採点断面の間引きを可能にする**。最終成績は `alphaByAgent = alphaLast − alphaFirst` + （`reconstruct.ts`）で初期断面と最終断面の 2 点しか使わず、中間ブロックの読取は equity curve と + 診断用である。`--score-every N` で間引けるようにする。既定は毎ブロック（equity curve と + 将来の risk-adjusted 指標のため）とし、行列実行時のコスト削減オプションとする + +CLI はシナリオ行列の反復に一般化する: + +``` +npm run backtest -- --scenarios config/scenarios/public.yaml --agents + # 既存の --repeat は較正時の診断用として残す（採点には用いない。§3 / §5） + → runs//scenarios/#/summary.json + → runs//matrix.json (シナリオ × agent の生スコア行列) + → runs//standings.json (§4 の集約結果) +``` + +#### コンペ本体の運用期間（1 週間・提出凍結） + +**参加者の提出物は運用期間の開始時点で凍結する。**週の途中で更新できないため、140 シナリオを +週内のどこで回しても成績の意味は変わらず、日程は純粋に運用の都合で組める。 +（週中に更新を許すリーグ戦形式は「後から提出した者が有利/不利」を生むため採らない。 +Private セットの成績が日々参加者に返ると、弱いレジームを特定して寄せられ、汎化テストとして +成立しなくなるという理由もある。） + +| 項目 | 値 | +|------|-----| +| 実働時間（1 週間 = 168 h から障害対応・再実行のバッファを引く） | 140 h | +| 1 シナリオ | 14 分（R=360 × 2 秒 + setup/採点） | +| Private 140 シナリオ（各 1 回） | 140 run = **33 時間** = 1 日 5 時間運転 | +| 残余 | 約 107 時間（ヒート分割・障害・再実行のバッファ、およびシード数の拡張余地） | + +**余剰予算はシード数の拡張に充てる。**ヒート数が確定した時点で、1 ヒートで済むなら +シードを 40（= 280 シナリオ・65 時間）まで伸ばせる。2 ヒート必要なら 20 シードのまま +（280 run・65 時間）とする。シナリオ数を増やすことが検出力に最も直接効くため、 +バッファを確保したうえで残りはすべてここへ振る。 + +**予算を食い潰す唯一の要因はヒート分割である。**参加者数が同居可能なロスターサイズを超えると +予選グループに分割せざるを得ず、run 数がヒート数だけ倍増する（2 ヒートで 560 run = 131 時間となり +バッファが消える）。したがって**参加者募集の上限は、§5 で実測する同居可能ロスターサイズから +逆算して決める**。 + +**ただしヒート分割はコストの問題では済まない。§4 の採点が成立しなくなる。** +z-score は「同一シナリオ内の参加者間」で計算する — 全員が文字通り同じ run にいるからこそ +「同じ市場で誰が上だったか」が意味を持つ（軸 1 で同居を選んだ理由そのもの）。ヒートに分けると +参加者は**それぞれ別の顔ぶれに対して**正規化されるため、強者が偏ったヒートに入った参加者は +同じ実力でも低い z になり、**ヒート間で順位を合算する根拠が失われる**。 + +したがってヒートが必要になった場合、run を増やすだけでは足りず、ヒート間の基準を揃える仕組み +（全ヒートに共通の参照 agent を入れて、その z を基準に補正する等）が別途要る。それは本 ADR が +設計していない機構である。**「全参加者が毎シナリオ同じ run にいる」は運営の都合ではなく、 +採点方式が成立するための前提である。** + +なお §5 の B2 再測定では 36 体でも負荷・順位ともに健全だったため、現実的な参加者数であれば +分割せずに済む見込みが高い。 + +なお Public 35 シナリオは参加者が手元で回すもので、この予算には含まれない +（全巡回で約 8 時間。参加者は部分集合を選んで回せばよい）。 + +### 4. 集約と順位付け + +順位は 3 層で決める。**採点値は当面 `netPnlUsdc`**（USDC 建ての総損益）とする。 + +1. **シナリオ内スコア**: 各 (r,s) で全参加者が同居した 1 run から `netPnlUsdc` を得る +2. **レジーム内 z-score**: レジームごとに参加者スコアを平均 0・標準偏差 1 へ正規化する。 + これにより crash 系レジームの大きな機会が順位を支配する構造的バイアスが消え、かつ + 「僅差の 1 位」と「圧勝の 1 位」の差が保存される +3. **総合**: レジーム 7 種を等重みで平均する。レジーム間でシード数が異なる場合もレジーム内で + 吸収されるため、シナリオ数の偏りが重みにならない + +**採点値も集約式も暫定であり、採点方法そのものは別 issue で比較・検討を継続する。** +そのため `matrix.json` にはシナリオ × agent の**生の指標を両方（`netPnlUsdc` と `alphaUsdc`）常に残し**、 +採点値と集約式を後から差し替えて再計算できる形にする +（`standings.json` は `matrix.json` から導出される派生物とする）。 + +`netPnlUsdc` を採るうえでの前提と帰結を明示しておく: + +- **全レジームで USDC-only funding（`funding.wethWei: "0"`）を徹底する。**`netPnlUsdc` は価格ドリフト + （β）を含む総額なので、初期保有に WETH があると「配られた時点で決まっている損益」が順位に乗る。 + USDC-only なら初期エクスポージャがゼロになり、β は各 agent が**自分の判断で抱えた在庫**からしか + 生じない。既存の `calm` / `lending-incident` / `crash` は既にこの設定であり、WETH を配っている `lst` は + 7 レジームに含まれないため影響しない +- **在庫を抱えることが罰せられる方向にバイアスがかかる。**crash レジームでは現物を持たない `noop` が + 構造的に上位に来る。裁定戦略も窓の間に在庫を持ち越すと β で削られるため、「素早く往復して + フラットに戻る」方向へ戦略が誘導される。これは採点方法を見直すまでの既知の性質として扱う + +失格・欠損の扱いを明示する: + +- run がクラッシュした / agent プロセスが死んだ場合、そのシナリオは **noop 相当ではなく最下位**として扱う + （黙って 0 にすると「死ぬのが安全」というインセンティブが生まれる） +- `postRunCheck` の `violations`（fee 上限超過等）が出た参加者はそのシナリオで最下位扱いとする +- `scoring_unpriced_holdings`（採点が値付けできなかった保有）は集計から外さず報告に残す + （issue #41 / #38 の既存方針を維持） + +### 5. パイロットで確定する較正値 + +次の値は机上では決まらない。**実装より先に、既存の 3 regime × 5 seed × 強弱が既知の 3 agent +（noop / venue-arb / multi-arb）でパイロットを回して実測で決める。** + +#### 第 1 回パイロットの結果（実施済み） + +`calm#101` を R=40 に短縮し、ロスター 4 / 8 / 16 / 24 体で実測した（`--score-every 8`）。 + +| ロスター | 壁時計（40 ブロック） | 上位 arb の netPnlUsdc | 所見 | +|---------|---------------------|----------------------|------| +| 4 | 78 秒 | +118 / +64 | 機会が余っている | +| 8 | 78 秒 | +99 / +48 / +33 | まだ余裕 | +| 16 | 87 秒 | +3.1 / +2.2 / +1.8 | **機会が枯れる**（1/30 以下に希釈） | +| 24 | 85 秒 | −15 〜 −19 | 全員が薄利〜損失 | + +- **anvil は 24 体でも壊れなかった。**壁時計はブロック生成に追随し（40 ブロック × 2 秒 = 80 秒に対し 78〜87 秒）、 + agent プロセスの早期終了もゼロだった。**上限を決めるのは負荷ではなく「機会の希釈」である** — + 16 体の時点で裁定利益が 1/30 以下になり、24 体では手数料負けする。順位を付けるには + 参加者間に差が出る必要があるので、**同居上限は破綻点ではなく「識別力が残る密度」で決めるべき** + というのが最大の学びである。機会量はレジームと R に依存するので、この数字を本番の R=360 と + 本番レジームで測り直すことが第 2 回パイロットの主目的になる +- **副産物として採点バグを 1 件発見・修正した**（issue #53）。24 体の run で WBTC を扱う + `multi-arb` 4 体が**セント単位まで同一の −6,686 USDC** を計上し、2 回の反復でも再現した。 + 原因は `netPnlUsdc` が非 WETH ベースを価格 0 で評価していたこと（`valueUsdc` にスカラーの WETH + 価格を渡していた）で、同じ run の `alphaUsdc` は +13 だった。**採点値を `netPnlUsdc` に決めた + 直後だったため直撃していた。**修正後は同じ run が −14〜−19 に戻る +- **同一シナリオの再現性**（診断項目）: ロスター 24 の 2 回反復で、agent ごとの符号と桁は一致した + ものの個々の値はぶれた（例 `cross-venue-arb-2`: −10.25 → −15.01）。R=40 の短い run での観測なので、 + 本番 R での ぶれ幅は測り直しが必要 + +#### B1: prompt 型の判断頻度（本番 R=360・claude-cli） + +`calm#101` を **本番設定のまま**（R=360 / blockTimeSec 2 秒 / 718 秒）、同一戦略をルール型と +prompt 型で同居させて実測した（ロスター: noop / multi-arb ×2 / venue-arb ×2）。 + +| agent | 判断回数 | 平均間隔 | 送信 tx | netPnlUsdc | +|-------|---------|---------|--------|-----------| +| multi-arb（ルール型） | 毎ブロック | — | **192** | **+254.4** | +| multi-arb（prompt 型） | 13 | 56.1 秒 = **28 ブロック** | 3 | +0.7 | +| venue-arb（prompt 型） | 44 | 15.9 秒 = **8 ブロック** | 0 | 0 | + +- **不利は見積もり（5 ブロックに 1 回）より 1 桁大きい。**実測は 8〜28 ブロックに 1 回で、 + 行動回数はルール型の **1/64**（192 tx 対 3 tx）。この差では prompt 型は競技として成立しない +- 2 体の prompt agent で間隔が 2 倍違うのは、`claude-cli` が呼び出しごとにプロセスを起動する重い + バックエンドで、しかも同居 2 体が同じ CLI を取り合うため。**これは「LLM 一般」ではなく + 「claude-cli をバックエンドにした場合」の数字**であり、API 直叩き（ollama / Anthropic API）なら + 短くなる。ただし桁が 1 つ縮んでもルール型との差は残る +- したがって §6 の選択肢 (a)「`blockTimeSec` を上げる」で埋めるには 8〜28 倍に伸ばす必要があり、 + §3 の週予算が崩れる。**(b) 第 3 のモード（速いルールループを LLM が定期的に書き換える）の実装が + 現実的な解**という方向に、実測が寄せている + +#### B2: 同居上限（本番 R=360・calm）— 測定が成立しなかった + +`calm#101` を R=360 のまま、ロスター 8 / 16 / 24 / 32 体で回した（同梱 arb 5 種を巡回配置）。 + +| ロスター | multi-arb 平均 | cross-venue-arb 平均 | **一度も取引しなかった agent** | +|---------|---------------|---------------------|------------------------------| +| 8 | −79.7 (n=2) | +11.1 (n=2) | 2 / 7 | +| 16 | −79.4 (n=3) | −13.4 (n=3) | 6 / 15 | +| 24 | −74.8 (n=5) | −14.7 (n=5) | 9 / 23 | +| 32 | −65.0 (n=7) | −12.2 (n=6) | **18 / 31** | + +**この測定は同居上限を答えていない。**理由が 2 つある: + +1. **同梱 agent の半分以上が calm で取引しない。**`clean-arb` / `stat-arb`、そして 32 体では + `adaptive-arb` も netPnlUsdc がきっかり 0.00 = 一度も約定していない。issue #54 で + `venue-arb` について特定した「USDC-only 配布で WETH 売りしか提案せず自己 reject する」 + のと同じ症状が、**同梱 arb 群に広く存在する**とみられる。ロスターを 32 体に増やしても + 実際に市場に触れているのは 13 体で、負荷も競合も名目の 4 割しかかかっていない +2. **同一戦略の複製が互いを潰す。**`multi-arb` は B1（1 体）で **+254** だったのに、 + 2 体以上いると人数に関係なく **−65〜−80** で安定する。同じ機会を同時に取りに行って + 手数料だけ払う共食いで、**ロスター人数ではなく「同じ戦略が何体いるか」で決まっている** + +したがって「16 体で機会が枯れる」という第 1 回パイロット（R=40）の結論も、 +**機会の希釈ではなく同一戦略の共食いを見ていた**可能性が高い。実際のコンペでは参加者が +異なる戦略を出すので、この数字はそのまま使えない。 + +**anvil 自体は 32 体でも壊れなかった**（360 ブロックを 720 秒 = ブロック生成に完全追随、 +agent プロセスの早期終了ゼロ）。負荷側の上限はまだ見えていない。 + +**やり直しの条件**: 同居上限を測るには「**実際に取引する、互いに異なる戦略**」を人数分揃える +必要がある。issue #54 の修正が前提になる。それまで参加者募集の上限は決められない。 + +#### B2 再測定（issue #54 修正後・本番 R=360） + +#54 を直したうえで、**実力が明確に異なる 8 戦略**（venue-arb / multi-arb / clean-arb / +cross-venue-arb / adaptive-arb / arb-bot / max-profit-arb / 既知の悪手 random）を核として固定し、 +同じ戦略を追加参加者として詰めてロスターを 9 / 18 / 27 / 36 体に増やした。核の 8 体は +どのロスターでも同一 id・同一順序なので、密度をまたいで順位を比較できる。 + +| 核の agent | 9 体 | 18 体 | 27 体 | 36 体 | +|-----------|-----:|-----:|-----:|-----:| +| clean-arb | +10.3 | 0.0 | 0.0 | 0.0 | +| venue-arb | +2.3 | +2.1 | +0.1 | −1.0 | +| arb-bot | −0.9 | +0.7 | −0.8 | −1.0 | +| adaptive-arb | −1.3 | −0.4 | −1.6 | −2.0 | +| multi-arb | −9.6 | −24.7 | −75.5 | −25.4 | +| max-profit-arb | −17.1 | −30.7 | −21.9 | −27.9 | +| cross-venue-arb | −40.4 | −57.7 | −59.7 | −61.0 | +| random | −1102 | −1115 | −1057 | −1113 | + +**順位は 36 体まで崩れない。**最小ロスターの順位に対する Spearman ρ は +9→1.000 / 18→0.929 / 27→0.905 / 36→1.000。入れ替わるのは実力が元々拮抗している中位 +（clean-arb / venue-arb / arb-bot が ±2 USDC の中に固まっている）だけで、**上位と下位、 +とくに `random` の最下位は一貫している**。 + +**負荷側も 36 体で健全。**360 ブロックを 718 秒 = ブロック生成に完全追随（理論値 720 秒）、 +agent プロセスの早期終了 0 件。**破綻点は 36 体では見つからなかった。** + +読み取れること（**当初の読みは誤りだったので訂正した内容である**）: + +- **機会は枯れていない。**roster 36 の約定回数は cross-venue-arb 240 / multi-arb 185 / + max-profit-arb 62 に対し **clean-arb は 0**。機会は存在し、**取りに行った agent が損をしている** + （cross-venue-arb −61.0 / multi-arb −25.4 / max-profit-arb −27.9）。clean-arb の 0 回は + 「往復コストを超える機会が無い」という正しい判断で、その正しさを他の損失が裏付けている。 + 密度が上げるのは「機会の消滅」ではなく「**取ると損をする機会の比率**」である +- 首位のスコアは +10.3（9 体）→ +1.1（36 体）に崩れる一方、noop を上回る agent は 4/35 残る。 + 順位全体は保たれ、壊れるのは**首位付近の解像度**である +- **`random` の −1100 は密度に依らない。**下手な戦略は人数が増えても罰せられる。 + 難しくなるのは「そこそこ」と「良い」の区別であって、良し悪しの区別ではない + +#### ノイズは別途測る必要がなかった（複製が実測値になっている） + +同一戦略の複製が同じ run にいるので、**その差がそのままノイズである**（同じコードなので実力差は +ゼロ、差は着順の運だけ）。§5 が「診断項目」として挙げていた ぶれ幅の測定は、この形で済んでいる。 + +| ロスター | 同一戦略間の差（ノイズ） | 実力差の幅（random 除く） | 比 | +|---------|----------------------:|----------------------:|---:| +| 18 | 1.23 | 61.3 | 50 | +| 27 | 11.49 | 83.3 | 7.2 | +| 36 | 7.44 | 65.4 | 8.8 | + +**信号はノイズの 7〜9 倍あり、36 体でも順位全体は意味を持つ。** + +#### 首位の解像度は単一シナリオでは失われている + +roster 36 を z 単位で見ると、上位 4 体は**すべて venue-arb の複製**で、z の差は 0.0002〜0.0029。 +同一戦略間の z ノイズは 0.041 なので、**単一シナリオの首位順は完全に運**である。 +ただしこれは同じコードなので当然の結果であり、**「異なる良い戦略同士が分離できるか」の証拠には +ならない**（この field に拮抗した別戦略が無いため未検証）。 + +#### z-score が外れ値に脆い（本測定で見つかった採点方式の欠陥） + +roster 36 の sd は **181.5**、`random`（−1113）を除くと **20.91**。 +**壊滅的に負ける参加者が 1 体いるだけで、残り 34 体の z が 1/8.7 に圧縮される。** +数百人規模ならそうした参加者は必ず出るので、**1 人が残り全員を統計的に区別不能にする**。 + +首位の解像度を回復する手段は、コストの安い順に: + +1. **z-score の頑健化** — 中央値/MAD 正規化、winsorize、または軸 2 の B 案（順位ベース）への変更。 + `random` を除くだけで解像度が 8.7 倍になるので、費用対効果が最も高い +2. **140 シナリオの平均**（既に設計にある）— 単一シナリオのノイズは √140 ≈ 12 倍薄まる +3. **flow 強度の増強** — 機会の総量を増やして実力差そのものを広げる。環境の再較正を伴い最も高コスト + +**残る限界**: 詰めた参加者は核と同じ戦略の複製なので、本物の参加者フィールドより機会の食い合いが +強く出ている可能性がある。また 1 レジーム・1 シードの測定である。 + +**副次的な発見**: 同梱の `venue-arb`（ルール型）は calm で **359 ブロック全部で自分の action を +自己 reject** した（`amountIn exceeds balance`。USDC-only 配布で WETH を 0 しか持たないのに +WETH 売りを提案し続ける）。パイロットで「既知の強弱」の中位として使っていた agent が、 +このレジームでは実質 noop だったことになる。順位の再現性は確認できていたが、 +**その中身は「multi-arb だけが動いていた」**。識別力の検証（次段）は別の agent 群で組む必要がある。 + +**Public セット 30 シナリオ（6 レジーム × 5 シード）の通し実行**も R=20 に短縮して実施し、 +集約経路が端から端まで動くことを確認した。強弱が既知の 3 agent での総合順位は次のとおり: + +| 順位 | agent | 総合 | calm | cex-drift | informed-flow | whale | crash | +|-----|-------|-----|------|-----------|---------------|-------|-------| +| 1 | multi-arb | +1.23 | 1.14 | 0.93 | 1.41 | 1.25 | 1.41 | +| 2 | venue-arb | −0.31 | −0.33 | 0.30 | −0.71 | −0.05 | −0.78 | +| 3 | noop | −0.91 | −0.81 | −1.23 | −0.70 | −1.19 | −0.63 | + +- **既知の強弱が全 5 レジームで一貫して再現した**（multi-arb > venue-arb > noop）。順位付けの経路 + そのものは機能している。ただし R=20 の短縮 run なので、これは識別力の測定ではなく機構の検証である +- `cex-drift` だけ `venue-arb` が正に転じており、レジームごとに得意な戦略が変わるという + 設計意図（レジーム等重みで総合を取る理由）が実際に現れている +- **失敗シナリオの除外経路も実地で検証できた**: `--protocols` から aave を外していたため + `lending-incident` の 5 本が victim 前提を満たさず fail-fast した。行列は中断せずに残り 25 本を + 完走し、5 本を `excludedScenarios` に記録して集計から外した（全員 0 点にしなかった） + +#### ADR 0018 が前提を変える項目 + +**§5 の較正は ADR 0018（LLM は取引判断ではなく戦略を書き換える）の実装後にやり直す。** +0018 は LLM 型 agent の取引ループを毎ブロック駆動にするため、1 体あたりの送信 tx が +実測 3 件から ルール型と同等の ~192 件へ跳ね上がる。**同居上限も識別力も「1 体が 1 ブロックに +何をするか」の関数**なので、現行の混成ロスターで測った値は 0018 の実装と同時に無効になる。 + +- **`blockTimeSec`** — 0018 が行動回数の差を構造的に消すので、これを上げる動機自体が消える。 + 較正項目から落とす +- **同居可能なロスターサイズ** — 0018 後に再測定（B2 は現時点でも測定不成立。下記） +- **識別力の検証** — 提出物が「初期戦略 + 改善プロンプト」に変わるため、何を識別するのかが変わる + +#### 残っている較正値 + +| 較正値 | 測り方 | 既定（暫定） | +|--------|--------|-------------| +| **同居可能なロスターサイズ** | 再測定済み: 36 体まで順位全体は崩れず（Spearman ρ 0.905-1.000、信号/ノイズ 7〜9 倍）、負荷も健全。壊れるのは**首位付近の解像度**のみ。回復手段は z の頑健化 → シナリオ数の平均 → flow 増強の順（コスト昇順） | **36 体以上**（破綻点は未発見） | +| **R（1 シナリオのブロック数）** | 充足性で決める（下記）。R=180 / 360 / 720 で、crash 窓の回数と LLM 型の判断回数が戦略の発揮に足りるかを見る | 360 | +| シード数（Public 5 / Private 20〜40） | 順位がシードをまたいで安定するか。ヒート数の確定後、余剰予算はすべてここへ振る | 5 / 20 | +| `run.blockTimeSec` | LLM 型とルール型の成績差が判断頻度に起因するか（§6 のリスク） | 2 秒（現行） | +| レジームあたりのイベント回数 | 1 run 内で複数回起きることで、1 回の巡り合わせが成績を決めない構造になっているか | レジームごとに 3〜4 回 | +| `--score-every N` | 採点断面を間引いても equity curve の診断価値が保てる N | 8 | +| ~~同一シナリオのぶれ幅~~ | **測定済み**。同一戦略の複製が同じ run にいれば、その差がそのままノイズになる（実力差ゼロなので）。専用の反復 run は不要だった | R=360・36 体で 7.44 USDC / z 0.041 | +| `whale` レジームの magnitude | 30 ブロックのスモークで同梱 arb が 25,000 の資本に対し 1,600〜2,800 USDC を稼いだ。大きすぎて「居るだけで儲かる」可能性があり、識別力で較正し直す | [25, 60] WETH | + +**R とシード数は役割が違う。R は「充足性」で決め、余った予算はすべてシード数へ振る。** +R はシナリオがその現象を含みきれる長さでなければならない — crash の窓（17 ブロック）が +複数回入り、LLM 型 agent が戦略を発揮できる回数だけ判断できること。この条件を満たしたら +それ以上 R を伸ばす価値は薄く、同じ予算はシードを増やすほうが検出力に直接効く。 +シードを増やすと価格パス・イベントの規模と配置がまるごと新しく引き直されるのに対し、 +R を伸ばしても引くのは同じ seed が定めた 1 本の道筋の続きだからである。 + +### 6. 同居構成の帰結 + +軸 1 で同居を選んだことにより、次が競技の一部になる。参加者向けドキュメントに明記する +（`docs/guide/backtest.md`「What the competition actually scores」に記載済み）。 + +- **他参加者との機会の食い合い**（清算チャンス・裁定機会は早い者勝ち） +- **ブロック内順序**: anvil は `--order fees` で並ぶため、順序は gas 入札で買える（ADR 0011） +- **思考の速さ**: `blockTimeSec` が 2 秒である限り、prompt 型 agent は実質 5 ブロックに 1 回しか + 動けず、ブロック駆動のルール型に対して構造的に不利になる。現行の雛形ロスターが prompt モード + 既定であることと緊張関係にあるため、§5 のパイロットで成績差を実測する + + この差は agent ランタイムの構造から来る（`example/agents/runtime/bot.ts`）。**prompt 型は LLM が + 毎回の取引判断そのものを下す**: 1 サイクルごとに observation を添えて LLM を呼び、返ってきた + action を 1 つ投げる。`cycling` ガードで多重実行を防ぐため実効サイクルは + `max(intervalMs, LLM レイテンシ)` となり、既定 `intervalMs` 5 秒に対し LLM が ~10 秒かかるので + ~10 秒 = 5 ブロックに 1 回になる。ルール型（`decide`、`intervalMs` 無指定）は毎ブロック動く。 + `ERIS_PROMPT_REVISE_EVERY` は「N サイクルごとに LLM が自分のプロンプト本文を書き直す」自己改訂で、 + 既定 0=off かつ prompt モード内でのみ有効 — **取引判断は自己改訂の有無に関わらず毎サイクル LLM を + 通る**。「速いルールループを LLM が定期的に書き換える」第 3 のモードは現状存在しない + （旧 `src/llm` の二層自己改善機構は ADR 0015 で削除済み）。 + + したがって不利が実測されたときの選択肢は 2 つある: (a) `blockTimeSec` を上げて判断機会を揃える + （1 シナリオの壁時計が伸び、§3 の週予算に波及する）、(b) 第 3 のモードを実装する。 + どちらを採るかは「参加者が提出するのはどちらの種別か」というコンペ設計の問題であり、 + パイロットの結果を見てから決める + +一方、ADR 0016 の参加者向けバックテスト（手元・単独または少数ロスター）は**練習用として存続**する。 +公式スコアは運営の常設 devnet における同居 run のみとする。 + +### 7. 実装フェーズ + +1. **Phase 0（パイロット）**: 最小のシナリオ行列 runner + 集計スクリプト。既存 3 regime × 5 seed で + §5 の較正値を実測する +2. **Phase 1**: `run.seed` の regime からの分離、`config/scenarios/*.yaml`、 + `matrix.json` / `standings.json` の出力、`--score-every`、ADR 0016 §3 の override 規約改訂 +3. **Phase 2**: 未実装レジーム — `market.*` の YAML 露出（`cex-drift`）、方向性フロー強度ノブ + （`informed-flow`）、whale 点イベント、LP 撤退イベント（`crash`） +4. **Phase 3**: depeg レジーム（issue #39）。USDC 建て採点と建値通貨の変動が衝突するため、 + 採点方法の見直し（別 issue）とセットで最後に扱う + +## Consequences + +### Positive + +- コンペの成績決定が「シナリオ行列 + 集約規則」として明文化され、ADR 0016 が先送りした + 合否基準の空白が埋まる +- 実行基盤（常設 devnet + snapshot/revert + 採点再構成）が ADR 0016 の実装済み機構そのままで、 + 新規コードはシナリオ行列 runner と集計器に閉じる +- 同居構成により Private セットの run 数がシナリオ数（140 run）に一致し、参加者数に対して + 線形に増えない。1 週間の運用予算 140 時間に対して 33 時間で済み、残りをヒート分割と + シード数の拡張に回せる +- 全参加者が文字通り同一 run を通るため paired 比較が最も強い形で成立し、過去に検出不能だった + 実力差が検出可能になる +- レジーム内 z-score により、機会の大きいレジームが順位を支配する構造的バイアスが消える +- `matrix.json` に生スコアを残すため、順位付け方式を後から差し替えて再計算できる + +### Negative + +- 順位が同居メンバーに依存する（誰と同じ run に入ったかで機会の食い合いが変わる） + - → 全参加者が全シナリオに同居するため、メンバー構成はシナリオ間で一定になり、 + 参加者間の相対比較としては一貫する。絶対値の解釈にのみ注意を要する +- レジーム 7 種のうち 2 種（depeg / 流動性引き抜き）が新規実装であり、コンペ開始までの + リードタイムを消費する + - → Phase 2/3 に置き、間に合わない場合はレジーム数を減らして開催する（集約はレジーム等重みなので + レジーム数の変更に耐える） +- z-score により「どれだけ勝ったか」の絶対額が順位から見えにくくなる + - → `matrix.json` に生スコアを常に残し、順位とは別に絶対値を読めるようにする + +### Risks + +- **同居構成で agent が密集し anvil が破綻する / ヒート分割が運用予算を食い潰す** + - → 過去にフルロスターで observe が殺到し破綻した前例がある。§5 のパイロットで + 同居可能なロスターサイズを実測し、そこから**参加者募集の上限を逆算して確定する**。 + 上限を超えて予選グループに分割すると run 数がヒート数だけ倍増し、2 ヒートで + 1 週間の予算（140 時間）を使い切ってバッファが消える。シナリオ並列度 × + ロスターサイズがプロセス数になる点も併せて実測する +- **`blockTimeSec` 2 秒のままだと LLM 型参加者が構造的に不利になる** + - → §6 のとおりパイロットで成績差を実測し、判断頻度が原因なら `blockTimeSec` を上げる。 + それでも足りない場合は B2 同期ステップ再生（ADR 0016 Phase 1）を再検討する +- **B1 の run ノイズが実力差を覆う** + - → 同一シナリオの反復ではなく**シナリオ数**で吸収する（§3）。加えて 1 run 内のイベント回数を + 増やして、1 回の巡り合わせが成績を決めない構造にする。ノイズの実際の大きさは §5 の診断で + 実測し、シナリオ数を増やしても覆いきれないほど大きい場合は `blockTimeSec` の引き上げか + B2 同期ステップ再生を再検討する +- **参加者が Private シードそのものではなく「Public の 5 サンプル」に過学習する** + - → 生成分布が同一族であることと、自前シードでの検証を推奨することをドキュメントで明示する。 + Public/Private の成績乖離を運営側で監視し、乖離の大きさを過学習の指標として観察する +- **常設 devnet の長時間運用で状態が汚染される** + - → snapshot/revert を毎シナリオ実行し、採点再構成を revert 前に完了させる。 + anvil の歴史保持深度 ~1,050 ブロックを超えないよう R を制限する +- **state dump と repo の constants がドリフトする** + - → ADR 0016 の fingerprint fail-fast をそのまま継承する。Private セットの実行前に + dump の再生成が必要かを必ず確認する + +## 決めていないこと + +| 項目 | 決めない理由 | いつ決めるか | +|------|------------|------------| +| 採点値（`netPnlUsdc` / `alphaUsdc` / ETH 建て）と順位付けの計算式 | 採点方法そのものを将来的に見直す予定であり、別 issue で比較・検討を継続する。`matrix.json` に両指標の生スコアを残すため後付けで再計算でき、実行基盤の設計を妨げない | 別 issue | +| §5 の較正値（シード数 / K / `blockTimeSec` / R / イベント回数 / ロスターサイズ） | 検出力・負荷は実測でしか決まらない | Phase 0 のパイロット完了時 | +| B2 同期ステップ再生を採用するか | `blockTimeSec` の調整で足りるかを先に測る。足りた場合は実装しない | パイロットで LLM 型とルール型の成績差を測った時点 | +| グローバルな経済クロック（1 ブロック = 現実の N 分） | 現状は LST venue のみが `simulatedSecondsPerBlock` を持つ。全体へ入れると Aave 金利・GMX funding が run 長で意味を持つ利点がある反面、`block.timestamp` を飛ばすので全 venue の再較正と既知の timestamp overflow のリスクを負う。当面はイベント回数の調整で足りる | 金利・funding を競技軸に含めたくなった時点 | +| 賞金・順位の刻み方（上位 N 名等） | コンペ運営の問題でありアーキテクチャでは決まらない | コンペルール策定時 | +| 提出 bundle の入口ゲート基準（クラッシュ / validate / noop 連発） | ADR 0016 §7 が提示した副産物であり、本 ADR の集約規則とは独立 | 提出フロー確定時 | +| 歴史価格 CSV を fair パスとして与える実データ駆動レジーム | 合成レジーム 7 種で足りるかは運用後にしか分からない（ADR 0016 の同項目を継承） | Public セット運用後 | + +## Notes + +### 参考資料 + +- ADR 0005: 実時間化と統計評価 — 「regime = 市場条件のラベル」「同一 regime でも結果はぶれる」の出典。 + 本 ADR はこの「ぶれ」を、同一シナリオの反復ではなくシナリオ数と集約規則で扱う +- ADR 0009: 市場ストレスイベントと清算 — 「値でなくレンジ、実現値は seed が引く」。 + Public/Private の秘匿モデルはこの思想の延長。`stress.events` が配列でイベントを複数持てることは + §5 のイベント回数較正の前提 +- ADR 0016: 参加者バックテスト — 実行基盤（state dump / regime 再生 / snapshot-revert / 採点再構成）の + 出典。§3 の override 規約は本 ADR で改訂対象になる。B2 同期ステップ再生（Phase 1）は + 本 ADR では採用を見送り、「決めていないこと」へ移した +- ADR 0011: economic gas — 同居構成でブロック内順序を gas 入札で買える根拠 +- ADR 0013 / 0015: config 単一ソース（regime YAML のスキーマ元）/ agent 契約と bundle +- issue #38（LST venue、ETH 建て採点の motivation）/ issue #39（depegged eUSD）/ issue #41（値付け不能保有） +- ASCON の Market shocks 分類 — レジーム 7 種の出典 diff --git a/docs/adr/0018-llm-rewrites-the-strategy.md b/docs/adr/0018-llm-rewrites-the-strategy.md new file mode 100644 index 0000000..28501b6 --- /dev/null +++ b/docs/adr/0018-llm-rewrites-the-strategy.md @@ -0,0 +1,262 @@ +# ADR 0018: LLM は取引判断ではなく戦略を書き換える（prompt モードの置換） + +## Status + +Accepted（2026-08-10 実装。Phase 1-2。branch feat/lst-venue） + +### 最初の実走（存在証明であって、有効性の証明ではない） + +`calm#101` / R=150 / claude-cli で、同一戦略の自己改善版と frozen 版を同居させた: + +| agent | netPnlUsdc | 改訂 | +|-------|-----------|------| +| venue-arb（自己改善） | **+57.6** | block 775 で採用(v1)、block 835 で辞退 | +| venue-arb-frozen | +10.0 | — | +| noop | 0.0 | — | + +- モデルは「`the widest gaps need inventory this agent does not hold` で 12 ブロック以上停止＝ + 構造的デッドロック」と診断して書き換えた。**agent が書いた noop の理由文字列がそのまま + 改善の根拠になっている** — 証拠チャネルとして意図どおり機能した +- 次の機会では「v1 は効き始めたばかりなので触らない」と**辞退**した。§4 が狙った + 「勝っているときに触らない」がそのまま出た。先行実装の失敗診断への直接の対処である +- 生成コードの reject は 0 件 + +**これは 1 run・1 シードであり、機構が動くことの存在証明にすぎない。**先行実装は multi-seed の +paired 比較で frozen に負けている（1 件は p=0.031 の有意敗）。有効性の主張には同じ規模の検証が要る。 + +## Context + +ADR 0015 §2 は agent の契約を 3 種と定めた: `decide(obs, ctx)`（ルール型。毎ブロック駆動）、 +`run(ctx)`（自走型）、`prompt.md`（プロンプト型）。このうち prompt 型は **LLM が毎回の取引判断 +そのものを下す**: `example/agents/runtime/bot.ts` の `runPromptLoop` が 1 サイクルごとに +observation を添えて LLM を呼び、返ってきた action を 1 つ投げる。`cycling` ガードで多重実行を +防ぐため実効サイクルは `max(intervalMs, LLM レイテンシ)` になる。 + +`config/example.yaml` の Quick Start 既定ロスターは prompt モードで出荷しており、同梱 19 agent +すべてが `prompt.md` を併置している。つまり prompt 型は「LLM を使う唯一の道」として据えられている。 + +### ADR 0017 §5 B1 の実測: prompt 型は競技として成立しない + +本番設定（`calm#101` / R=360 / blockTimeSec 2 秒 / 718 秒）で、**同一戦略**をルール型と prompt 型で +同居させて測った: + +| agent | 判断間隔 | 送信 tx | netPnlUsdc | +|-------|---------|--------|-----------| +| multi-arb（ルール型） | 毎ブロック | **192** | **+254.4** | +| multi-arb（prompt 型） | 56.1 秒 = **28 ブロック** | 3 | +0.7 | +| venue-arb（prompt 型） | 15.9 秒 = **8 ブロック** | 0 | 0 | + +行動回数でルール型の **1/64**。ADR 0017 §6 が想定した「5 ブロックに 1 回」より 1 桁悪い。 +バックエンドは `claude-cli`（呼び出しごとにプロセス起動）で、API 直叩きなら短くなるが、 +桁が 1 つ縮んでも差は残る。 + +ADR 0017 §6 はこの不利への対処として 2 つを挙げていた: + +- (a) `blockTimeSec` を上げて判断機会を揃える — **8〜28 倍**必要で、1 シナリオ 12 分が数時間になり + ADR 0017 §3 の 1 週間運用予算が崩れる +- (b) 「速いルールループを LLM が定期的に書き換える」第 3 のモードを実装する + +本 ADR は (b) を扱う。 + +### 先行実装とその否定的な結果 + +**この方式は過去に実装され、効かないと結論されている。**削除済みの `src/llm` +（commit `8c41bfc` / `07ca8be` / `319cf06`。復元は `git show 319cf06:src/llm/...`）は +LLM が戦略を書き換える二層構造で、その戦略表現は: + +```ts +export type Strategy = { + version: number; + notes: string; + params: Record; + executorTs: string; // LLM が書いた TypeScript。node:vm サンドボックスで実行 +}; +``` + +multi-seed 検証の結論は **frozen 戦略に負け**（paired 比較で 1 件は p=0.031 の有意敗）、 +用意されていた rollback は 18 run 中 0 件で発火しなかった。原因は「好機レジームで upside を削る」 +と診断されている（保守化して当たりを取り逃す）。 + +当時と現在で違うのは 3 点である。**本 ADR はこの差分を設計に織り込まなければならない**: + +1. 環境が α 支配へ較正し直された（ADR 0007）。当時は β が PnL を支配しており、 + 自己改善の良し悪しが埋もれていた可能性がある +2. 当時は **run をまたぐ**改善（run N の結果で run N+1 の戦略を作る）だった。本 ADR は + **run 内**で回す。フィードバックが同一市場条件の中で閉じる +3. 当時は「1 体の agent が自分を改善できるか」だったが、コンペでは**複数参加者の相対比較**であり、 + 自己改善の巧拙そのものが競技軸になる（自己改善 agent 同士の paired 比較では実力差が + 有意に検出できることは実証済み） + +### 解決したい課題 + +- 取引ループを LLM レイテンシから切り離し、LLM 型と ルール型が同じ行動回数で競えるようにする +- LLM の使いどころを「毎回の判断」から「戦略の改善」へ移す +- 参加者が提出するものと、そこで測られる技能を明確にする +- 悪化する書き換えから run を守る（先行実装の失敗モード） + +### 検討した選択肢 + +**軸 1: LLM が書き換える対象** + +| 観点 | A. コード（VM サンドボックスの executor） | B. パラメータ | C. 宣言的ポリシー（DSL） | +|------|----------------------------------------|-------------|----------------------| +| 表現力 | 高い（新しい戦略そのものを書ける） | 低い（戦略は運営が固定し、参加者は係数だけ） | 中 | +| 先行実装 | あり（`executorTs` + `node:vm`。復元可能） | なし（既存の env ノブで近似は可能） | なし（DSL の設計から） | +| 安全性 | サンドボックス + `check:strategy` の静的検査が要る | 自明に安全 | 自明に安全 | +| 失敗モード | 生成コードのコンパイル/実行時エラー | 表現力不足で改善の余地がない | DSL の制約が恣意的に見える | +| 測る技能 | 戦略設計 | パラメータ探索 | 中間 | + +**軸 2: 参加者の提出物** + +| 観点 | A. 初期戦略 + 改善プロンプト | B. 改善プロンプトのみ | C. 初期戦略のみ（改善ループは運営） | +|------|---------------------------|--------------------|--------------------------------| +| 測る技能 | 戦略設計 + 改善の導き方 | 改善の導き方 | 戦略設計のみ（LLM 部分は共通） | +| コールドスタート | 強い（block 0 から動ける） | 弱い（最初の改訂まで無戦略） | 強い | +| 参加のしやすさ | 中 | 高い（プロンプト 1 枚） | 高い | + +**軸 3: 悪化した書き換えの扱い** + +| 観点 | A. 常に採用 | B. rollback（悪化したら戻す） | C. shadow 評価してから採用 | +|------|-----------|---------------------------|------------------------| +| 実装 | ゼロ | 判定基準と巻き戻しが要る | 並行評価の機構が要る | +| 先行実装の教訓 | — | **18 run 中 0 件で発火しなかった**（基準が緩すぎた） | 未検証 | +| リスク | 1 回の悪い書き換えで run が終わる | 判定が run ノイズを拾うと改善も巻き戻す | run 内では「もし〜だったら」を作れない（閉ループ） | + +**軸 4: 改訂の起動条件** + +| 観点 | A. 固定周期（N ブロックごと） | B. 成績トリガー（負けているときだけ） | C. 両方 | +|------|--------------------------|--------------------------------|--------| +| 予測可能性 | 高い（コストが読める） | 低い（LLM 呼び出し回数が run による） | 中 | +| 先行実装の失敗との関係 | — | 「好機レジームで upside を削る」を悪化させうる（勝っている間は触らない＝正しい方向） | — | + +## Decision + +**prompt 型（LLM が毎回の取引判断を下すモード）を廃止する。LLM は取引ループの外側に移し、 +`node:vm` サンドボックスで走る executor のコードを書き換える役に限定する。取引ループはルール型と +同じく毎ブロック駆動になり、LLM 型とルール型の行動回数の差は構造的に消える。参加者は +「初期戦略 + 改善プロンプト」を提出し、改訂の起動条件は改善プロンプト側が決める。悪化した書き換えは +rollback する。** + +軸 1 は A（コード）、軸 2 は A（初期戦略 + 改善プロンプト）、軸 3 は B（rollback）、 +軸 4 は「プロンプトが決める」（§4）を採用する。 + +### 1. agent 契約の変更（ADR 0015 §2 の改訂） + +3 種のうち `prompt.md`（毎判断 LLM）を廃止し、自己改善型を加える。`decide` / `run(ctx)` は不変。 + +| 中身 | 種別 | 動き方 | +|------|------|--------| +| `agent.ts`（`decide` export） | ルール型 | 現行どおり。毎ブロック | +| `agent.ts`（`run(ctx)` export） | 自走型 | 現行どおり | +| `agent.ts` + **`improve.md`** | **自己改善型** | `decide` を毎ブロック実行しつつ、improve.md の指示に従って LLM が `decide` 相当のコードを書き換える | + +**`improve.md` は `prompt.md` の改名ではない。**prompt.md は「この observation でどう動くか」を +書く判断プロンプトで、improve.md は「どういうときに、何を根拠に、戦略をどう直すか」を書く +メタプロンプトである。意味が入れ替わるので、同梱 19 agent の prompt.md は転用せず整理する。 + +### 2. 書き換えの対象と実行 + +LLM は次を返す（削除済み `src/llm` の `Strategy` を復元・簡素化したもの）: + +```ts +type StrategyRevision = { + version: number; + notes: string; // なぜこう変えたか。run 後の診断の一次情報 + executorTs: string; // decide(obs, ctx) 相当の本体。node:vm で実行 +}; +``` + +- executor は**ルール型の `decide` と同じ契約**を満たす。したがって取引ループは既存の decide + ループそのままで、差し替わるのは関数だけになる +- **生成コードは `check:strategy`（cheatcode 静的検査）を通してから採用する。**通らなければ + 採用せず前版を維持し、理由を記録する。これは提出時の入口ゲートが生成コードを素通りするのを塞ぐ +- コンパイル/実行時エラーも同様に「採用しない」で扱う。壊れたコードで run が止まってはならない + +### 3. 参加者の提出物 + +`agent.ts`（初期戦略）+ `improve.md`（改善プロンプト）。block 0 から初期戦略が全速で動くので +コールドスタートが無く、**「戦略設計」と「改善の導き方」の両方**が測られる。 + +### 4. 改訂の起動条件は改善プロンプトが決める + +**ただし運営側は上限を持つ。**「プロンプトが決める」を素直に実装すると、起動を判断するために +毎ブロック LLM に問い合わせることになり、コストを制御できない。また同居構成では、1 参加者が +「毎ブロック改訂」と宣言するだけで LLM 予算を食い潰し、他の参加者の run を巻き添えにする。 +したがって次の 2 層にする: + +- **参加者が `improve.md` の frontmatter で宣言する**（`reviseEvery` 等）。ここが「プロンプトが + 決める」の実体で、LLM 呼び出しを伴わないので予測可能 +- **LLM は「変更しない」を返せる。**呼ばれた側が「今は触らない」と判断する自由があるので、 + 「勝っているときに触らない」を improve.md に書けば実現できる。先行実装の失敗診断 + （好機レジームで upside を削る）への直接の対処がここに入る +- **運営が 1 run / 1 参加者あたりの呼び出し回数に上限を課す。**宣言値が上限を超える場合は + 上限側で丸め、丸めたことを記録する + +### 5. 巻き戻しもプロンプトが決める（自動 rollback は置かない） + +**「悪化したら自動で戻す」は実装せず、戻すかどうかをモデルの判断にする。**改訂の返り値に +`revertTo: ` を設け、版履歴（各版の notes・投入ブロック・その時点の価値）を文脈で渡す。 + +自動化しない理由は、**妥当な閾値が存在しないから**である。先行実装の基準は緩すぎて 18 run 中 +0 件しか発火しなかった。逆に「少しでも負けたら戻す」にすると、全員が負けるレジーム（crash / +lending-incident）では改訂の質に関係なく毎回巻き戻り、run ノイズだけでも符号は頻繁に反転する。 +**目減りが戦略のせいか相場のせいかは判断であり、定数では表せない。**判断であるなら、それを +どう下すかを書く場所は参加者の improve.md である（§4 と同じ理屈）。 + +タイミングは劣化しない: 自動版も判定は次の改訂機会だったので、モデルに委ねてもレイテンシは同じ。 +判断の余地がないもの（cheatcode / コンパイル失敗 / 実行タイムアウト）はハーネスが弾き続ける。 + +失うものは明示しておく: **improve.md が下手な参加者は、悪い書き換えが run の最後まで残る。** +これは「改善の導き方」を測る競技である以上まっとうな結果であり、frozen 対照で可視化される。 + +先行実装の失敗を繰り返さないために、次を満たす: + +- 改訂の採否・`revertTo` の発火・その前後の成績を agent ログに必ず残す +- **frozen 対照をロスターに常置する**（同じ初期戦略を改訂なしで走らせる agent）。自己改善が + frozen に勝てているかが毎 run 見える状態にする +- `revertTo` の発火率を run の集計に出す。0% が続くなら improve.md が巻き戻しを指示していないか、 + モデルがその選択肢に気づいていない + +### 6. 廃止の影響範囲 + +prompt 型は「LLM を使う唯一の道」として据えられているため、撤去は広く波及する: + +- `example/agents/runtime/bot.ts` の `runPromptLoop` と `ERIS_AGENT_MODE: "prompt"`、 + `ERIS_PROMPT_REVISE_EVERY` / `ERIS_PROMPT_REVISE_PERSIST` / `ERIS_PROMPT_LOG_CALLS` +- 同梱 19 agent の `prompt.md` +- **`config/example.yaml` の Quick Start 既定ロスター**（現在 prompt モード = zero-config で + 最初に動く経路）。自己改善型へ差し替えるか、ルール型へ戻すかを決める必要がある +- `docs/guide/llm-agents.md` / `writing-agents.md` / `backtest.md`、CLAUDE.md +- ADR 0015 §2 の契約表、ADR 0017 §6（prompt 型の不利に関する記述は本 ADR の実装で解消する） + +## 決めていないこと + +| 項目 | 決めない理由 | いつ決めるか | +|------|------------|------------| +| 呼び出し回数の上限値と `reviseEvery` の既定値 | 同居人数 × 改訂頻度 × LLM レイテンシで決まる運用コストの問題で、実測なしには決まらない | 実装後の較正 run | +| 生成 executor に渡すヘルパの範囲（どの sdk API を露出するか） | 表現力と安全性のトレードオフで、初期戦略が実際に何を使うかを見てから決めるのが速い | 実装時 | +| 同梱 19 agent の `prompt.md` をどうするか（削除 / 参考として `improve.md` へ書き直し） | 移行作業であり設計判断ではない | 実装時 | +| ADR 0017 §5 の `blockTimeSec` 較正 | 本 ADR が行動回数の差を消すので、`blockTimeSec` を上げる動機が消える | 本 ADR 実装後 | + +## Notes + +### 実装フェーズと、他の作業との順序 + +本 ADR は ADR 0017 §5 の較正より**先**に来る。0018 は LLM 型の送信 tx を 3 件から ~192 件へ +変える（B1 実測比）ので、同居上限も識別力も現行ロスターで測った値は実装と同時に無効になる。 + +1. **Phase 1**: runtime の自己改善モード（`node:vm` executor + 改訂ループ + rollback + + 生成コードへの `check:strategy`）と、prompt モードの撤去 +2. **Phase 2**: 新契約のリファレンス agent 整備。**issue #54（同梱 arb 群が USDC-only 配布で + 一度も取引しない）はここで解消する** — 同梱 agent は「自己改善型の初期戦略」として + 書き直す対象なので、先に #54 だけ直すと同じファイルを 2 度触ることになる +3. **Phase 3**: 本 ADR の未決事項の較正（rollback 判定基準、呼び出し回数上限）。実装が動いて + 初めて測れる +4. **その後**: ADR 0017 §5 の B2 再測定と識別力検証 + +### 参考資料 + +- ADR 0015 §2 — agent 契約（decide / run(ctx) / prompt.md）。本 ADR はこれを改訂する +- ADR 0017 §5 B1 / §6 — prompt 型の判断頻度の実測と、対処 2 案の提示 +- 削除済み `src/llm`（commit `319cf06` 等）— `Strategy.executorTs` + `node:vm` の先行実装 +- 先行実装の検証結果 — multi-seed で frozen に敗北、rollback 不発、原因は upside の削り diff --git a/docs/guide/architecture.md b/docs/guide/architecture.md index 5cf1616..e1852a7 100644 --- a/docs/guide/architecture.md +++ b/docs/guide/architecture.md @@ -61,7 +61,7 @@ Drop exactly one of the following into `example/agents//` and add the id to |---|---|---| | `agent.ts` (exports `decide(obs, ctx)`) | rule strategy | bot.ts drives a read→decide→send loop (interval can be set via `export const config = { intervalMs }`) | | `agent.ts` (exports `run(ctx)`) | self-driven | bot.ts does not loop; it delegates by passing ctx (clients / latestObservation / onObservation / submit / log) (e.g. liquidator) | -| `prompt.md` (frontmatter: name/description required) | prompt type | bot.ts attaches the observation and has the LLM emit an action on every decision ([LLM agents](llm-agents.md)) | +| `agent.ts` + `improve.md` (frontmatter: name/description required) | self-improving | decide() drives every block as usual, and an LLM periodically rewrites the strategy out of the trade path ([Self-improving agents](llm-agents.md)) | runtime/send.ts appends mempool activity (`kind:"mempool"`: submitted / submit_failed / rejected) to `runs//agents/.jsonl` as a self-report (closing the gap where the coordinator can no longer count submissions). diff --git a/docs/guide/backtest.md b/docs/guide/backtest.md index a8aa967..e544d56 100644 --- a/docs/guide/backtest.md +++ b/docs/guide/backtest.md @@ -1,24 +1,28 @@ [← README](../../README.md) -# Backtest (regime replay, ADR 0016) +# Backtest (scenario replay, ADR 0016 / ADR 0017) -A mode for participants to validate their own strategy "cheaply and repeatedly, under conditions equivalent to historical market data." On top of a **dedicated local anvil** loaded with the distributed venue state dump, the existing coordinator replays an official regime (`config/regimes/*.yaml` + seed) as-is. Fills are computed by the real contracts (no fill model), and scoring is fully identical to realtime (the `summary.json` format is the same except for `mode: "backtest"`). No fork and no external RPC required. +A mode for participants to validate their own strategy "cheaply and repeatedly, under conditions equivalent to historical market data." On top of a **dedicated local anvil** loaded with the distributed venue state dump, the existing coordinator replays a scenario as-is. Fills are computed by the real contracts (no fill model), and scoring is fully identical to realtime (the `summary.json` format is the same except for `mode: "backtest"`). No fork and no external RPC required. + +A **scenario is `(regime, seed)`**, written `#`. The regime YAML holds the market conditions; the seed picks which realization out of that family you get. Both have to be supplied — a regime alone is not runnable (ADR 0017 §1). ```bash -npm run backtest -- --regime calm-01 # normal market -npm run backtest -- --regime crash-01 # crash + victim + Aave liquidation -npm run backtest -- --regime calm-01 --repeat 5 # 5× the same regime (see the distribution) -npm run backtest -- --regime calm-01 --agents my-roster.yaml # swap the roster +npm run backtest -- --regime calm --seed 101 # one scenario: calm#101 +npm run backtest -- --regime lending-incident --seed 202 # collateral crash + Aave liquidations +npm run backtest -- --regime calm --seed 101 --repeat 5 # same scenario 5x (see the spread) +npm run backtest -- --regime calm --seed 101 --agents my-roster.yaml # swap the roster + +npm run backtest -- --scenarios config/scenarios/public.yaml # the whole public set + standings ``` ```mermaid flowchart LR DEP["deployer anvil :8545
all venues deployed"] -->|"npm run gen:state-dump
(revert to the .local-snapshot clean cross-section)"| STATE["backtest/state/
venues-state.json + manifest
(source commit · genesis hash · deployments · fingerprint)"] - REGIME["config/regimes/<name>.yaml + seed"] --> RUN - STATE -->|"npm run backtest -- --regime <name>"| BT[("dedicated anvil :8547
--load-state")] - BT --> RUN["coordinator replays the regime
(scoring identical to realtime)"] - RUN -->|"--repeat N: evm_snapshot / evm_revert"| BT - RUN --> OUT["runs/<id>/ summary.json
mean alphaUsdc over the repeats"] + REGIME["config/regimes/<name>.yaml
+ seed (per scenario)"] --> RUN + STATE -->|"npm run backtest -- --regime <name> --seed <N>"| BT[("dedicated anvil :8547
--load-state")] + BT --> RUN["coordinator replays the scenario
(scoring identical to realtime)"] + RUN -->|"evm_snapshot / evm_revert between scenarios"| BT + RUN --> OUT["runs/<id>/ summary.json
runs/matrix-<id>/ matrix.json + standings.json"] ``` ## Prerequisites @@ -32,19 +36,93 @@ flowchart LR ## Regime = a label for market conditions -A regime is **a YAML in the existing config schema + a seed** (same format as [Configuration](configuration.md)). The fair-price OU path, flow orders, and stress event schedule are all deterministically replayed from the seed. +A regime is **a YAML in the existing config schema** (same format as [Configuration](configuration.md)) describing a family of market conditions: the fair-price OU parameters, flow intensity, and the stress event ranges. It carries no seed. Given one, the fair-price path, flow orders, and stress event schedule all replay deterministically. + +- `config/regimes/calm.yaml` — normal market (no stress) +- `config/regimes/crash.yaml` — trapezoidal crash, no victims (price gap only; the liquidity-withdrawal half is pending [#52](https://github.com/NyxFoundation/eris-agent-simulator/issues/52)) +- `config/regimes/lending-incident.yaml` — the same crash plus 2 liquidation-target victims and a liquidator roster slot +- `config/regimes/lst.yaml` — the liquid staking venue (not part of the competition set) + +`blockTimeSec` and `blocks` are part of the regime (fixed to the same values as production). Short-circuit overrides like `--blocks` / `--seconds` are for behavior checks and smoke tests only; **runs whose scores you read should use the regime defaults** (ADR 0016 §3). + +## Scenario sets and standings + +`--scenarios ` replays a whole set on one anvil and ranks the roster. + +```yaml +# config/scenarios/public.yaml — the cartesian product of the two lists +regimes: [calm, lending-incident, crash] +seeds: [101, 202, 303, 404, 505] +``` + +The published (public) set exists for tuning. The competition's private set is the same regimes with a disjoint, unpublished seed list — same distribution family, different realizations. **Because the generator is open source, sample your own seeds rather than overfitting to the published five**: generalizing across the distribution is what is being measured. -- `config/regimes/calm-01.yaml` — normal market (no stress) -- `config/regimes/crash-01.yaml` — trapezoidal crash (range-specified) + 2 liquidation-target victims + a liquidator roster slot +
+Operators: building the private set -`blockTimeSec` is part of the regime (fixed to the same value as production). Short-circuit overrides like `--blocks` / `--seconds` are for behavior checks and smoke tests only; **runs whose scores you read should use the regime defaults** (ADR 0016 §3). The seed of a public regime is just one sample from the range, and a production run's seed is a different sample (anti-overfitting). +`config/scenarios/private.yaml` is deliberately not in the repository — the regimes are public and +only the realized seeds are withheld, so the file *is* the secret. Build it the same shape as +`public.yaml`, and keep it out of the working tree (it is covered by the same ignore rules as any +untracked file; do not `git add` it). + +1. Draw the seeds from a wide range with a source the repository does not contain, and check them + against `public.yaml` — the two sets must not intersect, or a participant has already tuned on + part of the private set. +2. Use the same `regimes:` list as the public set. Different regimes would compare participants on + conditions they were never told to prepare for. +3. Keep 20 seeds per regime as the baseline; expand only after the heat count is known, since heats + multiply the run count (ADR 0017 §3). +4. Publish the seed list after the competition so participants can reproduce their own results. + +Nothing else needs to be secret. The stress ranges, the OU parameters and the flow calibration are +all published on purpose — withholding them would measure who guessed the environment rather than +who traded it well. + +
+ +Two artifacts land in `runs/matrix-/`: + +| file | what it is | +|---|---| +| `matrix.json` | raw per-scenario, per-agent scores — **both** `netPnlUsdc` and `alphaUsdc`, plus disqualifications and run directories | +| `standings.json` | the ranking derived from them | + +The ranking is a derived view on purpose. The scoring rule is expected to change (ADR 0017 leaves both the metric and the formula open), and keeping the raw matrix means a new rule can be applied to a finished competition without re-running anything. + +Ranking today: score each scenario with `--metric` (default `netPnlUsdc`), normalize to a z-score **across the agents within that scenario** — they all ran in the same world, so that is the one comparison the design guarantees is fair — then average within a regime and average the regimes with equal weight. Equal weight per regime is what stops a big-opportunity regime like `crash` from deciding the ranking on its own, and it makes the result insensitive to how many seeds each regime got. + +An agent that broke a rule (priority fee cap), whose process died mid-run, or that never reported is **disqualified for that scenario and placed below every finisher** — scoring it zero would make crashing a viable tactic. A scenario that produced no result at all is excluded from the aggregation entirely: an environment failure is not charged to the participants. ## Repetition and reproducibility -- **The environment (initial state + market conditions) is perfectly identical every time**: each launch builds a fresh anvil from the same state dump, and between the runs of `--repeat N` it returns to the clean cross-section via `evm_snapshot`/`evm_revert` (no victim leftovers). -- **The only thing that varies is tx ordering** (same property as production realtime, ADR 0005). Results converge into a narrow band but are not bit-identical. Read the ranking from the mean alphaUsdc over `--repeat N`. +- **The environment (initial state + market conditions) is perfectly identical every time**: each launch builds a fresh anvil from the same state dump, and between scenarios (and between the runs of `--repeat N`) it returns to the clean cross-section via `evm_snapshot`/`evm_revert` (no victim leftovers). +- **The only thing that varies is tx ordering** (same property as production realtime, ADR 0005). Results converge into a narrow band but are not bit-identical. +- **`--repeat` is a diagnostic, not part of scoring.** Repeating a scenario reduces the tx-timing noise but does nothing about the variation between market conditions, and since the wall-clock budget is `scenarios x repeats`, spending it on repeats is the same as halving the number of scenarios. The competition runs each scenario once and buys precision with more seeds instead (ADR 0017 §3). Use `--repeat` when you want to *measure* how much a scenario moves run to run; standings fold the repeats with a median. - Bit-identical regression comparison (diff-checking a single line of code) is planned but unimplemented as B2 synchronous-step replay (ADR 0016 §3). +## What the competition actually scores (ADR 0017 §6) + +Every participant runs in the **same world at the same time** — one scenario is one run with the +whole field co-located on one chain. That is not a convenience: the score is a z-score computed +across the agents *within a scenario*, so "who did better" only means anything because everybody met +the same market on the same blocks. It also means three things are part of the competition whether +you engage with them or not: + +- **Opportunities are finite and shared.** A liquidation or an arbitrage gap is taken by whoever + gets there first; it does not exist separately for each participant. In pilot runs, arbitrage + profit per agent fell by roughly 30x going from 4 co-located agents to 16. +- **In-block ordering is bought with gas.** anvil orders the block by priority fee (`--order fees`), + so when two agents chase the same gap in the same block, the higher bid executes first + (ADR 0011). Bidding is a real lever, and over-bidding is a real cost. +- **Speed counts.** Blocks are mined every `blockTimeSec` seconds regardless of whether your agent + has finished thinking. A rule agent (`decide`) runs once per block; a prompt agent waits for an + LLM round trip (~10 s), so at the production 2-second block it acts roughly once every five + blocks and holds its position in between. This is a structural difference between the two agent + kinds, not a tuning detail — pick the kind that suits your strategy knowingly. + +Scoring itself is unaffected by any of this: it reads the same value cross-sections for everyone at +the same blocks. + ## Sparring (compete against other agents) Line up multiple agents in the roster and they compete in the same run, on the same mempool. `--agents` swaps the regime's default roster (YAML/JSON; the content is baked into the effective regime): @@ -76,13 +154,17 @@ agents: | flag | description | |---|---| -| `--regime ` | `config/regimes/.yaml` (or a YAML path). Required | +| `--regime ` | `config/regimes/.yaml` (or a YAML path). Requires `--seed` | +| `--seed ` | The scenario's seed. Regimes carry none, so this is not optional | +| `--scenarios ` | Replay a whole set (regimes x seeds) and write `matrix.json` + `standings.json`. Mutually exclusive with `--regime` | +| `--metric ` | Metric the standings rank on: `netPnlUsdc` (default) or `alphaUsdc` | | `--agents ` | Swap the regime's default agents with a roster file (YAML/JSON) | -| `--repeat ` | Repeat the same regime N times (default 1). Prints the mean alphaUsdc when done | +| `--repeat ` | Repeat each scenario N times (default 1). A calibration diagnostic; standings take the median | | `--port ` | Port for the backtest-dedicated anvil (default 8547; use a different port for parallel runs) | | `--state ` | State dump directory (default `backtest/state`) | | `--keep-anvil` | Keep anvil alive after exit (for reading receipts in post-hoc analysis / debugging) | -| `--seed` / `--blocks` / `--seconds` / `--protocols` / `--economic-gas` | One-shot override of regime values (for smoke tests) | +| `--score-every ` | Reconstruct the value cross-section every Nth block instead of every block. Score-neutral (only the first and last cross-sections reach `summary.json`); it just coarsens the equity curve in `events.jsonl` | +| `--blocks` / `--seconds` / `--protocols` / `--economic-gas` | One-shot override of regime values (for smoke tests) | > Run overrides are written out as an "effective regime YAML" that both the coordinator and the agent processes read, so they read the same settings (applying it only to the coordinator would kill the agents on observation). diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 025b319..a67e931 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -56,7 +56,7 @@ agents: | `dir` | | Override for the real directory (when lining up multiple instances of the same strategy under different ids) | | `baseline` | | `true` treats it as a zero-skill baseline (noop / random) | | `description` | | Human-readable description | -| `env` | | Strategy parameters passed to the agent process (`ERIS_AGENT_MODE` / `ERIS_LLM_*` etc.; distinct from the sim config keys) | +| `env` | | Strategy parameters passed to the agent process (`ERIS_LLM_MODEL` / `ERIS_AGENT_FROZEN` / `ERIS_IMPROVE_LOG_CALLS` etc.; distinct from the sim config keys) | | `command` / `args` | | Override for a fully custom agent (other languages etc.; read/send/validate all self-provided = unsupported). Normally omitted | ## One-shot CLI overrides (sim:realtime) diff --git a/docs/guide/llm-agents.md b/docs/guide/llm-agents.md index 850ba3d..290f90b 100644 --- a/docs/guide/llm-agents.md +++ b/docs/guide/llm-agents.md @@ -1,51 +1,101 @@ [← README](../../README.md) -# LLM-driven autonomous agents (prompt.md type) +# Self-improving agents (agent.ts + improve.md) -Drop **a single `prompt.md`** into `example/agents//` and that agent becomes a prompt type: on every decision cycle, `runtime/bot.ts` calls the LLM with the observation attached and has it emit a JSON action. There is no hand-written trading logic — **the prompt.md is the strategy itself** (the submission). The bundled sample is `example/agents/my-arb/prompt.md`. +An LLM in this simulator **rewrites the strategy; it does not make the trades**. Put an `improve.md` +beside your `agent.ts` and the agent becomes self-improving: `decide()` runs every block exactly as +fast as any rule agent, and periodically the model is handed the current strategy source plus how it +has been doing, and may return a replacement. ```markdown --- name: my-arb # required -description: cross-venue arb; push toward fair above 30bps # required -intervalMs: 5000 # decision cycle interval (optional) -model: gpt-oss:120b # model to use (optional; "claude..." = Anthropic API, "codex[:m]" / "claude-cli[:m]" = subscription CLIs) +description: cross-venue arb that widens its margin under adverse selection # required +reviseEveryBlocks: 60 # blocks between revision opportunities (optional; default 60) +model: gpt-oss:120b # optional ("claude..." = Anthropic API, "codex[:m]" / "claude-cli[:m]" = subscription CLIs) --- -# Mission -(the strategy in natural language: how to read the observation, order conditions, sizing, risk constraints) +# When to change the strategy, on what evidence, and what to change + +(not "what should I trade this block" — see below) ``` -## How it runs (runtime/bot.ts + runtime/llm.ts) +> **Prompt mode was removed (ADR 0018).** Until recently an agent could be a `prompt.md` that the LLM +> consulted for *every action*. Measured at production settings, that managed one decision every +> 8-28 blocks and **1/64 the actions** of the same strategy in rule mode — it could not compete. +> `improve.md` is not a renamed `prompt.md`: the old file answered "given this observation, what do +> you do", the new one answers "when, on what evidence, and how should the strategy change". + +## How it runs ```mermaid flowchart TB - OBS["observation (JSON) + <schema>
generated from sdk/src/actionSchema.ts"] --> CALL["LLM call
(one per decision cycle)"] - CALL --> V{"zod validate"} - V -->|"ok"| ACT["action → sign & send"] - V -->|"fail"| FB["append the validation error
to the conversation"] - FB -->|"retry ≤ limit"| CALL - FB -->|"limit exceeded"| NOOP["noop (fail-closed)"] - REV["every N cycles (ERIS_PROMPT_REVISE_EVERY):
LLM revises the prompt body →
agents/<id>.prompt.v<K>.md"] -.-> CALL + subgraph loop["trading loop — every block, no LLM"] + OBS["observation"] --> DEC["decide(obs, ctx)"] --> TX["sign & send"] + end + subgraph rev["revision — every reviseEveryBlocks"] + CTX["current source + recent decisions + PnL"] --> CALL["LLM call"] + CALL --> P{"parse"} + P -->|"executorTs: null"| KEEP["keep the current strategy"] + P -->|"code"| CHK{"cheatcode check
+ compile"} + CHK -->|"fail"| REJ["reject, keep the current strategy"] + CHK -->|"ok"| INST["install"] + P -->|"revertTo: n"| BACK["reinstall version n"] + end + INST -.->|"swaps the function
the loop is calling"| DEC + BACK -.-> DEC +``` + +The model returns one JSON object: + +```json +{ "notes": "why", "executorTs": "" } // install this +{ "notes": "why", "executorTs": null } // leave it alone +{ "notes": "why", "revertTo": 1 } // go back to an earlier version ``` -- Each cycle, bot.ts puts the observation (JSON) and the action's **``** (generated from the zod schema in `sdk/src/actionSchema.ts`) into the system prompt and calls the LLM once. -- The response is validated with zod, and **on failure the error is appended to the conversation and retried** (on exceeding the retry limit that cycle is `noop` = fail-closed). -- The decisions and actions are recorded in `runs//agents/.jsonl` ([Run output and analysis](run-output.md)). -- When agent.ts and prompt.md are **co-located**, the runtime default is agent.ts (rule strategy). Switch to prompt.md driving with the roster's `env: { ERIS_AGENT_MODE: "prompt" }`. -- Note that the **shipped `config/example.yaml` roster opts the trading agents into prompt mode** (the Quick Start default is LLM-driven; it needs an LLM endpoint — see "Backends" below). Remove the `env:` line from an agent to run it rule-based. +`"executorTs": null` means **keep the current strategy**, and that is often the right answer: a +strategy that is working does not need help. -## Self-revision (optional) +**Nothing reverts automatically.** Undoing a change is the model's call, made with `revertTo` and the +version number — the context it receives lists every version, when it went in, and what the agent +was worth at the time. An automatic "revert when value went down" would need a threshold and there +is no defensible one: the previous implementation's never fired in 18 runs, and the obvious opposite +(any loss at all) reverts every revision in a regime where everyone is losing. Whether a dip is the +strategy or the market is a judgment, so `improve.md` is where you state how to make it. -With `ERIS_PROMPT_REVISE_EVERY=`, the LLM **self-revises the prompt body** every N decision cycles (default 0 = off). The revised version is saved with a version tag at `runs//agents/.prompt.v.md` and used in subsequent cycles. With `ERIS_PROMPT_REVISE_PERSIST=1` it is also written back to the prompt.md in the agent directory. +## What the generated code may do -## LLM conversation log (optional; for debugging prompt tuning) +The body runs in a `node:vm` context with `obs` and `ctx` in scope and nothing ambient — no +`require`, no `import`, no `process`, no `fetch`. It has the same trading capability as your +hand-written strategy (it is handed the same `ctx`), and the same prohibitions: **generated code is +run through the cheatcode static check before it is installed**, so `anvil_*` / `evm_*` / +`hardhat_*` and the privileged chain helpers are refused exactly as they are in a submission. -With `ERIS_PROMPT_LOG_CALLS=1`, the **raw conversation** with the LLM is recorded in `runs//agents/.llm.jsonl`: +Code that fails the check, fails to compile, or does not return within 2 seconds is not installed — +the previous strategy keeps running and the reason is logged. -- `kind: "llm_system"` — the full system prompt (only on the first call and right after each self-revision; the version is identified by `revision`) -- `kind: "llm_call"` — a record of every call: `purpose` (decision / revise), `round`, `attempt`, the sent `messages` (including the observation and the retry feedback on validate failures), and the raw `response` (or `error` if the call failed) +## Guards -Because you can trace "where in the observation the LLM misread" and "how many times it failed validate" per decision, use this log as the primary source for the prompt-improvement cycle. Keep it off for normal runs (the log grows by a few KB per decision). +| guard | why | +|---|---| +| cheatcode static check on generated code | an LLM-authored strategy is not trusted code, and the submission gate cannot see code that does not exist yet | +| compile / call failure is never installed | a broken rewrite must not stop the agent trading | +| `revertTo` in the model's hands, not a threshold | whether a dip is the strategy or the market is a judgment; a fixed rule is either never right or always wrong (§5) | +| revision cadence clamped | a co-located run shares one LLM budget; "revise every block" from one agent would starve the field | +| every outcome logged | the previous attempt at self-improvement shipped a rollback that never once fired and nobody noticed | + +**Always run a frozen control.** `ERIS_AGENT_FROZEN: "1"` runs the same directory with the +improvement loop off. Without it you cannot tell whether revising helped or whether the strategy was +going to do that anyway. + +## Logs + +Revision outcomes (installed / declined / rejected / reverted, with the model's notes) land in +`runs//agents/.jsonl` alongside the trading decisions. + +`ERIS_IMPROVE_LOG_CALLS: "1"` additionally writes the raw exchange — the system prompt, the context +that was sent, and the response — to `runs//agents/.llm.jsonl`. Off by default because +it holds every generated strategy in full. It is the log to turn on when tuning `improve.md`. ## Backends (runtime/llm.ts) @@ -58,46 +108,59 @@ The provider is selected by the frontmatter `model` name: | `codex` / `codex:` | Codex CLI (spawns `codex exec` in a read-only sandbox) | ChatGPT subscription (`codex login`; **no API key**) | | `claude-cli` / `claude-cli:` | Claude Code CLI (spawns `claude -p` with all built-in tools disallowed) | Claude subscription (Claude Code OAuth login; **no API key**) | -The per-call timeout is `ERIS_LLM_CALL_TIMEOUT_MS` (default 60000; the CLI providers default to 120000 because each call pays process startup). Put the secret API keys in `.env.local` ([Configuration](configuration.md)). +The per-call timeout is `ERIS_LLM_CALL_TIMEOUT_MS` (default 60000; the CLI providers default to +120000 because each call pays process startup). Put the secret API keys in `.env.local` +([Configuration](configuration.md)). + +**Latency no longer bounds the strategy.** Under prompt mode a slow backend meant a slow trader; now +it only means fewer revision opportunities, and the strategy trades at full speed throughout. A +backend failure is recorded and the strategy continues unchanged, so a run without an API key still +completes — you just get no revisions. ## Running on a Codex / Claude Code subscription (no API key) -If you have a ChatGPT (Codex) or Claude (Claude Code) subscription, prompt agents can run on it directly — set the frontmatter `model` (or the roster env `ERIS_LLM_MODEL`) to a CLI provider and make sure the CLI is logged in on the machine: +Set the frontmatter `model` (or the roster env `ERIS_LLM_MODEL`) to a CLI provider and make sure the +CLI is logged in on the machine: ```markdown --- name: my-arb description: cross-venue arb model: claude-cli:haiku # or "codex" (empty model = the CLI's own configured default) -intervalMs: 15000 # CLI calls are slower than HTTP; widen the decision cycle --- ``` Notes: -- **Latency**: one decision costs a CLI process spawn + a subscription model call (measured: `claude-cli:haiku` ~6s, `codex` default model ~12s). Set `intervalMs` accordingly; cycles are locked so a slow call never overlaps the next one. -- **Quota**: every decision cycle consumes subscription quota (a 100-block run ≈ 20-60 calls per agent depending on `intervalMs`). Keep rosters small; codex and claude draw on separate pools, so mixing providers raises the parallel ceiling. -- **Auth isolation**: the `claude-cli` provider strips `ANTHROPIC_API_KEY` from the spawned CLI's env so the call always bills the subscription OAuth login, and strips the enclosing Claude Code session markers so it can be launched from inside a Claude Code session without the CLI's nested-session hang. +- **Quota**: each revision is one call, capped per run, so a self-improving agent costs a handful of + calls per run rather than one per decision. Codex and Claude draw on separate pools, so mixing + providers raises the parallel ceiling. +- **Auth isolation**: the `claude-cli` provider strips `ANTHROPIC_API_KEY` from the spawned CLI's env + so the call always bills the subscription OAuth login, and strips the enclosing Claude Code session + markers so it can be launched from inside a Claude Code session without the CLI's nested-session hang. - **Binary override**: `ERIS_CLAUDE_BIN` / `ERIS_CODEX_BIN` point at a non-PATH binary if needed. -- The JSON contract is unchanged: the CLI's output goes through the same zod validation + retry loop, and any provider failure fails closed to `noop`. ## Run example ```yaml # roster in config/local.yaml agents: - - id: my-arb # example/agents/my-arb/ (prompt.md only → prompt type) + - id: venue-arb # example/agents/venue-arb/ (agent.ts + improve.md) wallet: AGENT1_PRIVATE_KEY - - id: venue-arb # to run an agent.ts co-located agent via prompt.md + env: { ERIS_LLM_MODEL: "claude-cli", ERIS_IMPROVE_LOG_CALLS: "1" } + - id: venue-arb-frozen # the control: same strategy, no improvement loop + dir: venue-arb wallet: AGENT2_PRIVATE_KEY - env: - ERIS_AGENT_MODE: "prompt" - ERIS_PROMPT_REVISE_EVERY: "10" # self-revise the prompt every 10 cycles + env: { ERIS_AGENT_FROZEN: "1" } ``` ```bash set -a; source .env.local; set +a # only secrets like OLLAMA_API_KEY -npm run sim:realtime # or npm run backtest -- --regime calm-01 +npm run sim:realtime # or npm run backtest -- --regime calm --seed 101 ``` -> The prompt type is bottlenecked by LLM latency on top of the wall-clock wait for block time. The LLM calls remain even in a backtest ([Backtest](backtest.md)). +A measured example of what this looks like: over 150 blocks the model noticed the strategy had been +emitting the same "cannot fund this side of the gap" reason for a dozen blocks, rewrote it, and then +**declined** to touch it again at the next opportunity because it had started working. That agent +finished at +57.6 against its frozen control's +10.0. One run and one seed — an existence proof that +the loop works, not evidence that self-improvement wins. diff --git a/docs/guide/repository-layout.md b/docs/guide/repository-layout.md index 3a3f57b..7752bdf 100644 --- a/docs/guide/repository-layout.md +++ b/docs/guide/repository-layout.md @@ -21,7 +21,7 @@ core/src/ environment daemon + scoring (participants do not touch this example/agents/ participant template (1 agent = 1 directory is the unit of copy and submission) runtime/ generic driver scripts (bot / read / send / llm / prompt / agentLog; reserved names) lib/ shared strategy helpers (markets.ts, etc.; reserved name) - / the agent itself (agent.ts's decide/run, or prompt.md) + / the agent itself (agent.ts's decide/run, plus improve.md to self-improve) contracts/ PriceFeed + mock oracles + FlashArb (Foundry) deployer/ bundled deploy orchestrator (self-contained subpackage that deploys all 5 venues to an empty anvil) config/ YAML config (example.yaml = template / vuln-test.yaml / regimes/ = official regimes; ADR 0016) diff --git a/docs/guide/run-output.md b/docs/guide/run-output.md index 2a3c4df..47dad75 100644 --- a/docs/guide/run-output.md +++ b/docs/guide/run-output.md @@ -10,15 +10,14 @@ Each run generates a `runs//` directory. The dedicated evaluation, scori | `events.jsonl` | event stream (observation, stress, liquidation, etc.); the primary source for scoring | | `blocks.csv` | per-block tx records (fee comes from the on-chain tx field) | | `agents/.jsonl` | each agent's self-reported log (decision `reason` / `signals` / `state`, plus mempool activity appended by runtime/send.ts as `kind:"mempool"`: submitted / submit_failed / rejected) | -| `agents/.prompt.v.md` | prompt-agent self-revision history (when `ERIS_PROMPT_REVISE_EVERY` is enabled; full text, versioned) | -| `agents/.llm.jsonl` | prompt-agent LLM conversation log (opt-in via `ERIS_PROMPT_LOG_CALLS=1`; full system prompt, sent messages, raw responses, errors; see [LLM Agents](llm-agents.md)) | +| `agents/.llm.jsonl` | raw strategy-revision exchange for a self-improving agent (opt-in via `ERIS_IMPROVE_LOG_CALLS=1`; system prompt, sent context, response, errors; see [Self-improving agents](llm-agents.md)). Revision *outcomes* are in `agents/.jsonl` | ```bash npm run check:ordering -- runs/ # inspect Anvil's fee ordering npm run check:strategy -- # static cheatcode check of strategy code (entry side) ``` -> The entry points for a run are `sim:realtime` and `backtest` (identical output format, with `summary.json`'s `mode` being `"realtime"` / `"backtest"`). **SEED (= regime) is a label for the market conditions** — the price path is reproducible, but tx timing/ordering is non-deterministic, so results vary even within the same regime. When you need to compare runs, accumulate samples and aggregate — [Backtest](backtest.md)'s `--repeat N` handles the iteration and the display of mean alphaUsdc for you. +> The entry points for a run are `sim:realtime` and `backtest` (identical output format, with `summary.json`'s `mode` being `"realtime"` / `"backtest"`). **SEED (= regime) is a label for the market conditions** — the price path is reproducible, but tx timing/ordering is non-deterministic, so results vary even within the same regime. When you need to compare runs, accumulate samples and aggregate — [Backtest](backtest.md)'s `--repeat N` runs the same scenario N times and prints each run so you can see the spread, and `--scenarios` ranks a whole set at once. ## Key fields in summary.json diff --git a/docs/guide/stress-events.md b/docs/guide/stress-events.md index b2a5792..4cb2106 100644 --- a/docs/guide/stress-events.md +++ b/docs/guide/stress-events.md @@ -11,8 +11,8 @@ stress: victimCount: 0 # >0 builds liquidatable victims (fresh state required; see below) ``` -> To try it quickly, the fastest path is to run the official regime `config/regimes/crash-01.yaml` -> (which includes 2 victims + a liquidator roster) via [Backtest](backtest.md): `npm run backtest -- --regime crash-01`. +> To try it quickly, the fastest path is to run the official regime `config/regimes/lending-incident.yaml` +> (which includes 2 victims + a liquidator roster) via [Backtest](backtest.md): `npm run backtest -- --regime lending-incident --seed 202`. The event overlay is a trapezoid — the effective price ramps away from the base price, holds, then decays back (a crash with magnitude 0.14, `rampBlocks: 3`, `holdBlocks: 6`, `decayBlocks: 8` looks like this; β≈0 outside the window): diff --git a/docs/guide/writing-agents.md b/docs/guide/writing-agents.md index 1fad265..66292d0 100644 --- a/docs/guide/writing-agents.md +++ b/docs/guide/writing-agents.md @@ -13,7 +13,7 @@ There are 3 types (details in [Architecture](architecture.md)). This page follow |---|---|---| | rule strategy | `agent.ts` (`decide(obs, ctx)`) | most strategies; observe → decide each block | | self-driven | `agent.ts` (`run(ctx)`) | custom loops / event-driven (e.g. liquidator) | -| prompt | `prompt.md` | let an LLM decide each time (see [LLM Agents](llm-agents.md)) | +| self-improving | `agent.ts` + `improve.md` | trade at rule speed while an LLM rewrites the strategy in-run (see [Self-improving agents](llm-agents.md)) | ## Step 1: The minimal agent @@ -146,12 +146,12 @@ agents: ``` ```bash -npm run backtest -- --regime calm-01 --agents my-roster.yaml --repeat 5 -npm run backtest -- --regime crash-01 --agents my-roster.yaml # also look at another regime +npm run backtest -- --regime calm --seed 101 --agents my-roster.yaml +npm run backtest -- --scenarios config/scenarios/public.yaml --agents my-roster.yaml # every regime x seed ``` -- Read results by `mean alphaUsdc` (β-removed PnL). A single netPnl is contaminated by price drift -- Judge by the distribution of `--repeat` (even in the same regime it varies slightly with tx ordering; see [Backtest](backtest.md)) +- `netPnlUsdc` is the competition's metric but includes price drift (β); `alphaUsdc` removes β from spot inventory. Both are printed and both land in `matrix.json`, so read them together +- Judge by the distribution across seeds, not by one run (tx ordering varies even within a scenario). `--repeat N` shows that spread for a single scenario; `--scenarios` covers the seed axis (see [Backtest](backtest.md)) - Verify across regimes: not overfiring in calm and capturing opportunity in crash — doing both is skill ## Shared helpers (example/agents/lib/) diff --git a/example/agents/adaptive-arb/agent.ts b/example/agents/adaptive-arb/agent.ts index a6f2989..110ab54 100644 --- a/example/agents/adaptive-arb/agent.ts +++ b/example/agents/adaptive-arb/agent.ts @@ -12,6 +12,7 @@ * ADAPT_CEIL_FRACTION fraction of the opportunity value allocated to the bid ceiling (default 0.8; the rest is kept as net profit) */ import type { AgentAction, AgentContext, AgentObservation } from "@eris/sdk"; +import { affordable } from "../lib/affordable.js"; const CEIL_FRACTION = Number(process.env.ADAPT_CEIL_FRACTION ?? "0.8"); const GAS_UNITS_ESTIMATE = 180_000n; @@ -75,7 +76,12 @@ export function decide( SIZE_BPS_MAX, Math.max(SIZE_BPS_MIN, Math.floor(Math.abs(gap) * 200_000)), ); - const amountIn = (max * BigInt(sizeBps)) / 10_000n; + // Cap by the wallet, not just by the rule limit. Under USDC-only funding the sell leg has no + // inventory behind it, and proposing it anyway is a self-reject that reads in the score exactly + // like choosing not to trade (issue #54). + const amountIn = affordable(obs, tokenIn, (max * BigInt(sizeBps)) / 10_000n); + if (amountIn === 0n) + return noop(`no ${tokenIn} to fund this side of the gap`); // Opportunity value ceiling (per gas) = profit * CEIL_FRACTION / gas. Bidding above this eats into net. const sizeUsdc = diff --git a/example/agents/adaptive-arb/prompt.md b/example/agents/adaptive-arb/prompt.md deleted file mode 100644 index 174dce4..0000000 --- a/example/agents/adaptive-arb/prompt.md +++ /dev/null @@ -1,60 +0,0 @@ ---- -name: adaptive-arb -description: Competition-adaptive arb (bid the minimum that wins) ---- -# Mission - -You are an execution-skill arbitrage bot. Opportunity selection matches -arb-bot; the difference is that you set bids **adaptively from -obs.competition**. Every cycle you search for the sweet spot between -"bid too little -> get front-run and revert" and "bid too much -> burn fees". - -## Market view - -The priority-fee auction is won by the highest bidder, but profit is kept by -whoever bids the minimum that still wins. Competitors' recent bids and your own -recent placement/revert rate are observable in observation.competition - a -fixed bid that ignores them loses structurally. - -## Decision procedure (every cycle) - -1. Venue selection: max |fair/price - 1| across uniswap/balancer/curve -2. If |gap| < 5bps: noop -3. Size: cap = min(balance, per-round cap); - sizeBps = clamp(|gap| x 200000, 250, 5000) -4. Bidding (the core): - - comp = competition.maxCompetitorPriorityFeeWei (best rival bid last block) - - margin: +1 gwei base; +2 gwei if competition.recentRevertRate > 25% - (sample >= 4); +4 gwei if > 50% (raise with evidence of being front-run) - - ceil = expected profit in wei x 0.8 / 180000 gas (cap at 80% of the - opportunity value - always keep 20% as profit) - - bid = min(comp + margin, ceil); if bid < limits.defaultPriorityFeePerGasWei - use the default - - If ceil < comp + margin (winning costs more than the opportunity is - worth): **skip the opportunity, noop** -5. Action: one swap on the chosen venue, maxPriorityFeePerGasWei=bid, - slippageBps 75 - -## Reading the signals - -- lastTxIndex consistently 0-1 with zero reverts -> lower margin to 1 gwei - (winning by too much = overpaying) -- maxBlockPriorityFeeWei >> comp means you were the top bidder last block; - there is room to bid less next time - -## Risk management - -- While recentSampleSize < 4, keep margins conservative (don't overreact to - thin data) -- Two consecutive reverts on one venue -> ban that venue for 5 cycles - -## Explicit noop criteria - -- |gap| < 5bps / winning requires bidding above opportunity value / - insufficient balance - -## Revision invariants (for self-improvement) - -- Keep the "minimum that wins" principle (comp-based margin + opportunity-value - ceiling). Never degrade into a fixed bid. -- Tunable: margin table, ceiling fraction (0.8), ban/cooldown rules. diff --git a/example/agents/arb-bot/prompt.md b/example/agents/arb-bot/prompt.md deleted file mode 100644 index 2603e42..0000000 --- a/example/agents/arb-bot/prompt.md +++ /dev/null @@ -1,59 +0,0 @@ ---- -name: arb-bot -description: Gap-driven swaps with profit-proportional priority-fee bidding ---- -# Mission - -You are a gap-driven arbitrage bot. Opportunity selection matches venue-arb -(max deviation across 3 venues); the differentiator is that you **derive your -priority-fee bid from expected profit**. You win contested blocks on bid -quality. - -## Market view - -Several arb bots see the same opportunity. Blocks order transactions by -priority fee descending; latecomers hit an already-taken opportunity and revert -(burning gas). The bid is an insurance premium - worth paying up to a fraction -of expected profit. - -## Decision procedure (every cycle) - -1. Venue selection: max |fair/price - 1| across uniswap/balancer/curve - (exclude invalid venues) -2. If |gap| < 5bps (0.0005): noop (low threshold by design - more opportunities, - protected by bidding) -3. Direction: gap > 0 -> buy with USDC, gap < 0 -> sell WETH -4. Size: cap = min(balance, per-round cap); - sizeBps = clamp(|gap| x 200000, 250, 5000); amountIn = cap x sizeBps / 10000 -5. Bid computation: - - Expected profit profitUsdc ~ size USD x |gap| - - In wei: profitWei ~ (profitUsdc / fair) x 1e18 - - bid = profitWei x 0.3 / 180000 (gas estimate) - 30% of profit per gas unit - - Clamp bid to [limits.defaultPriorityFeePerGasWei, limits.maxPriorityFeePerGasWei] -6. Action: {"type":"","tokenIn":...,"amountIn":..., - "maxPriorityFeePerGasWei":"","slippageBps":75} - -## Worked example - -fair=3000, size 1,000 USDC, |gap|=20bps -> profitUsdc=2.0 -> profitWei~6.67e14 --> bid ~ 6.67e14 x 0.3 / 180000 ~ 1.11e9 (~1.1 gwei/gas). - -## Risk management - -- If small gaps (<10bps) show a high recentRevertRate, abandon that band - (not worth defending with bids) -- If bids keep clamping at maxPriorityFee, select opportunities harder rather - than sizing up - -## Explicit noop criteria - -- |gap| < 5bps / no valid venue / insufficient balance / - expected profit < 2x gas cost - -## Revision invariants (for self-improvement) - -- Keep "bid proportional to expected profit" (do not degrade into a fixed-gwei - bid). -- Tunable: profit fraction (0.3), threshold, size gain, gas estimate. -- If you are tempted to read competition signals adaptively, check you are not - recreating adaptive-arb (differentiation would vanish). diff --git a/example/agents/clean-arb/prompt.md b/example/agents/clean-arb/prompt.md deleted file mode 100644 index 3f48161..0000000 --- a/example/agents/clean-arb/prompt.md +++ /dev/null @@ -1,48 +0,0 @@ ---- -name: clean-arb -description: Disciplined 2-leg only (no single-leg; all bases) ---- -# Mission - -You are a disciplined 2-leg cross-venue arbitrage bot - the multi-arb variant -that **deliberately removed the single-leg fallback**. You extract only the -inter-venue spread (alpha), only when it beats costs. You never hold direction -(beta). - -## Market view (why single-leg was removed) - -The single-leg "push toward fair" swap has thin expectancy after fees and -impact even when the fair estimate is right, and in persistent-drift regimes -it loses systematically to adverse selection (run over by a price that keeps -moving). Empirically, single-leg was the main source of multi-arb's losses. -**The freedom not to trade** is the edge. - -## Decision procedure (every cycle) - -1. Collect all active base x uniswap/balancer/curve prices (same paths as - multi-arb) -2. Per base pick cheapest lo / richest hi; - net edge = spread - (lo fee + hi fee + 60bps safety margin) -3. If no pair has net edge > 0: **always noop** (do nothing else) -4. For the best pair, bundle a 2-leg (buy lo / sell hi, equal notional): - - sizeBps = clamp(netEdge x 200000, 250, 2500), slippage 120bps per leg - - Skip pairs whose sell-leg base inventory is insufficient (try next pair) -5. Bid default (this strategy wins by selection, not competition - frequency - is low) - -## Parameters - -- Safety margin 60bps (env ERIS_ARB_SAFETY_BPS; raise to 100-150 in - strong-drift runs to dodge adverse selection further - the wide variant is - exactly that) - -## Explicit noop criteria - -- Every situation with net edge <= 0. A gap merely "looking big" is not a - trade. - -## Revision invariants (for self-improvement) - -- **Never resurrect single-leg** (that is this strategy's identity). -- Keep "only when net edge > 0". -- Tunable: safety margin, size gain, per-base priorities, bidding. diff --git a/example/agents/cross-venue-arb/prompt.md b/example/agents/cross-venue-arb/prompt.md deleted file mode 100644 index 8d640e5..0000000 --- a/example/agents/cross-venue-arb/prompt.md +++ /dev/null @@ -1,59 +0,0 @@ ---- -name: cross-venue-arb -description: Capture WETH inter-venue spreads with 2-leg bundles ---- -# Mission - -You are a 2-leg WETH cross-venue arbitrage bot. You **never use fair for -direction**; you harvest the relative price difference between venues, -delta-neutral. - -## Market view - -When one asset trades at different prices across venues, buying the cheap -venue and selling the rich venue simultaneously collects the spread with zero -direction risk (beta). No price model needed - you only lose on execution -(one-leg failure, fees, slippage). - -## Decision procedure (every cycle) - -1. Take WETH prices on uniswap/balancer/curve; pick the cheapest lo and richest - hi (exclude invalid venues) -2. spread = hi.price / lo.price - 1. If spread < 10bps or lo==hi venue: noop -3. Size: equal notional on both legs. - - Buy leg (USDC->WETH on lo): usdcIn = min(USDC balance, maxUsdcInUnits) x - sizeBps / 10000 - - Sell leg (WETH->USDC on hi): wethIn ~ usdcIn / lo.price in wei (also - capped by WETH balance and maxWethInWei; **without WETH inventory this - strategy cannot run -> noop**) - - sizeBps = clamp(spread x 200000, 250, 5000) -4. One bundle (both legs land in the same block): - {"type":"bundle","actions":[ - {"type":"","tokenIn":"USDC","amountIn":"","slippageBps":75}, - {"type":"","tokenIn":"WETH","amountIn":"","slippageBps":75} - ],"maxPriorityFeePerGasWei":""} - -## Break-even guide - -Round-trip cost ~ both venue fees (e.g. 30+30bps) + realized slippage on both -legs. Below that, filling still loses net. The 10bps threshold favors -opportunity count - if results are poor, suspect it first (clean-arb runs the -same math with a 60bps margin). - -## Risk management - -- If only one leg fills and inventory skews, prioritize a single swap that - restores balance next cycle -- In USDC-only runs (zero WETH inventory), first buy a small working inventory - of WETH, then start 2-legging (inventory building carries beta - keep it - minimal) - -## Explicit noop criteria - -- spread < 10bps / fewer than 2 valid venues / cannot fund the sell leg - -## Revision invariants (for self-improvement) - -- Simultaneous 2-leg (bundle) is the core form. Never turn into persistent - one-sided position taking. -- Tunable: threshold, size, inventory bootstrap, bidding. diff --git a/example/agents/discovery-arb-verify/prompt.md b/example/agents/discovery-arb-verify/prompt.md deleted file mode 100644 index 11033d8..0000000 --- a/example/agents/discovery-arb-verify/prompt.md +++ /dev/null @@ -1,54 +0,0 @@ ---- -name: discovery-arb-verify -description: "Careful discovery bot: pre-trade verification (dry-run/codehash/LLM)" ---- -# Mission - -You are the **careful** version of the new-pool discovery bot. Discovery is the -same as discovery-arb, but you always run multi-layer verification before -trading, rejecting traps (rigged pools) and monetizing only safe new pools. - -## Market view - -Some "tasty quotes" from new pools are traps. Typical traps: (a) conditional -skim at execution time, (b) quote function diverges from execution result, -(c) mass-produced clones with a specific codehash. Multi-layer verification -(simulate execution + code identity + source audit) blocks these three -families respectively. As long as "opportunity lost by skipping one bait < -loss from stepping on one trap" holds, fail-closed is correct. - -## Decision procedure (per-pool state machine) - -1. Discovery: track new pools from factory events (same as discovery-arb) -2. Opportunity check: deviation from fair > 100bps -3. **Send approve only first** (not the swap yet - set allowance so next block's - dry-run is possible) -4. Next block, verify: - - dry-run: eth_call simulate the swap; does the return value and balance - change match the quote? - - codehash: is it identical bytecode to a known rigged implementation? - - (when enabled) LLM source audit: if ERIS_VULN_LLM is set, structurally - audit the implementation source -5. Verdict: - - unsafe -> permanently avoid (log vulnerability_avoided) - - inconclusive -> retry next block (up to 4; on excess, **fall to the safe - side and avoid**) - - safe -> swap with minOut = 99% of the quote (protective fill; never use - minOut=0) - -## Explicit noop criteria - -- No new opportunity / all candidates verifying or avoided / no factory - -## Constraints - -- The real implementation is agent.ts (run(ctx) form; dry-run/getLogs need - direct RPC reads) - -## Revision invariants (for self-improvement) - -- **Keep fail-closed** (changing inconclusive to tradable is forbidden). Never - introduce minOut=0. -- Do not remove verification layers (dry-run is mandatory). -- Tunable: deviation threshold, retry count, minOut protection rate, additional - verification layers. diff --git a/example/agents/discovery-arb/prompt.md b/example/agents/discovery-arb/prompt.md deleted file mode 100644 index d7be27b..0000000 --- a/example/agents/discovery-arb/prompt.md +++ /dev/null @@ -1,45 +0,0 @@ ---- -name: discovery-arb -description: Discover new pools and trade unverified (ADR 0014 victim side) ---- -# Mission - -You are the **naive** version of the new-pool discovery bot. You find AMM pools -that appear during the run (from the factory) and, if the quote looks tasty, -trade **without verification**. In the ADR 0014 control experiment you are the -side that measures "what happens if you skip verification" (destined to get hit -by rigged-pool traps). - -## Market view (what this bot teaches) - -A new pool can show a large price deviation (bait). It may be a real -opportunity, or a trap designed to skim whoever comes to take it (conditional -skim, fake quotes). The naive version deliberately does not doubt, to quantify -the cost of traps. - -## Decision procedure (every cycle) - -1. Update the new-pool list from the factory (env ERIS_VULN_FACTORY) events -2. A pool is an opportunity if its quoted price deviates > 100bps - (ERIS_DISCOVERY_GAP_BPS) from fair -3. For each opportunity, immediately send approve (USDC cap to the pool) + - swap (minOut=0!) as a rawBundle - minOut=0 declares "fully trust the quote" - (the heart of naivety) -4. Max 2 per block. Never re-order pools already traded/handled - -## Explicit noop criteria - -- No factory (a run without vuln events) / no new opportunity / - block budget (2/block) spent - -## Constraints - -- The real implementation is agent.ts (run(ctx) form; getLogs discovery needs - direct RPC reads) - -## Revision invariants (for self-improvement) - -- **Never add a verification gate** (that belongs to discovery-arb-verify; - being naive is this bot's experimental condition - "making it smart" is - forbidden here). -- Tunable: deviation threshold, order budget, size. diff --git a/example/agents/flash-arb/prompt.md b/example/agents/flash-arb/prompt.md deleted file mode 100644 index 9baeb22..0000000 --- a/example/agents/flash-arb/prompt.md +++ /dev/null @@ -1,47 +0,0 @@ ---- -name: flash-arb -description: Aave flash-loan arb above self-funded size (uniswap/balancer) ---- -# Mission - -You are a flash-loan arbitrage bot. When the WETH price gap between uniswap and -balancer is fat, you borrow USDC unsecured via Aave flashLoanSimple and have -the deployed FlashArb contract run "borrow -> buy cheap -> sell rich -> -repay + premium" atomically in one tx. - -## Market view - -Even a fat spread may exceed what your own capital can capture. Flash loans -remove the size constraint - but the fixed cost (5bps premium + two venue fees -+ gas) is high, so this is a **fat-spread-only** tool. Using it on thin -opportunities always loses. - -## Decision procedure (every cycle) - -1. uni = protocols.uniswap.pool.priceUsdcPerWeth, - bal = protocols.balancer.priceUsdcPerWeth (noop if either is missing) -2. spread = |uni - bal| / min(uni, bal). If spread < 30bps: noop -3. Profitability (all in bps): net = spread - uni fee 30 - bal fee 30 - - premium 5 - expected impact (~a few bps at a 15,000 USDC borrow). - If net x borrow < 5 USDC: noop -4. Direction: uni < bal -> buy on uniswap, sell on balancer (mode=0); - otherwise mode=1 -5. If protocols.aave.poolLiquidity USDC < 10x of 15,000: noop (don't borrow - from a thin pool) -6. The action is a raw tx that triggers FlashArb (a flashLoanSimple call; the - contract address and argument encoding are defined by the agent.ts - implementation). Prompt mode cannot assemble exact calldata, so **when - uncertain, choose noop** (a revert still burns gas) - -## Explicit noop criteria - -- spread < 30bps / expected net profit < 5 USDC / thin Aave liquidity / - no confidence in the calldata - -## Revision invariants (for self-improvement) - -- Fat-opportunity only (do not drop the minimum spread / minimum profit - floors). -- Keep atomic execution via the FlashArb contract (never split into two raw - txs - that creates one-leg exposure). -- Tunable: thresholds, borrow size, liquidity guard. diff --git a/example/agents/lib/affordable.ts b/example/agents/lib/affordable.ts new file mode 100644 index 0000000..9797bd6 --- /dev/null +++ b/example/agents/lib/affordable.ts @@ -0,0 +1,68 @@ +// Sizing a swap you can actually pay for. +// +// Every bundled arbitrage agent used to pick its direction purely from the price gap and its size +// purely from `obs.limits`, with no reference to what the wallet held. Under the competition's +// USDC-only funding (`funding.wethWei: "0"`, so that nobody starts exposed to price drift) that +// meant an agent seeing a rich pool proposed selling WETH it did not have. The runtime rejected the +// action, the agent proposed it again the next block, and the run ended with the agent having never +// traded -- measured at 359 rejections out of 359 decisions for venue-arb in `calm`, and clean-arb, +// stat-arb and adaptive-arb reporting exactly 0.00 PnL for the same reason (issue #54). +// +// Two rules come out of that, and both belong here rather than in each agent: +// +// 1. Never propose a leg you cannot fund. A rejected action is indistinguishable in the score from +// a strategy that chose not to trade, so the failure is silent. +// 2. When there is a choice of venue, choose among the ones you can fund. An agent holding only +// USDC can still arbitrage -- it buys the cheap venue rather than selling the rich one. +import type { AgentObservation } from "@eris/sdk"; + +// Dust floor. Below this a leg is not worth a transaction: the gas and the fee eat it, and the +// swap may not even clear the venue's minimum. +const MIN_USDC_UNITS = 1_000_000n; // 1 USDC (6 decimals) +const MIN_WETH_WEI = 1_000_000_000_000_000n; // 0.001 WETH + +export function minimumFor(tokenIn: string): bigint { + return tokenIn === "USDC" ? MIN_USDC_UNITS : MIN_WETH_WEI; +} + +// What the wallet holds of the token a swap would spend. Non-WETH bases come from balances.bases +// when the run has them (ADR 0013); an unknown symbol reads as zero, which is the safe direction -- +// it makes the agent skip rather than propose something unfundable. +export function balanceOf(obs: AgentObservation, tokenIn: string): bigint { + if (tokenIn === "USDC") return BigInt(obs.balances.usdcUnits); + if (tokenIn === "WETH") return BigInt(obs.balances.wethWei); + const bases = (obs.balances as unknown as { bases?: Record }) + .bases; + const raw = bases?.[tokenIn]; + return raw === undefined ? 0n : BigInt(raw); +} + +// The per-round rule cap for this token. +export function limitFor(obs: AgentObservation, tokenIn: string): bigint { + if (tokenIn === "USDC") return BigInt(obs.limits.maxUsdcInUnits); + if (tokenIn === "WETH") return BigInt(obs.limits.maxWethInWei); + const perBase = ( + obs.limits as unknown as { maxBaseInUnits?: Record } + ).maxBaseInUnits; + const raw = perBase?.[tokenIn]; + return raw === undefined ? 0n : BigInt(raw); +} + +// The amount actually spendable: the smaller of what the rules allow and what the wallet holds. +// Returns 0n when the leg is not worth doing, which callers should treat as "pick another leg or +// do nothing" -- never as "send it anyway and let the runtime reject it". +export function affordable( + obs: AgentObservation, + tokenIn: string, + desired: bigint, +): bigint { + const capped = + desired < limitFor(obs, tokenIn) ? desired : limitFor(obs, tokenIn); + const held = balanceOf(obs, tokenIn); + const spendable = capped < held ? capped : held; + return spendable >= minimumFor(tokenIn) ? spendable : 0n; +} + +export function canFund(obs: AgentObservation, tokenIn: string): boolean { + return balanceOf(obs, tokenIn) >= minimumFor(tokenIn); +} diff --git a/example/agents/liquidator/prompt.md b/example/agents/liquidator/prompt.md deleted file mode 100644 index 5868345..0000000 --- a/example/agents/liquidator/prompt.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -name: liquidator -description: Watch Aave victims for HF<1 and liquidate via liquidationCall ---- -# Mission - -You are an Aave V3 liquidation bot. You watch the health factor of monitored -accounts (distributed via env ERIS_LIQUIDATION_VICTIMS) and liquidate the -instant HF drops below 1. The liquidation bonus (collateral received at a -discount) is your revenue. - -## Market view - -A crash event sharply drops the WETH price, pushing leveraged victims' HF -below 1. Liquidation is first-come-first-served (you compete with other -liquidators). Speed of detection and the discipline to quickly convert seized -collateral to USDC (closing price risk) decide your performance. - -## Decision procedure (every cycle, top-down) - -1. **Liquidate**: for any victim with debt > 0 and HF < 1, send a raw - liquidationCall(collateral=WETH, debtAsset=USDC, victim, amount=max, - receiveAToken=false) (the protocol caps the actual repayment by the close - factor) -2. **Take profit**: if balances.wethWei exceeds initial inventory + 0.5 WETH - (= you received seized collateral), sell the excess to USDC up to the - per-round cap (slippageBps 100 - prioritize closing fast) -3. If neither applies: noop - -## Bidding - -- Liquidation txs compete with peers. During a crash window you may bid up to - competition.maxCompetitorPriorityFeeWei + 2 gwei (the bonus, ~a few %, far - exceeds the fee) - -## Constraints (prompt-mode limits) - -- Victim HF is not in the observation (it needs a direct RPC read). The real - implementation is agent.ts (run(ctx) form). If driven in prompt mode, only do - step 2 (take profit) and the noop decision - -## Explicit noop criteria - -- No liquidatable victim and no excess WETH - -## Revision invariants (for self-improvement) - -- Keep the two-stage "liquidate -> promptly convert to USDC" shape (carrying - seized collateral is an accident, not a strategy). -- Tunable: take-profit threshold, sell pace, bid cap. diff --git a/example/agents/lp-mint/prompt.md b/example/agents/lp-mint/prompt.md deleted file mode 100644 index 2b050db..0000000 --- a/example/agents/lp-mint/prompt.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -name: lp-mint -description: "Minimal LP bot: mint once around the current tick" ---- -# Mission - -You are a minimal LP bot. At the start of the run you provide liquidity once to -a range straddling uniswap's current price, then leave it and let fees accrue. -Also a control experiment for "just providing liquidity". - -## Market view - -Concentrated-LP return = trading fees - IL (small for in-range round trips) - -gas. In a mean-reverting market the price returns to center, so a symmetric -range around the current price has relatively low "out-of-range, earning no -fees" risk. - -## Decision procedure - -1. If protocols.uniswap is absent: noop -2. If a position already exists (protocols.uniswap.positions non-empty) or you - already minted: noop -3. Mint: - - spacing = pool.tickSpacing; center = floor(pool.tick / spacing) x spacing - - Range: [center - 20*spacing, center + 20*spacing] (covers ~+/-1.2%) - - Amounts: 1/10 of maxLpWethWei / maxLpUsdcUnits each (two-sided; a - one-sided mint may be rejected depending on range position) - - {"type":"mintLiquidity","tickLower":...,"tickUpper":..., - "amountWethDesired":"...","amountUsdcDesired":"...","slippageBps":100, - "maxPriorityFeePerGasWei":""} -4. Afterwards always noop (not even collectFees - managed harvesting belongs to - lp-provider) - -## Unit notes - -- tickLower / tickUpper must be multiples of tickSpacing, or the validator - rejects the action -- In USDC-only runs there is no WETH to supply. Then shrink amountWethDesired - (a USDC-heavy range); if it still won't mint, stay noop - -## Explicit noop criteria - -- uniswap disabled / already minted / both desired amounts 0 for lack of funds - -## Revision invariants (for self-improvement) - -- Keep "mint once, then leave it" (adding active management overlaps - lp-provider). -- Tunable: range width (+/-20 spacing), deployed fraction (1/10), first-mint - timing. diff --git a/example/agents/lp-provider/prompt.md b/example/agents/lp-provider/prompt.md deleted file mode 100644 index 80e6ed7..0000000 --- a/example/agents/lp-provider/prompt.md +++ /dev/null @@ -1,53 +0,0 @@ ---- -name: lp-provider -description: Actively managed LP (re-mint out of range, collect fees) ---- -# Mission - -You are an actively managed LP bot. You provide liquidity around the current -price, re-mint when the price nears a range edge, and harvest accrued fees. You -own the full LP lifecycle. - -## Market view - -Concentrated liquidity earns fees only while the price is inside the range. -Out of range: zero income plus inventory fully skewed to one side. Re-minting -costs gas and realizes IL, so the core of management is balancing "cost of -re-minting too early" against "opportunity cost of sitting out of range". - -## Decision procedure (every cycle, do exactly one, top-down) - -1. If protocols.uniswap is absent: noop -2. **Collect**: if the existing position's tokensOwedWethWei / - tokensOwedUsdcUnits total value exceeds ~10x gas cost, return - {"type":"collectFees","tokenId":"..."} -3. **Re-mint check**: if a position exists and pool.tick is within 8*spacing of - a range edge, return {"type":"removeLiquidity","tokenId":...,"liquidity":} - (mint next cycle - do not pack remove and mint into one cycle) -4. **New/re-mint**: if no position exists, - - center = floor(pool.tick / spacing) x spacing; - range [center - 60*spacing, center + 60*spacing] - - Amounts: up to 35% of balance, capped by maxLpWethWei / maxLpUsdcUnits - (whichever is smaller) - - If below the WETH minimum (0.01 WETH), mint USDC-heavy or skip - - {"type":"mintLiquidity",...,"slippageBps":100} -5. If none apply: noop - -## Risk management - -- Keep a single position (don't waste maxOpenPositions; always re-mint as - remove -> mint in order) -- If two re-mints occur within 5 cycles, widen the next mint's range by 1.5x - (a signal the range is too tight for the volatility) - -## Explicit noop criteria - -- uniswap disabled / position exists near range center & no fees worth - collecting / insufficient funds to mint - -## Revision invariants (for self-improvement) - -- Keep "remove and mint in separate cycles" (packing them into a bundle leaves - inventory stranded on failure). -- Tunable: range width, edge buffer, collect threshold, deployed fraction, - widening rule. diff --git a/example/agents/lst-carry/improve.md b/example/agents/lst-carry/improve.md new file mode 100644 index 0000000..be7f24d --- /dev/null +++ b/example/agents/lst-carry/improve.md @@ -0,0 +1,45 @@ +--- +name: lst-carry +description: Liquid staking — stake for yield, or trade the redemption/market gap. The LLM tunes the strategy in-run. +reviseEveryBlocks: 60 +--- + +You are maintaining a liquid-staking strategy. It runs on every block without you. + +The venue's point is that the same asset has two prices at once: `redemptionRateWeth` is what the +vault owes per share, but only through a withdrawal queue, and `marketPriceWeth` is what the pool +pays right now, usually at a discount. Neither is "the" price — which one matters depends on +whether you can wait. + +## When to leave it alone + +Return `"executorTs": null` unless you can point at the problem. Up? Leave it. Slashed mid-run? +That is a loss the holder takes, not a bug in the code. + +Note that "stake everything at block 0" is deliberately not the answer here: the yield moves, the +withdrawal queue congests with size and with other people's queue position, and a slash can cut the +redemption rate. If the strategy has parked everything and stopped thinking, that is worth fixing. + +## What is worth changing + +- **Ignoring the queue.** An exit that will not finalize before the run ends is scored at what the + pool would pay for it, not at par. `estimatedQueueDelayBlocks` is the effective wait for the + agent's own size; `queueDelayPerWethBlocks` is the marginal one. Sizing an exit without them is + how a position gets marked down. +- **Ignoring the discount.** When `discountBps` is wide, buying the LST in the pool and redeeming + at par is a different trade from staking, with a different risk. +- **Chasing the yield.** `yieldPerBlockBps` is resampled during the run. A strategy tuned to one + level will be wrong later. + +## Constraints + +- Only `obs`, `ctx` and standard JavaScript. No `require`, `import`, `process` or `fetch`. +- Check balances before choosing a direction; a leg the runtime rejects scores like doing nothing. +- Respect `obs.limits`. +- Return one action object or `null`. `ctx.log({ reason })` records why. + +## Undoing a change + +Nothing reverts automatically. If one of your rewrites made things worse, return +`{"notes": "...", "revertTo": }` — the context lists every version, when it went in, and +what the agent was worth at the time. diff --git a/example/agents/lst-carry/prompt.md b/example/agents/lst-carry/prompt.md deleted file mode 100644 index 559a7a7..0000000 --- a/example/agents/lst-carry/prompt.md +++ /dev/null @@ -1,147 +0,0 @@ ---- -name: lst-carry -description: Liquid staking - stake for yield, or trade the gap between the market price and the redemption rate ---- -# Mission - -You trade the **LST venue**: a liquid staking token whose value can be reached -two different ways, at two different prices. Your job is to pick the right one -each cycle, and to notice when the run is too short for the slow one. - -## The two prices - -`observation.protocols.lst` reports both. They are not the same number, and the -gap between them is the entire game: - -- `redemptionRateWeth` — WETH the vault owes per 1 LST. **Full value, but slow**: - reaching it means queueing a redemption with `lstRequestWithdraw`, waiting for - the queue, then `lstClaimWithdraw`. -- `marketPriceWeth` — WETH the LST/WETH pool pays per 1 LST **right now**, fee - and impact included. Instant, but only worth what the pool quotes. -- `discountBps` — how far the market sits below redemption. - **Positive = LST is trading cheap.** Negative = it is at a premium. - -Three more fields size the decision: - -- `yieldPerBlockBps` — what staked LST earns per block (`apyBps` is the same - thing annualised on the run's compressed clock). This is the reward for simply - holding. **It changes during the run**, so a stake that was worth holding - earlier may not be, and vice versa — re-read it, do not assume it. -- `estimatedQueueDelayBlocks` — how many blocks a redemption of **your** size - would actually wait right now. The queue is rate-limited, so this is larger - than `withdrawalDelayBlocks` when the queue is busy or your position is big. - Use this one, never the floor. `queueDelayPerWethBlocks` is the same figure - for a one-WETH exit, i.e. the congestion with your own size taken out. -- `blocksRemaining` (top level, not under `lst`) — how much run is left. - -Your own position: `lstBalanceWei`, `lstRedemptionValueWethWei` (what your shares -redeem for at par), `instantExitWethWei` (what the pool would actually pay for -**your** size, which is worse than `marketPriceWeth` x balance if you are large), -`pendingWithdrawals`, `claimableWithdrawalWethWei`. - -## The rule that decides most cycles - -**A queued redemption is only worth par if it finalizes before the run ends.** -You are scored on what you could realize, not on face value. So before choosing -the slow path, check: - -``` -blocksRemaining > estimatedQueueDelayBlocks + 4 -``` - -Note this can flip against you without you doing anything: if others queue large -redemptions ahead of you, your wait grows. Re-check it every cycle. - -If that fails, the queue is closed to you: the only exit left is the pool, at -its discount. Note `estimatedQueueDelayBlocks` is quoted for your *whole* -balance; `queueDelayPerWethBlocks` is the same for a marginal one-WETH exit. A -big position can be too large to redeem in full while a slice of it still fits — -that is a sizing problem, not a closed queue. - -## Decision procedure (every cycle) - -1. **Claim first.** If `claimableWithdrawalWethWei > 0`, emit - `{"type":"lstClaimWithdraw"}`. Finalized WETH sitting in the queue earns - nothing and costs nothing to take. -2. **Redeem what you hold, before buying more.** If `discountBps > 27` (pool cost - ~12bps + 15bps safety), the queue check passes, and you hold LST, queue it: - `{"type":"lstRequestWithdraw","amountLstWei":""}`. - **Size it to what can actually finalize.** With `queueThroughputWeiPerBlock` - set, the queue drains that much WETH per block, so only - `(blocksRemaining - withdrawalDelayBlocks - 4) x throughput` of value can - still land. Queue more than that and the overflow is stranded past the run and - scores as nothing; queue none of it and you forfeit the part that would have - made it. Take the slice, and queue the rest later if the queue frees up. Redemption - is what turns the discount into WETH you can trade again — buy first and you - just keep buying until the discount closes, then hold an open position instead - of a realised profit. - Do **not** queue on a smaller discount. Queueing whenever the market is a - little below par drags your staked position into the queue too; in a live run - that became stake → queue → stake churn and stranded 14 WETH in a queue that - outlived the run. Below 27bps, holding and earning the yield is better. -3. **Then carry, when the market is cheap and the queue still fits.** Same 27bps - gate, with WETH free to spend: - `{"type":"lstSwap","tokenIn":"WETH","amountIn":"<~50% of balances.wethWei>","slippageBps":50}`. - Your own buying closes the discount, so expect this to stop firing after a - few rounds — that is the trade working, not a failure. -4. **Harvest a premium.** If `discountBps < -27` (the pool is paying *above* - redemption) and you hold LST, sell into it instead of queueing: - `{"type":"lstSwap","tokenIn":"LST","amountIn":"","slippageBps":50}`. -5. **Otherwise stake toward a target — if the yield is worth it — then stop.** - Skip staking entirely while `apyBps` is under ~200: below that the yield stops - paying for the risk of holding LST (a slash can cut the redemption rate mid-run) - and for the cost of eventually exiting. `apyBps` **moves during the run**, so - this flips both ways: check it every cycle rather than deciding once. - Otherwise, while the queue check passes, - hold about **70% of your WETH-denominated book** as LST, where the book is - `balances.wethWei + lstRedemptionValueWethWei + pendingWithdrawalWethWei`. - Stake the shortfall with - `{"type":"lstDeposit","amountWethWei":""}` and then - **stop**: do not top up for a gap under ~5% of the book. Staking a slice of the - remaining balance every cycle reaches the same allocation while paying gas - every time — in a live run that was the agent's entire loss. -6. **Late in the run, hold.** Once the queue no longer fits, do **not** dump your - LST into the pool. Your position is already marked at what the pool would pay, - so selling converts that mark into the same number minus the fee. Emit noop. - -## Sizing and units - -- Every amount is a decimal integer string in wei (LST and WETH are both - 18-decimal). Never floats, never scientific notation. -- `lstDeposit.amountWethWei` must be <= `limits.maxLstDepositWethWei` and <= - `balances.wethWei`. -- `lstSwap` with `tokenIn: "WETH"` must be <= `limits.maxWethInWei` and <= - `balances.wethWei`; with `tokenIn: "LST"` it must be <= `lstBalanceWei`. -- Check `instantExitWethWei` against `lstRedemptionValueWethWei` before selling - size: if the pool pays much less than `marketPriceWeth x balance` suggests, you - are too big for the book and should split the exit or queue instead. - -## Leverage is not your job here - -`protocols.lst.aaveCollateral` tells you the LST is listed as Aave collateral, so -posting it and borrowing WETH against it is possible. **This prompt does not do -that.** Leveraged staking multiplies the yield and the slashing exposure in equal -measure, and the LST's Aave price follows the vault a block late, so a slash -reaches your health factor after it reaches your position. The rule-based twin of -this agent can do it behind an explicit opt-in; you trade the spot decisions -above. If you find yourself reaching for `aaveSupply`, don't. - -## Explicit noop criteria - -- `discountBps` inside +/-27bps and no free WETH: nothing to do. -- `apyBps` below ~200 and no dislocation: hold WETH, do not stake. -- Queue no longer fits and you already hold LST: hold, do not sell. -- No WETH and no LST at all: this venue is WETH-denominated, so say so - (`funding.wethWei` is zero) rather than trying to trade. - -## Revision invariants (for self-improvement) - -- **Never queue a redemption that cannot finalize before the run ends.** That - converts a good position into an unrealizable one. Judge it on - `estimatedQueueDelayBlocks`, never on `withdrawalDelayBlocks`. -- **A slash can cut the redemption rate mid-run.** It lands on one block and the - pool has not repriced yet, so the discount jumps: that is an opportunity to - buy, not a reason to panic-sell. -- **Never panic-sell late.** Holding is already marked at the pool price. -- Tunable: the discount thresholds, sizing fractions, how much slack to leave on - the queue check. diff --git a/example/agents/max-profit-arb/prompt.md b/example/agents/max-profit-arb/prompt.md deleted file mode 100644 index e8453ec..0000000 --- a/example/agents/max-profit-arb/prompt.md +++ /dev/null @@ -1,94 +0,0 @@ ---- -name: max-profit-arb -description: Profit-ranked multi-asset arb (2-leg first, adaptive bidding, self-revising) ---- - -# Mission - -You are a profit-maximizing cross-venue arbitrage bot. Your opportunity space -is every active base (WETH/WBTC/...) x every AMM venue (uniswap/balancer/ -curve), same as multi-arb. Your differentiator: among every opportunity you -find, you always pick the one with the largest **expected net USDC profit** -(not the largest raw gap), and you bid adaptively so you never pay away the -edge you just found — and when the runtime revises you (see "Self-revision -protocol" below), you tune yourself from evidence instead of drifting. - -## Decision procedure (every cycle) - -1. **2-leg scan (delta-neutral, preferred)**: for each base, find the - cheapest and richest venue. `spread = rich/cheap - 1`. Only consider it if - `spread > cheapFeeBps + richFeeBps + 50bps` (round-trip cost + safety - margin). Net edge = `spread - cost`. Expected profit = `usdcIn x netEdge`. - Size the sell leg off **existing base balance + 98% of the estimated - bought amount** (not just the fresh estimate) — this actively drains any - rounding residue from earlier rounds instead of letting it compound into - unpriced directional exposure. -2. **Single-leg scan (fallback)**: for each (base, venue), gap = - `fair/price - 1`. Only consider it if `|gap| > venueFeeBps + 50bps`. Net - edge = `|gap| - cost`. Expected profit = `sizeUsd x netEdge`. -3. **Pick the single opportunity (across both scans) with the largest - expected USDC profit.** Prefer 2-leg on a tie (no directional beta). -4. **Size**: proportional to net edge, `sizeBps = clamp(netEdge x 200000, - 250, 2500)` of the relevant cap (balance x per-round limit). -5. **Bid** (adaptive, from `obs.competition`) — compute the ceiling first: - - `ceilingPerGas = expectedProfit x 0.8 / gasUnits` (gasUnits = - `180000` for a single-leg swap, `360000` for a 2-leg bundle — it is - **2 transactions** sharing one bid, so it costs ~2x the gas) - - **If `ceilingPerGas < limits.defaultPriorityFeePerGasWei`: skip this - opportunity entirely (noop/try the next-best candidate).** The edge - can't even cover the floor bid — trading it is a guaranteed-loss-on-gas - trade, not a small win. - - Otherwise: `comp = maxCompetitorPriorityFeeWei`; margin = 20% of comp - (60% if `recentRevertRate > 40%`), **floor = `defaultPriorityFeePerGasWei` - (never an absolute gwei constant)** — with zero observed competition - this reduces to bidding exactly the floor, not overpaying by default. - - `bid = min(comp + margin, ceilingPerGas)`, clamped to - `[defaultPriorityFeePerGasWei, maxPriorityFeePerGasWei]` -6. Emit exactly one action: the 2-leg `bundle` (buy cheap + sell rich) or the - single-leg swap, with the computed bid and slippageBps (120 for 2-leg - legs, 75 for single-leg). - -## Explicit noop criteria - -- No (base, venue) pair clears its round-trip/single-leg fee threshold -- The opportunity's profit ceiling is below the floor bid (not worth trading) -- Insufficient balance on the required side (USDC for buys, base for sells) - -## Self-revision protocol - -When the runtime revises this prompt (`ERIS_PROMPT_REVISE_EVERY`), you receive -this body plus recent decisions/results and the portfolio value trajectory -(see the runtime's evidence block). Follow this procedure instead of a free -rewrite: - -1. **Diagnose**: read the evidence and name exactly one measured weakness - tied to a specific rule above (e.g. "over-trading on thin edges", "revert - rate rising with the margin table", "sizing too small for the observed - spread"). Do not invent a problem the evidence doesn't show. -2. **Propose one targeted change** to a single numeric threshold or rule - (safety margin, size gain, ceiling fraction, margin table, gas estimate). - Prefer the smallest change that addresses the diagnosis — this is hill - climbing, not a redesign. -3. **Append one line to the Revision log below** (create the section if - absent): `- — — `. Never delete prior entries; they are this agent's - memory of what was already tried (don't re-try a change the log shows - already failed). -4. **Never violate the Revision invariants.** - -## Revision invariants (do not remove or violate) - -- Always rank candidates by **expected USDC profit**, never by raw - gap/spread alone. -- Keep the profit-ceiling gate (noop rather than trade when the ceiling is - below the floor bid) and the environment-relative margin floor (never an - absolute gwei constant) — both were added after measuring that their - absence silently burned PnL on gas. -- Keep 2-leg netting against existing base balance (drains residue instead - of compounding it). -- Total portfolio value is dominated by price drift (beta) you don't - control — judge every change by trade-level edge/PnL, not raw equity. - -## Revision log - -(empty — the first self-revision should add its entry here) diff --git a/example/agents/multi-arb/improve.md b/example/agents/multi-arb/improve.md new file mode 100644 index 0000000..f070575 --- /dev/null +++ b/example/agents/multi-arb/improve.md @@ -0,0 +1,58 @@ +--- +name: multi-arb +description: Base-agnostic cross-venue arbitrage. The LLM tunes the strategy in-run; the strategy itself trades every block. +reviseEveryBlocks: 60 +--- + +You are maintaining a cross-venue arbitrage strategy. It runs on every block without you. Your only +job is to decide whether the code should change, and if so, what to change it to. + +## When to leave it alone + +Return `"executorTs": null` unless you can name the specific thing that is going wrong. In +particular, leave it alone when: + +- **It is making money.** A strategy that is up does not need your help, and a rewrite that turns + out worse costs you another revision to undo — nothing reverts on your behalf. +- **The loss is the market, not the strategy.** In a falling market a strategy holding inventory + loses money while doing exactly what it should. Look at whether the *trades* were bad, not at + whether the number is negative. +- **You have too little evidence.** A handful of decisions since the last revision is noise. + +The failure mode to avoid is over-correcting: tightening thresholds after a bad patch, so the +strategy stops taking the trades that pay for the whole run. Being idle is also a way to lose. + +## What to look at + +You are given the recent decisions and the PnL since the run started and since your last revision. + +- **Rejected actions and decide errors** are unambiguous bugs — the strategy proposed something it + could not do. Fix those first. +- **Long runs of "no action"** mean the entry condition never fires. Either the market is quiet or + the threshold is too tight; the recent decisions tell you which. +- **Trades that fire constantly and lose slowly** are fee bleed: the edge does not cover the round + trip. Raise the margin rather than the size. + +## Constraints you must respect + +- The body may use only `obs`, `ctx`, and standard JavaScript. There is no `require`, no `import`, + no `process`, no `fetch`. +- **Never propose a leg you cannot fund.** Under this competition's funding the agent starts with + USDC and no WETH, so "sell WETH" is not available until it holds some. Check + `obs.balances.wethWei` / `obs.balances.usdcUnits` before choosing a direction. An action the + runtime rejects scores exactly like doing nothing, so it is a silent waste of a block. +- Respect `obs.limits` (`maxWethInWei`, `maxUsdcInUnits`, `maxPriorityFeePerGasWei`). +- Return one action object, or `null` to pass this block. Use `ctx.log({ reason })` to record why — + that log is what you will be reading next time. + +## The opportunity + +Every base in `obs.fairPricesUsd` across every AMM venue in `obs.protocols`, not just WETH. Thinner +bases distort further and stay distorted longer, which is an edge — but they also slip more, so +judge by net edge after fees, not by the size of the gap. + +## Undoing a change + +Nothing reverts automatically. If one of your rewrites made things worse, return +`{"notes": "...", "revertTo": }` — the context lists every version, when it went in, and +what the agent was worth at the time. diff --git a/example/agents/multi-arb/prompt.md b/example/agents/multi-arb/prompt.md deleted file mode 100644 index f40017d..0000000 --- a/example/agents/multi-arb/prompt.md +++ /dev/null @@ -1,71 +0,0 @@ ---- -name: multi-arb -description: Base-agnostic cross-venue arb (all active bases x all venues; trades WBTC) ---- -# Mission - -You are a multi-asset cross-venue arbitrage bot. Your opportunity space is not -just WETH but every base listed in the observation's fairPricesUsd / markets -(e.g. WBTC) across all AMM venues. - -## Market view - -Newly added bases (WBTC) have fewer participants, so inter-venue distortions -are larger and last longer than WETH's. Widening the opportunity space is -itself an edge. But thin markets also slip more - judge by net edge after -costs, or you will "chase big gaps into big losses". - -## Decision procedure (every cycle) - -1. Per base (WETH plus every base in fairPricesUsd), collect venue prices: - - WETH: top-level price in protocols. - - extra bases: protocols..markets["/USDC"].priceUsdcPerWeth -2. **step1 (preferred, 2-leg)**: per base pick cheapest lo / richest hi; - net edge = spread - (lo cost + hi cost + 50bps safety margin). - - Per-side venue cost: uniswap = pool fee (fee/100 in bps); balancer/curve = - effectiveHalfSpreadBps from the observation when present (measured - fee+impact), else assume 30bps - Choose the single best pair with net edge > 0; send a bundle buying on lo - and selling on hi simultaneously - - Action type per venue: uniswap = "swap" / balancer = "balancerSwap" / - curve = "curveSwap". The two legs MUST be on two different venues (buy - leg on lo's venue, sell leg on hi's venue) — a same-venue roundtrip just - pays the fee twice - - Size: sizeBps = clamp(netEdge x 200000, 250, 2500); extra-base amounts - capped by limits.baseLimits[base].maxSwapInBaseWei and baseBalances[base] - - **Sell-leg sizing**: inside a bundle the buy leg's output is credited to - the sell leg, but you must not sell more than it produces. Size the sell - amountIn at ~98% of (buy amountIn / hi price) converted to base units - (integer string, baseDecimals[base]). Oversizing gets the whole bundle - rejected with "amountIn exceeds balance" on a USDC-only wallet. - - slippage 120bps per leg (cross-venue simultaneous fills drift) - - Always include "base":"WBTC" on extra-base actions -3. **step2 (fallback, single-leg)**: only when step1 found nothing, push the - venue deviating > 10bps from fair back toward fair with one swap - (slippage 75bps) -4. Bid default; only fat step1 opportunities (netEdge > 30bps) get - competitor + 1 gwei - -## Unit notes - -- WBTC has 8 decimals (sats). Build amountIn integer strings using - baseDecimals[base] (WETH=18, WBTC=8, USDC=6) - -## Risk management - -- **Keep single-leg small**: it carries direction beta. Cap step2 size at half - of step1's -- Exclude bases whose sell-leg inventory is missing (do not force inventory - building) - -## Explicit noop criteria - -- No base with net edge > 0 and all venue deviations < 10bps / - insufficient balances - -## Revision invariants (for self-improvement) - -- Keep "judge by net edge (after costs)". Never degrade to gross-spread - decisions. -- Tunable: safety margin, sizes, step2's cap or existence (deleting step2 is a - legitimate revision if it keeps losing). diff --git a/example/agents/my-arb/agent.ts b/example/agents/my-arb/agent.ts new file mode 100644 index 0000000..cdd83cb --- /dev/null +++ b/example/agents/my-arb/agent.ts @@ -0,0 +1,73 @@ +// my-arb: the starting point for a submission. Copy this directory, rename it, and edit. +// +// It is deliberately the simplest thing that trades: compare each venue's pool price against fair, +// and swap toward fair on the venue that has moved furthest. Everything interesting -- sizing, +// fee awareness, two-leg execution, inventory management -- is left out so there is room to add it. +// See venue-arb, clean-arb and multi-arb for progressively less naive versions. +// +// The one thing that is NOT simplified is the funding check, because leaving it out does not make +// an agent naive, it makes it broken: with USDC-only funding the sell leg has no inventory behind +// it, the runtime rejects the action, and the agent scores exactly like one that chose not to trade +// (issue #54 -- four bundled agents shipped with that bug). +import type { AgentAction, AgentObservation } from "@eris/sdk"; +import { affordable, canFund } from "../lib/affordable.js"; + +// Only trade when a venue is this far from fair. Too low and fees eat the edge; too high and the +// agent sits out the run. A good first thing to tune. +const MIN_GAP = 0.001; // 10 bps +// Fraction of the per-round limit to send. Flat on purpose -- scaling this with the gap is an +// obvious improvement. +const SIZE_BPS = 1000n; // 10% + +type Venue = { + swapType: "swap" | "balancerSwap" | "curveSwap"; + price: number; +}; + +export function decide(obs: AgentObservation): AgentAction | null { + const fair = obs.fairPriceUsdcPerWeth; + const p = obs.protocols ?? {}; + const venues: Venue[] = []; + if (p.uniswap?.pool) + venues.push({ swapType: "swap", price: p.uniswap.pool.priceUsdcPerWeth }); + if (p.balancer) + venues.push({ + swapType: "balancerSwap", + price: p.balancer.priceUsdcPerWeth, + }); + if (p.curve) + venues.push({ swapType: "curveSwap", price: p.curve.priceUsdcPerWeth }); + + let best: Venue | undefined; + let bestGap = MIN_GAP; + for (const v of venues) { + if (!Number.isFinite(v.price) || v.price <= 0) continue; + const gap = Math.abs(fair / v.price - 1); + // Pool below fair -> WETH is cheap -> buy it with USDC. Above -> sell WETH, which needs WETH. + if (gap <= bestGap || !canFund(obs, v.price < fair ? "USDC" : "WETH")) + continue; + bestGap = gap; + best = v; + } + if (!best) return { type: "noop", reason: "no fundable gap worth taking" }; + + const tokenIn = best.price < fair ? "USDC" : "WETH"; + const amountIn = affordable( + obs, + tokenIn, + (BigInt( + tokenIn === "WETH" ? obs.limits.maxWethInWei : obs.limits.maxUsdcInUnits, + ) * + SIZE_BPS) / + 10_000n, + ); + if (amountIn === 0n) return { type: "noop", reason: "size below the floor" }; + + return { + type: best.swapType, + tokenIn, + amountIn: amountIn.toString(), + maxPriorityFeePerGasWei: obs.limits.defaultPriorityFeePerGasWei, + slippageBps: 75, + }; +} diff --git a/example/agents/my-arb/improve.md b/example/agents/my-arb/improve.md new file mode 100644 index 0000000..aca55af --- /dev/null +++ b/example/agents/my-arb/improve.md @@ -0,0 +1,47 @@ +--- +name: my-arb +description: The starting-point sample — a naive cross-venue arb, with an LLM improving it in-run. +reviseEveryBlocks: 60 +--- + +You are maintaining a cross-venue arbitrage strategy. It runs on every block without you. Decide +whether the code should change, and if so, what to change it to. + +The strategy you were shipped is deliberately naive: it takes the widest gap above a fixed +threshold, in whichever direction it can fund, at a flat fraction of the limit. It ignores fees, it +ignores how much the pool will move against it, and it never plans a round trip. Those are the +obvious things to improve — but improve them because the evidence says so, not because the list +above says so. + +## When to leave it alone + +Return `"executorTs": null` unless you can name what is going wrong. + +- **It is making money.** Being up is not a problem to solve. +- **The loss is the market.** A strategy holding inventory loses when the price falls, while doing + exactly what it was told to. Look at whether the trades were bad, not at the sign of the number. +- **Too little evidence.** A few decisions since the last revision is noise. + +Over-correcting is the failure mode to watch for: tighten after every bad patch and the strategy +stops taking the trades that pay for the run. Doing nothing is also a way to lose. + +## What to look at + +- **Rejections and decide errors** are bugs. Fix them first. +- **Long runs of the same noop reason** mean a condition never fires. The reason string tells you + which one. +- **Frequent trades that bleed slowly** mean the edge does not cover the round trip. + +## Constraints + +- Only `obs`, `ctx` and standard JavaScript. No `require`, `import`, `process` or `fetch`. +- **Check balances before choosing a direction.** `obs.balances.wethWei` starts at zero. An action + the runtime rejects scores the same as doing nothing. +- Respect `obs.limits`. +- Return one action object or `null`. `ctx.log({ reason })` records why, and you will read it back. + +## Undoing a change + +Nothing reverts automatically. If one of your rewrites made things worse, return +`{"notes": "...", "revertTo": }` — the context lists every version, when it went in, and +what the agent was worth at the time. diff --git a/example/agents/my-arb/prompt.md b/example/agents/my-arb/prompt.md deleted file mode 100644 index 35a156d..0000000 --- a/example/agents/my-arb/prompt.md +++ /dev/null @@ -1,37 +0,0 @@ ---- -name: my-arb -description: cross-venue arb; push toward fair above 30bps -intervalMs: 5000 -model: gpt-oss:120b ---- -# Mission - -You are a cross-venue arbitrage bot (the participant-template sample). Only when -the deviation between fair and a venue is large enough, swap toward fair. - -## Decision procedure (every cycle) - -1. Compare WETH prices on uniswap / balancer / curve against fair; pick the - venue with the largest |fair/price - 1| -2. If deviation <= 30bps: {"type":"noop","reason":"gap<=30bps"} -3. Direction: - - price < fair (cheap) -> buy WETH with USDC (tokenIn="USDC") - - price > fair (rich) -> sell WETH (tokenIn="WETH"; **noop if no balance**) -4. Size: notional at most 2 WETH equivalent per trade, and never above the - per-round caps (maxWethInWei / maxUsdcInUnits) or your balance. Use decimal - integer strings -5. Bidding: up to 10% of expected profit (size USD x deviation), bidding just - above competition.maxCompetitorPriorityFeeWei. If that breaks even, noop -6. The action is one swap of the chosen venue's type (swap / balancerSwap / - curveSwap), slippageBps 75 - -## Explicit noop criteria - -- deviation <= 30bps / zero tokenIn-side balance / bidding breaks even / - no confidence - -## Revision invariants (for self-improvement) - -- Keep "toward fair only" and "one action per cycle". -- Tunable: threshold (30bps), notional cap, bid rate. Ground changes in the - measured revert rate and PnL. diff --git a/example/agents/noop/prompt.md b/example/agents/noop/prompt.md deleted file mode 100644 index cea84bf..0000000 --- a/example/agents/noop/prompt.md +++ /dev/null @@ -1,29 +0,0 @@ ---- -name: noop -description: Do nothing (baseline) ---- -# Mission - -You are the measurement baseline bot. **Never trade, under any circumstances.** - -## Why you exist - -Every other agent's performance is interpreted as "how much it beat noop". -Your final portfolio value reflects pure price drift (beta) - the yardstick for -"what would have happened doing nothing". The moment noop trades, the whole -run loses comparability. - -## Decision procedure (every cycle) - -1. Regardless of the observation, return: - {"type":"noop","reason":"baseline"} - -## Explicit noop criteria - -- Always noop. No exception for any gap size or any balance. - -## Revision invariants (for self-improvement) - -- Never add trading behavior to this prompt. This agent is exempt from - improvement (you do not sharpen a yardstick). -- Only clarity-of-wording edits are allowed. diff --git a/example/agents/random/prompt.md b/example/agents/random/prompt.md deleted file mode 100644 index 0d35dd7..0000000 --- a/example/agents/random/prompt.md +++ /dev/null @@ -1,42 +0,0 @@ ---- -name: random -description: Swap at random (noise benchmark) ---- -# Mission - -You are the "random trading" noise benchmark. You never look at the market; -you trade probabilistically, measuring the pure cost of uninformed trading -(fees + slippage + price impact). - -## Why you exist - -If a strategy cannot beat random's PnL, its signal does not cover its costs. -Random measures "how much you lose by trading with zero information". - -## Decision procedure (every cycle) - -1. With 35% probability return {"type":"noop","reason":"random skip"} -2. Otherwise pick a direction 50/50: - - Sell WETH: tokenIn="WETH" (if balances.wethWei is 0, fall back to buying) - - Sell USDC (= buy WETH): tokenIn="USDC" -3. Size: uniform random 1-51% of min(your balance, per-round cap - - limits.maxWethInWei for WETH, limits.maxUsdcInUnits for USDC), - rounded to a decimal integer string -4. Action: - {"type":"swap","tokenIn":"USDC","amountIn":"","slippageBps":75, - "maxPriorityFeePerGasWei":""} - -## Unit notes - -- amountIn is a decimal integer string. WETH in wei (1e18), USDC in units (1e6). - Example: 20% of a 5,000 USDC cap ("5000000000") -> "1000000000" - -## Explicit noop criteria - -- The 35% skip roll. Both sides' balances are zero. - -## Revision invariants (for self-improvement) - -- Never use market information (gap, fair, competition) in decisions. Making - this bot smart destroys its purpose as a noise yardstick. -- Only the skip rate and the size distribution may be tuned. diff --git a/example/agents/runtime/agentLog.ts b/example/agents/runtime/agentLog.ts index 80bbe74..31d633c 100644 --- a/example/agents/runtime/agentLog.ts +++ b/example/agents/runtime/agentLog.ts @@ -30,7 +30,7 @@ export type AgentLog = (entry: AgentLogEntry) => void; // Low-level append to runs//agents/.jsonl. // Shared implementation so the action log (createAgentLog) and mempool self-reports (send.ts) // write to the same file (no suffix), while the LLM conversation log (bot.ts's -// ERIS_PROMPT_LOG_CALLS) writes to a separate file (suffix ".llm"). +// ERIS_IMPROVE_LOG_CALLS) writes to a separate file (suffix ".llm"). export function createJsonlAppender( runDir: string | undefined, agentId: string, diff --git a/example/agents/runtime/bot.ts b/example/agents/runtime/bot.ts index f7e7a6c..55fcb11 100644 --- a/example/agents/runtime/bot.ts +++ b/example/agents/runtime/bot.ts @@ -6,18 +6,20 @@ * env ERIS_AGENT_DIR. bot.ts decides how to run from that directory's contents: * - agent.ts exports run(ctx) -> self-driven: pass ctx and delegate (no loop) * - agent.ts exports decide() -> rule strategy: drive a read->decide->send loop - * - prompt.md only -> prompt type: have the LLM emit an action every decision + * - agent.ts + improve.md -> self-improving: the same loop, plus an LLM that periodically + * rewrites the strategy out of the trade path (ADR 0018) * - * An agent that ships both agent.ts and prompt.md provides both ways of running (ADR 0015 §2's - * "agent.ts takes precedence when both are present" is the default). Switch it via the roster env: - * ERIS_AGENT_MODE=prompt run via prompt.md (LLM-driven) even when agent.ts exists - * ERIS_PROMPT_REVISE_EVERY= in prompt mode, every N decision cycles the LLM self-revises - * the prompt body (default 0 = off; revised versions are saved to - * runs//agents/.prompt.v.md and used by later cycles) - * ERIS_PROMPT_REVISE_PERSIST=1 also write the revision back to the agent directory's prompt.md - * ERIS_PROMPT_LOG_CALLS=1 record the raw LLM conversation (system / sent messages / - * raw response / errors) to runs//agents/.llm.jsonl - * (opt-in debug log for prompt tuning) + * Prompt mode (an LLM producing an action every decision) was removed in ADR 0018: measured at + * 8-28 blocks per decision and 1/64 the actions of the same strategy in rule mode, it could not + * compete. The LLM now improves the strategy instead of driving it. + * + * ERIS_AGENT_FROZEN=1 ignore improve.md and run the strategy unchanged. This is the + * frozen control every roster needs (ADR 0018 §5), without + * duplicating the agent directory + * ERIS_LLM_MODEL= backend for the revision call (improve.md frontmatter wins) + * ERIS_IMPROVE_LOG_CALLS=1 record the raw revision exchange (system / context / response) + * to runs//agents/.llm.jsonl. Off by default: it holds + * every generated strategy in full * * Environment variables (passed by the environment; the ADR 0006 contract is unchanged): * ERIS_AGENT_ID / ERIS_AGENT_DIR / ERIS_AGENT_PRIVATE_KEY / ERIS_RPC_URL / @@ -46,22 +48,23 @@ import type { ProtocolId, } from "@eris/sdk/types.js"; import { createAgentLog, createJsonlAppender } from "./agentLog.js"; -import { callLlm, type LlmMessage } from "./llm.js"; +import { callLlm } from "./llm.js"; import { + buildRevisionContext, buildRevisionSystem, - buildRevisionUser, - buildSystemPrompt, - buildUserMessage, - DEFAULT_PROMPT_INTERVAL_MS, - DEFAULT_PROMPT_MODEL, - DEFAULT_PROMPT_REVISE_EVERY, - loadPromptAgent, - type RecentAction, -} from "./prompt.js"; + compileExecutor, + effectiveReviseInterval, + loadImproveAgent, + MAX_REVISIONS_PER_RUN, + parseRevision, + type RevisionOutcome, + type StrategyVersion, +} from "./improve.js"; import { createMempoolLog, Sender } from "./send.js"; import { Reader } from "./read.js"; -const LLM_MAX_ATTEMPTS = 4; // retry cap on validation failure (append the error to the conversation; ADR 0015 §4) +// Backend for the revision call when neither improve.md nor the roster names one. +const DEFAULT_IMPROVE_MODEL = "gpt-oss:120b"; async function main(): Promise { const privateKey = process.env.ERIS_AGENT_PRIVATE_KEY as Hex | undefined; @@ -132,52 +135,62 @@ async function main(): Promise { }); // ---- resolve the agent module (1 agent = 1 directory) ---- - // Default is agent.ts precedence (ADR 0015 §2). A co-located agent can be switched to - // LLM-driven (prompt.md) via ERIS_AGENT_MODE=prompt (both ways of running are always provided). + // agent.ts is always the strategy (ADR 0015 §2). If improve.md sits beside it, the same strategy + // runs at the same speed and an LLM is periodically offered the chance to rewrite it (ADR 0018). + // The retired prompt mode put the LLM in the trade path instead, which cost 8-28 blocks per + // decision -- 1/64 the actions of the same strategy in rule mode (ADR 0017 §5 B1). + // A roster still asking for prompt mode would otherwise run as a plain rule agent and look fine, + // which is the worst outcome: the participant thinks an LLM is involved and nothing says otherwise. + const retired = [ + "ERIS_AGENT_MODE", + "ERIS_PROMPT_REVISE_EVERY", + "ERIS_PROMPT_REVISE_PERSIST", + "ERIS_PROMPT_LOG_CALLS", + ].filter((k) => process.env[k] !== undefined); + if (retired.length > 0) { + process.stderr.write( + `[bot] ${retired.join(", ")} is retired (ADR 0018 removed prompt mode). An agent is agent.ts, ` + + `optionally with improve.md beside it for LLM-driven self-improvement; ` + + `use ERIS_AGENT_FROZEN=1 to run it without the improvement loop\n`, + ); + process.exit(1); + return; + } const agentTsPath = join(agentDir, "agent.ts"); const hasAgentTs = existsSync(agentTsPath); - const hasPrompt = existsSync(join(agentDir, "prompt.md")); - const forcedMode = process.env.ERIS_AGENT_MODE; - if ( - forcedMode !== undefined && - forcedMode !== "agent" && - forcedMode !== "prompt" - ) { + const hasImprove = existsSync(join(agentDir, "improve.md")); + // Opt out of the improvement loop while keeping the same directory: the frozen control that + // ADR 0018 §5 requires in every roster is this flag, not a second copy of the agent. + const frozen = process.env.ERIS_AGENT_FROZEN === "1"; + if (!hasAgentTs) { process.stderr.write( - `[bot] ERIS_AGENT_MODE must be "agent" or "prompt" (got: ${forcedMode})\n`, + existsSync(join(agentDir, "prompt.md")) + ? `[bot] ${agentDir} has prompt.md but no agent.ts. Prompt mode was removed (ADR 0018): ` + + `an agent is agent.ts, optionally with improve.md beside it\n` + : `[bot] ${agentDir} has no agent.ts (ADR 0015 §2 / ADR 0018 §1)\n`, ); process.exit(1); return; } - let mode: "run" | "decide" | "prompt"; - let agentModule: AgentModule | null = null; - if (forcedMode === "prompt" ? false : hasAgentTs) { - agentModule = (await import( - pathToFileURL(agentTsPath).href - )) as AgentModule; - if (typeof agentModule.run === "function") mode = "run"; - else if (typeof agentModule.decide === "function") mode = "decide"; - else { - process.stderr.write( - `[bot] ${agentTsPath} must export decide() or run(ctx)\n`, - ); - process.exit(1); - return; - } - } else if (hasPrompt) { - mode = "prompt"; - } else { + const agentModule = (await import( + pathToFileURL(agentTsPath).href + )) as AgentModule; + let mode: "run" | "decide" | "improve"; + if (typeof agentModule.run === "function") mode = "run"; + else if (typeof agentModule.decide === "function") + mode = hasImprove && !frozen ? "improve" : "decide"; + else { process.stderr.write( - forcedMode === "prompt" - ? `[bot] ERIS_AGENT_MODE=prompt but ${agentDir} has no prompt.md\n` - : `[bot] ${agentDir} has neither agent.ts nor prompt.md (ADR 0015 §2)\n`, + `[bot] ${agentTsPath} must export decide() or run(ctx)\n`, ); process.exit(1); return; } - if (forcedMode === "agent" && !hasAgentTs) { + if (hasImprove && typeof agentModule.run === "function") { + // run(ctx) owns its own loop, so there is no decide to swap out. process.stderr.write( - `[bot] ERIS_AGENT_MODE=agent but ${agentDir} has no agent.ts\n`, + `[bot] ${agentDir} has improve.md but exports run(ctx); self-improvement applies to ` + + `decide() strategies only (ADR 0018 §1)\n`, ); process.exit(1); return; @@ -207,18 +220,55 @@ async function main(): Promise { }; // ---- driving decide (rule strategy) ---- + // Held in a variable rather than called through agentModule so the improvement loop can swap the + // strategy underneath a running agent (ADR 0018). In every other mode this is just agentModule.decide. + let activeDecide = agentModule.decide; let deciding = false; + // What the strategy actually did recently. The self-improvement loop shows this to the model as + // the evidence for a rewrite, so it has to hold the decisions themselves -- recording only that a + // block happened rendered every entry as "no action" and left the model with nothing to reason + // about. Populated here rather than from the observation stream because that is where the outcome + // of a decision (an action, or an error) actually exists. + const recentDecisions: Array<{ + round: number; + action?: unknown; + reason?: string; + }> = []; + const rememberDecision = (entry: { + round: number; + action?: unknown; + reason?: string; + }): void => { + recentDecisions.push(entry); + if (recentDecisions.length > 32) recentDecisions.shift(); + }; const invokeDecide = async (obs: AgentObservation): Promise => { - if (!agentModule?.decide || deciding) return; + if (!activeDecide || deciding) return; deciding = true; try { - const action = await agentModule.decide(obs, ctx); + const action = await activeDecide(obs, ctx); if (action) ctx.submit(action); + rememberDecision({ round: obs.round, action: action ?? undefined }); + // Record the decision not to trade, with its reason. send.ts drops noops before they reach + // the log, so a strategy that passes every block used to leave nothing behind at all -- and an + // empty agent log cannot distinguish "never started" from "looked and declined". Both happened + // during this branch's calibration runs and both cost time to diagnose. + const declined = + action === null || + action === undefined || + (action as { type?: string }).type === "noop"; + if (declined) + agentLog({ + round: obs.round, + action: { type: "noop" }, + reason: + (action as { reason?: string } | null)?.reason ?? + "decide returned nothing", + }); } catch (error) { - agentLog({ - round: obs.round, - reason: `decide error: ${error instanceof Error ? error.message : String(error)}`, - }); + const reason = `decide error: ${error instanceof Error ? error.message : String(error)}`; + rememberDecision({ round: obs.round, reason }); + agentLog({ round: obs.round, reason }); } finally { deciding = false; } @@ -257,8 +307,10 @@ async function main(): Promise { // a subscriber failure must not affect the observation loop } } - // A decide type without intervalMs runs "once per new block" (same cadence as the old shim + readline). - if (mode === "decide" && intervalMs === undefined) + // A decide type without intervalMs runs "once per new block" (same cadence as the old shim + + // readline). Self-improving agents are on this path too -- that is the point: the trading loop + // is exactly as fast as a rule agent's, and only the strategy behind it changes (ADR 0018). + if ((mode === "decide" || mode === "improve") && intervalMs === undefined) void invokeDecide(snap.observation); } catch (error) { process.stderr.write( @@ -299,229 +351,231 @@ async function main(): Promise { return; } - if (mode === "prompt") { - await runPromptLoop(); + if (mode === "improve") { + await runImproveLoop(); } - // ---- prompt type: LLM every decision (Hermes JSON mode + validation retry; ADR 0015 §4) ---- - // When ERIS_PROMPT_REVISE_EVERY > 0, every N decision cycles the LLM self-revises the prompt body - // (self-improvement; revised versions are saved to runs//agents/.prompt.v.md and - // used by later cycles. ERIS_PROMPT_REVISE_PERSIST=1 also writes back to the agent directory's prompt.md). - async function runPromptLoop(): Promise { - const promptAgent = loadPromptAgent(agentDir); + // ---- self-improving type: the LLM rewrites the strategy, out of the trade path (ADR 0018) ---- + // + // The block loop above already drives activeDecide every block. All this does is periodically hand + // the model the current source plus how it has been doing, and swap activeDecide if what comes + // back is better. Every accept, decline, rejection and rollback is logged, because the previous + // attempt at this (deleted src/llm) shipped a rollback that never once fired and nobody noticed. + async function runImproveLoop(): Promise { + const improveAgent = loadImproveAgent(agentDir); const model = - promptAgent.model ?? process.env.ERIS_LLM_MODEL ?? DEFAULT_PROMPT_MODEL; - const schema = agentActionSchemaFor(config.enabledProtocols); - const jsonSchema = actionJsonSchema(config.enabledProtocols); - let body = promptAgent.body; - const rebuildSystem = (): string => - buildSystemPrompt( - { ...promptAgent, body }, - config.enabledProtocols, - jsonSchema, - ); - let system = rebuildSystem(); - const reviseEveryRaw = Number( - process.env.ERIS_PROMPT_REVISE_EVERY ?? DEFAULT_PROMPT_REVISE_EVERY, + improveAgent.model ?? process.env.ERIS_LLM_MODEL ?? DEFAULT_IMPROVE_MODEL; + // The raw exchange, opt-in. The outcome log says a revision was rejected or rolled back; only + // this says what was asked and what came back, which is what prompt tuning actually needs. + // Off by default because it holds every generated strategy in full. + const llmLog = + process.env.ERIS_IMPROVE_LOG_CALLS === "1" + ? createJsonlAppender(runDir, agentId, ".llm") + : undefined; + const { blocks: reviseEvery, clamped } = effectiveReviseInterval( + improveAgent.reviseEveryBlocks, + config.runBlocks, ); - const reviseEvery = - Number.isFinite(reviseEveryRaw) && reviseEveryRaw > 0 - ? Math.floor(reviseEveryRaw) - : 0; - const revisePersist = process.env.ERIS_PROMPT_REVISE_PERSIST === "1"; - const recent: RecentAction[] = []; - const interval = promptAgent.intervalMs ?? DEFAULT_PROMPT_INTERVAL_MS; - let cycling = false; - let lastDecidedRound = -1; - let decidedCycles = 0; - let revision = 0; - let initialValueUsdc: number | null = null; + if (clamped) + agentLog({ + reason: + `revision cadence clamped from ${improveAgent.reviseEveryBlocks} to ${reviseEvery} blocks ` + + `(a co-located run shares one LLM budget; ADR 0018 §4)`, + }); - // ---- LLM conversation log (opt-in via ERIS_PROMPT_LOG_CALLS=1; diagnostic aid for ADR 0015 §4) ---- - // So that "what part of the observation the LLM read and what it returned" can be traced after - // the run, record the raw conversation to runs//agents/.llm.jsonl. The system prompt - // is large and identical across decisions, so write its full text only on the first call and right - // after each self-revision as kind:"llm_system", and have each per-call kind:"llm_call" reference - // it by revision number (the validation-retry exchanges are kept inside messages). - const llmCallLog = - process.env.ERIS_PROMPT_LOG_CALLS === "1" - ? createJsonlAppender(runDir, agentId, ".llm") - : null; - const logSystem = (): void => - llmCallLog?.({ kind: "llm_system", revision, system }); - const loggedCallLlm = async ( - meta: Record, - req: Parameters[0], - ): Promise => { - if (!llmCallLog) return callLlm(req); - try { - const response = await callLlm(req); - llmCallLog({ - kind: "llm_call", - ...meta, - revision, - model: req.model, - messages: req.messages, - response, - }); - return response; - } catch (error) { - llmCallLog({ - kind: "llm_call", - ...meta, - revision, - model: req.model, - messages: req.messages, - error: error instanceof Error ? error.message : String(error), - }); - throw error; - } + // Every version that has run, version 0 being the strategy the participant shipped. Kept whole so + // the model can revert to any of them by number rather than by reproducing source, and so the + // log can be read back afterwards. + const versions: Array< + StrategyVersion & { executor: typeof activeDecide } + > = [ + { + version: 0, + source: readFileSync(agentTsPath, "utf8"), + notes: "the strategy as submitted", + installedAtBlock: 0, + valueAtInstall: null, + executor: activeDecide, + }, + ]; + const current = () => versions[versions.length - 1]; + let currentVersion = 0; + let revisions = 0; + // Block of the last revision opportunity. Seeded from the first observation, not 0: obs.round is + // the absolute chain block (read.ts passes `round: bn`), so starting at 0 made the very first + // observation satisfy `block - lastBlock >= reviseEvery` and fire a revision before the strategy + // had traded a single block -- with no performance to reason about, burning one of the + // participant's revisions on nothing. + let lastRevisionBlock: number | null = null; + // Value at the moment of the last revision, to judge whether that revision helped. + let valueAtRevision: number | null = null; + let initialValue: number | null = null; + ctx.onObservation((obs) => { + const value = obs.inventory?.valueUsdc; + if (typeof value === "number" && initialValue === null) + initialValue = value; + }); + + const valueNow = (): number | null => { + const v = latestObservation?.inventory?.valueUsdc; + return typeof v === "number" ? v : null; + }; + + const record = (outcome: RevisionOutcome, block: number): void => { + // `state`, not `signals`: signals is numeric-only, and a revision record is mostly text + // (the model's notes, a rejection reason). Post-run diagnosis reads this. + agentLog({ + round: block, + reason: `revision ${outcome.kind}`, + state: { ...outcome }, + }); }; - logSystem(); - // Prompt revision (self-improvement). Run inside the same cycling lock as the decision cycle so it never races the decision. - const revisePrompt = async (obs: AgentObservation): Promise => { + let revising = false; + const maybeRevise = async (block: number): Promise => { + if (revising || revisions >= MAX_REVISIONS_PER_RUN) return; + revising = true; try { - const reviseSystem = buildRevisionSystem(promptAgent); - const text = await loggedCallLlm( - // The revision system prompt differs from the decision one, so include it directly in the record. - { purpose: "revise", round: obs.round, system: reviseSystem }, - { - model, - system: reviseSystem, - messages: [ - { - role: "user", - content: buildRevisionUser(body, recent.slice(-16), { - cycles: decidedCycles, - initialValueUsdc, - currentValueUsdc: obs.inventory.valueUsdc, - recentRevertRate: obs.competition?.recentRevertRate, - recentSampleSize: obs.competition?.recentSampleSize, - }), - }, - ], - json: false, // revision is free text (markdown body) - }, - ); - const next = stripFences(text).trim(); - // Discard a broken revision (empty / extreme length) and keep the current prompt (fail-closed). - if (next.length < 40 || next.length > 20_000) - throw new Error(`revised body rejected (length=${next.length})`); - revision++; - body = next; - system = rebuildSystem(); - logSystem(); // record the full revised system prompt in the conversation log too, versioned - agentLog({ - round: obs.round, - reason: `prompt revised v${revision}`, - state: { - kind: "prompt_revision", - revision, - cycles: decidedCycles, - valueUsdc: obs.inventory.valueUsdc, - }, + // Nothing is judged here. Whether a revision helped, and whether to undo it, is the model's + // call -- an automatic revert needs a threshold and there is no defensible one (ADR 0018 §5). + // What the harness owes the model is the evidence: the history, and the value at each point. + const value = valueNow(); + const system = buildRevisionSystem(improveAgent, current().source); + const context = buildRevisionContext({ + block, + valueUsdc: value ?? 0, + initialValueUsdc: initialValue ?? 0, + sinceLastRevisionUsdc: + valueAtRevision !== null && value !== null + ? value - valueAtRevision + : null, + currentVersion, + history: versions.map(({ executor: _executor, ...v }) => v), + recent: recentDecisions, + observation: latestObservation, }); - // Keep the revision history in the run directory, versioned (primary source for post-run diagnostics/comparison). - if (runDir) { - const dir = join(runDir, "agents"); - mkdirSync(dir, { recursive: true }); - writeFileSync(join(dir, `${agentId}.prompt.v${revision}.md`), body); + revisions++; + let raw: string; + try { + raw = await callLlm({ + model, + system, + messages: [{ role: "user", content: context }], + }); + llmLog?.({ kind: "revision_call", block, model, system, context, raw }); + } catch (error) { + const reason = error instanceof Error ? error.message : String(error); + llmLog?.({ kind: "revision_call", block, model, system, context, error: reason }); + throw error; } - // Optional: write back to the agent directory's prompt.md (keep the frontmatter unchanged). - if (revisePersist) { - const path = join(agentDir, "prompt.md"); - const raw = readFileSync(path, "utf8"); - const m = raw.match(/^(---\n[\s\S]*?\n---\n)/); - if (m) writeFileSync(path, `${m[1]}${body}\n`); + let parsedJson: unknown; + try { + parsedJson = JSON.parse(stripFences(raw)); + } catch { + record( + { kind: "rejected", reason: "response was not valid JSON" }, + block, + ); + return; } - } catch (error) { - agentLog({ - round: obs.round, - reason: `prompt revision failed: ${error instanceof Error ? error.message : String(error)}`, - }); - } - }; - - const cycle = async (): Promise => { - const obs = latestObservation; - if (cycling || !obs || obs.round === lastDecidedRound) return; - cycling = true; - lastDecidedRound = obs.round; - initialValueUsdc ??= obs.inventory.valueUsdc; - try { - const messages: LlmMessage[] = [ - { role: "user", content: buildUserMessage(obs, recent.slice(-8)) }, - ]; - let lastError = ""; - let decided = false; - for (let attempt = 1; attempt <= LLM_MAX_ATTEMPTS; attempt++) { - let text: string; - try { - text = await loggedCallLlm( - { purpose: "decision", round: obs.round, attempt }, - { model, system, messages, jsonSchema }, + const parsed = parseRevision(parsedJson); + if (!parsed.ok) { + record({ kind: "rejected", reason: parsed.reason }, block); + return; + } + if (parsed.revision.revertTo !== null) { + const target = versions.find( + (v) => v.version === parsed.revision.revertTo, + ); + if (!target) { + record( + { + kind: "rejected", + reason: `revertTo ${parsed.revision.revertTo}: no such version (have ${versions + .map((v) => v.version) + .join(", ")})`, + }, + block, ); - } catch (error) { - lastError = error instanceof Error ? error.message : String(error); - break; // a failure of the call itself is not retried; skip this cycle - } - let parsed: unknown; - try { - parsed = JSON.parse(stripFences(text)); - } catch { - lastError = "response was not valid JSON"; - messages.push({ role: "assistant", content: text }); - messages.push({ - role: "user", - content: `Your response was not valid JSON. Respond with exactly one JSON object matching the . Error: ${lastError}`, - }); - continue; - } - const check = schema.safeParse(parsed); - if (!check.success) { - // Append the error (what violated the schema) to the conversation and retry (Hermes pattern). - lastError = check.error.issues - .map((i) => `${i.path.join(".") || "(root)"}: ${i.message}`) - .join("; "); - messages.push({ role: "assistant", content: text }); - messages.push({ - role: "user", - content: `Your action failed schema validation: ${lastError}. Fix it and respond with exactly one JSON object matching the .`, - }); - continue; + return; } - const action = parsed as Record; - agentLog({ round: obs.round, action, reason: "llm decision" }); - recent.push({ round: obs.round, action }); - if (recent.length > 16) recent.shift(); - ctx.submit(action); - decided = true; - break; - } - if (!decided) { - // fail-closed: exceeding the cap records this cycle as skipped (noop). An invalid action never reaches the chain. - agentLog({ - round: obs.round, - action: { type: "noop" }, - reason: `llm cycle skipped: ${lastError}`, - }); - recent.push({ - round: obs.round, - action: { type: "noop" }, - note: `skipped (${lastError.slice(0, 120)})`, + // Re-installed as a new version rather than by rewinding the list: the history is a record + // of what ran and when, and rewinding it would erase the fact that the reverted version + // ever did. + currentVersion += 1; + activeDecide = target.executor; + versions.push({ + ...target, + version: currentVersion, + notes: `reverted to v${target.version}: ${parsed.revision.notes}`, + installedAtBlock: block, + valueAtInstall: value, }); - if (recent.length > 16) recent.shift(); + valueAtRevision = value; + record( + { + kind: "reverted", + to: target.version, + from: currentVersion - 1, + notes: parsed.revision.notes, + }, + block, + ); + return; } - // ---- self-improvement: revise the prompt body every N decision cycles (within the same lock = serial with the decision) ---- - decidedCycles++; - if (reviseEvery > 0 && decidedCycles % reviseEvery === 0) - await revisePrompt(obs); + if (parsed.revision.executorTs === null) { + record({ kind: "declined", notes: parsed.revision.notes }, block); + return; + } + const compiled = compileExecutor(parsed.revision.executorTs); + if (!compiled.ok) { + record({ kind: "rejected", reason: compiled.reason }, block); + return; + } + currentVersion += 1; + activeDecide = compiled.executor; + versions.push({ + version: currentVersion, + source: parsed.revision.executorTs, + notes: parsed.revision.notes, + installedAtBlock: block, + valueAtInstall: value, + executor: compiled.executor, + }); + valueAtRevision = value; + record( + { + kind: "installed", + version: currentVersion, + notes: parsed.revision.notes, + }, + block, + ); + } catch (error) { + record( + { + kind: "rejected", + reason: `revision failed: ${error instanceof Error ? error.message : String(error)}`, + }, + block, + ); } finally { - cycling = false; + revising = false; } }; - setInterval(() => void cycle(), interval); + + ctx.onObservation((obs) => { + const block = obs.round; + // Seed the baseline from the first block seen rather than 0. obs.round is the absolute chain + // block, so a 0 baseline made the first observation instantly "overdue" for a revision. + if (lastRevisionBlock === null) { + lastRevisionBlock = block; + return; + } + if (block - lastRevisionBlock < reviseEvery) return; + lastRevisionBlock = block; + void maybeRevise(block); + }); } } diff --git a/example/agents/runtime/improve.ts b/example/agents/runtime/improve.ts new file mode 100644 index 0000000..2c44a73 --- /dev/null +++ b/example/agents/runtime/improve.ts @@ -0,0 +1,381 @@ +// Self-improving agent: the LLM rewrites the strategy, it does not make the trades (ADR 0018). +// +// The trading loop stays where it was -- `decide(obs, ctx)` on every block, at rule-agent speed. +// Out of band, an LLM is periodically handed the current executor source plus how it has been doing, +// and may return a replacement. That is the whole difference from the retired prompt mode, where the +// LLM was in the trade path and a decision cost one round trip: measured at 8-28 blocks per decision +// and 1/64 the actions of the same strategy in rule mode (ADR 0017 §5 B1). +// +// Three guards, all of them there because the deleted `src/llm` two-layer machinery lacked or +// under-used them (it lost to frozen strategies on multi-seed validation and its rollback never +// fired in 18 runs): +// +// 1. Generated code passes the cheatcode static check before it is installed. An LLM-authored +// strategy is not trusted code. +// 2. A revision that fails to compile, or throws on its first call, is not installed at all. +// 3. A revision that performs worse than what it replaced is rolled back, and every accept, +// reject and rollback is written to the agent log so "did self-improvement do anything" is +// answerable from a single run rather than from a study. +import { existsSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { createContext, Script } from "node:vm"; +import { parse as parseYaml } from "yaml"; +import { findCheatcodeUsage } from "@eris/sdk/strategyStaticCheck.js"; +import type { AgentContext } from "@eris/sdk/agent.js"; +import type { AgentAction, AgentObservation } from "@eris/sdk/types.js"; + +// How often the LLM is offered a chance to revise, in blocks, when improve.md does not say. +export const DEFAULT_REVISE_EVERY_BLOCKS = 60; +// Wall-clock bound on one call into a generated strategy. Blocks are 2 s in production, so a +// strategy that has not answered in this long has already missed its block. +export const EXECUTOR_TIMEOUT_MS = 2000; +// Ceiling the operator puts on the participant's declaration. A co-located run shares one LLM +// budget, so "revise every block" from one participant would starve the field; a declaration below +// this is honored as-is, above it is clamped and the clamp is recorded. +export const MAX_REVISIONS_PER_RUN = 12; + +export type ImproveAgent = { + name: string; + description: string; + // Blocks between revision opportunities. The participant's lever over cadence -- declarative, so + // it costs no LLM call to evaluate (ADR 0018 §4). + reviseEveryBlocks: number; + model?: string; + body: string; +}; + +// What the LLM returns. Three answers, all legitimate: +// executorTs: "" install this as the new strategy +// executorTs: null leave the strategy alone (how a prompt says "do not touch a winner") +// revertTo: go back to an earlier version +// +// Reverting is the model's call rather than the harness's. An automatic "roll back when value went +// down" needs a threshold, and there is no defensible one: the previous implementation's never +// fired in 18 runs, and the obvious opposite (any loss at all) reverts every revision in a regime +// where everyone is losing. The model already sees the PnL since each revision and the notes it +// wrote at the time, so the judgment belongs there -- and improve.md is where a participant states +// how to make it. Timing is unchanged either way: both fire at a revision opportunity. +export type StrategyRevision = { + version: number; + notes: string; + executorTs: string | null; + revertTo: number | null; +}; + +// One installed strategy and what happened after it. Handed to the model so a revert is an informed +// choice rather than a guess, and kept in the log so a run can be read back. +export type StrategyVersion = { + version: number; + source: string; + notes: string; + installedAtBlock: number; + valueAtInstall: number | null; +}; + +export type RevisionOutcome = + | { kind: "installed"; version: number; notes: string } + | { kind: "declined"; notes: string } + | { kind: "rejected"; reason: string } + | { kind: "reverted"; to: number; from: number; notes: string }; + +// improve.md: the improvement prompt. Not a renamed prompt.md -- prompt.md said "given this +// observation, what do you do", improve.md says "when, on what evidence, and how should the strategy +// change" (ADR 0018 §1). +export function loadImproveAgent(agentDir: string): ImproveAgent { + const path = join(agentDir, "improve.md"); + if (!existsSync(path)) throw new Error(`improve.md not found in ${agentDir}`); + const raw = readFileSync(path, "utf8"); + const m = raw.match(/^---\n([\s\S]*?)\n---\n?([\s\S]*)$/); + if (!m) + throw new Error( + `${path}: frontmatter (---) is required (name / description mandatory)`, + ); + const fm = parseYaml(m[1]) as Record | null; + if (!fm || typeof fm !== "object") + throw new Error(`${path}: frontmatter must be a YAML mapping`); + if (typeof fm.name !== "string" || fm.name.trim() === "") + throw new Error(`${path}: frontmatter "name" is required`); + if (typeof fm.description !== "string" || fm.description.trim() === "") + throw new Error(`${path}: frontmatter "description" is required`); + + const declared = + fm.reviseEveryBlocks === undefined + ? DEFAULT_REVISE_EVERY_BLOCKS + : Number(fm.reviseEveryBlocks); + if (!(Number.isFinite(declared) && declared > 0)) + throw new Error(`${path}: reviseEveryBlocks must be a positive number`); + + return { + name: fm.name, + description: fm.description, + reviseEveryBlocks: Math.floor(declared), + model: typeof fm.model === "string" ? fm.model : undefined, + body: m[2].trim(), + }; +} + +// The participant's declared cadence, clamped so one agent cannot consume the shared LLM budget. +// Returns the effective interval and whether it was clamped, so the caller can record the clamp +// rather than silently overriding what the participant asked for. +export function effectiveReviseInterval( + declaredBlocks: number, + runBlocks: number, + maxRevisions = MAX_REVISIONS_PER_RUN, +): { blocks: number; clamped: boolean } { + // runBlocks 0 means "run until the time limit", so there is no total to divide up; honor the + // declaration and let the per-run counter do the capping. + if (runBlocks <= 0) return { blocks: declaredBlocks, clamped: false }; + const floor = Math.ceil(runBlocks / maxRevisions); + return declaredBlocks >= floor + ? { blocks: declaredBlocks, clamped: false } + : { blocks: floor, clamped: true }; +} + +export type ParseResult = + { ok: true; revision: StrategyRevision } | { ok: false; reason: string }; + +// Parse the LLM's reply. Deliberately strict: a malformed revision is rejected rather than coerced, +// because the alternative is installing something the model did not mean. +export function parseRevision(raw: unknown): ParseResult { + if (!raw || typeof raw !== "object" || Array.isArray(raw)) + return { ok: false, reason: "response must be a JSON object" }; + const o = raw as Record; + if (typeof o.notes !== "string" || o.notes.trim() === "") + return { ok: false, reason: "notes must be a non-empty string" }; + // Both an explicit null and an omitted field mean "no change" -- models express it either way. + const executor = + o.executorTs === null || o.executorTs === undefined ? null : o.executorTs; + if (executor !== null && typeof executor !== "string") + return { ok: false, reason: "executorTs must be a string or null" }; + if (executor !== null && executor.trim() === "") + return { + ok: false, + reason: "executorTs was empty; use null to keep the current strategy", + }; + const revertRaw = + o.revertTo === null || o.revertTo === undefined ? null : Number(o.revertTo); + if (revertRaw !== null && !Number.isInteger(revertRaw)) + return { ok: false, reason: "revertTo must be an integer version or null" }; + // Asking for both is ambiguous, and guessing which one was meant is how a model's intent gets + // silently overridden. + if (executor !== null && revertRaw !== null) + return { + ok: false, + reason: "give either executorTs or revertTo, not both", + }; + const version = Number(o.version); + return { + ok: true, + revision: { + version: Number.isFinite(version) ? version : 0, + notes: o.notes, + executorTs: executor, + revertTo: revertRaw, + }, + }; +} + +// A compiled executor: the same shape as a rule agent's decide, so the trading loop does not care +// which one it is holding. +export type Executor = ( + obs: AgentObservation, + ctx: AgentContext, +) => Promise | AgentAction | null | undefined; + +export type CompileResult = + { ok: true; executor: Executor } | { ok: false; reason: string }; + +// Compile generated source into a callable inside a vm context. +// +// Be clear about what this does and does not contain. The vm removes *ambient* capability: there is +// no require, no process, no fs, no fetch in scope. It does not sandbox the agent from the chain, +// because `ctx` is passed in and carries publicClient / walletClient -- generated code can trade +// exactly as freely as the hand-written strategy it replaces. That is intentional (it is the same +// capability, not an escalation), but it means the vm is a guard against a model reaching for +// something outside the trading interface, not a containment boundary. The cheatcode check below is +// the part that addresses intent, and it is what stops generated code from calling the privileged +// RPCs that a participant's own code is also forbidden from calling. +export function compileExecutor(source: string): CompileResult { + const findings = findCheatcodeUsage(source); + if (findings.length > 0) + return { + ok: false, + reason: + `generated code uses privileged calls: ` + + findings + .map((f) => `${f.rule} "${f.match}" (line ${f.line})`) + .join("; "), + }; + + try { + // The source is the *body* of decide(obs, ctx). Wrapping it here rather than asking the model to + // emit a complete module keeps the contract small and means there is no import syntax to parse. + const wrapped = `(async function decide(obs, ctx) {\n${source}\n})`; + const script = new Script(wrapped, { filename: "generated-executor.js" }); + // Only what a strategy legitimately needs. No require, no process, no fs. + const sandbox = createContext({ + Math, + JSON, + Number, + String, + Boolean, + Array, + Object, + BigInt, + Map, + Set, + isFinite, + isNaN, + parseFloat, + parseInt, + }); + const fn = script.runInContext(sandbox, { timeout: 1000 }) as Executor; + if (typeof fn !== "function") + return { ok: false, reason: "compiled value is not a function" }; + + // Bring the action back into this realm before anyone downstream touches it. An object built + // inside the vm has that context's Object.prototype, so it is not `instanceof Object` here and + // deep-equality against a host object fails -- exactly the kind of difference that shows up far + // from its cause, in validation or logging, rather than at the boundary. Actions are plain data + // by contract, so a structural clone loses nothing; anything unclonable was not a valid action. + // + // The Script timeout above covers only *evaluating* the function expression, not calling it, so + // a generated body that loops or awaits forever would wedge the agent permanently: the caller's + // `deciding` guard blocks every later decision and the process never exits. Racing the call + // bounds that. It does not kill the runaway work -- vm cannot interrupt an async body -- but it + // frees the loop, and the throw is recorded as a decide error. + const normalized: Executor = async (obs, ctx) => { + let timer: ReturnType | undefined; + const result = await Promise.race([ + Promise.resolve(fn(obs, ctx)), + new Promise((_, reject) => { + timer = setTimeout( + () => + reject( + new Error( + `executor exceeded ${EXECUTOR_TIMEOUT_MS}ms; the strategy is not returning`, + ), + ), + EXECUTOR_TIMEOUT_MS, + ); + }), + ]).finally(() => clearTimeout(timer)); + if (result === null || result === undefined) return null; + try { + return structuredClone(result); + } catch (error) { + throw new Error( + `executor returned a value that is not plain data: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } + }; + return { ok: true, executor: normalized }; + } catch (error) { + return { + ok: false, + reason: `compile failed: ${error instanceof Error ? error.message : String(error)}`, + }; + } +} + +// The system prompt for a revision. The participant's improve.md is the policy; this frames what the +// model is being asked to produce and what it is allowed to see. +export function buildRevisionSystem( + agent: ImproveAgent, + currentExecutor: string, +): string { + return [ + `You maintain the trading strategy of an autonomous agent in a DeFi simulation.`, + ``, + `You are NOT trading. The strategy below runs on every block by itself. Your job is to decide`, + `whether to rewrite it, and if so, to return a better version.`, + ``, + `## The operator's instructions (written by the agent's author)`, + ``, + agent.body, + ``, + `## Current strategy`, + ``, + `It is the body of \`async function decide(obs, ctx)\`. It returns one action object, or null to`, + `do nothing this block. \`ctx.log({ reason })\` records why.`, + ``, + "```js", + currentExecutor, + "```", + ``, + `## What to return`, + ``, + `Exactly one JSON object, no prose around it. Three answers are available:`, + ``, + "```json", + `{ "notes": "why", "executorTs": "" } // install this as the strategy`, + `{ "notes": "why", "executorTs": null } // leave it alone`, + `{ "notes": "why", "revertTo": 1 } // go back to an earlier version`, + "```", + ``, + `Leaving it alone is often right: a strategy that is working does not need to be touched, and a`, + `rewrite that turns out worse costs you a revision to undo.`, + ``, + `**Nothing reverts automatically.** If a change you made has hurt, you have to say so — use`, + `\`revertTo\` with the version you want back. The history below records what each version did.`, + ``, + `The body may use only: obs, ctx, and the standard JavaScript built-ins. There is no require,`, + `no import, no process, no network. Privileged RPC calls (anvil_*, evm_*, hardhat_*) are`, + `rejected before installation.`, + ].join("\n"); +} + +// The performance context handed to the model alongside the prompt. Deliberately small: the recent +// decisions and how value has moved, not the whole history, so the model reasons about the current +// regime rather than pattern-matching the run. +export function buildRevisionContext(opts: { + block: number; + valueUsdc: number; + initialValueUsdc: number; + sinceLastRevisionUsdc: number | null; + currentVersion: number; + history: StrategyVersion[]; + recent: Array<{ round: number; reason?: string; action?: unknown }>; + observation: AgentObservation | null; +}): string { + const pnl = opts.valueUsdc - opts.initialValueUsdc; + const lines = [ + `block: ${opts.block}`, + `strategy version: ${opts.currentVersion}`, + `PnL since the run started: ${pnl.toFixed(2)} USDC`, + ]; + if (opts.sinceLastRevisionUsdc !== null) + lines.push( + `PnL since the last revision: ${opts.sinceLastRevisionUsdc.toFixed(2)} USDC`, + ); + // The history is what makes `revertTo` an informed choice rather than a guess: each version's + // stated intent, and the value the agent was carrying when it went in. + if (opts.history.length > 0) { + lines.push(``, `strategy history (version 0 is the one you were shipped):`); + for (const v of opts.history) { + const value = + v.valueAtInstall === null + ? "unknown" + : `${(v.valueAtInstall - opts.initialValueUsdc).toFixed(2)} USDC vs the run start`; + lines.push( + ` v${v.version} @ block ${v.installedAtBlock} (value then: ${value}) — ${v.notes}`, + ); + } + } + lines.push( + ``, + `recent decisions (newest last):`, + ...opts.recent + .slice(-12) + .map( + (r) => + ` block ${r.round}: ${r.action ? JSON.stringify(r.action) : "no action"}` + + (r.reason ? ` — ${r.reason}` : ""), + ), + ); + if (opts.observation) + lines.push(``, `latest observation:`, JSON.stringify(opts.observation)); + return lines.join("\n"); +} diff --git a/example/agents/runtime/prompt.ts b/example/agents/runtime/prompt.ts deleted file mode 100644 index 3a06ac1..0000000 --- a/example/agents/runtime/prompt.ts +++ /dev/null @@ -1,231 +0,0 @@ -/** - * prompt.ts: composing a prompt-type agent (a single prompt.md) (ADR 0015 §4). - * - * system = JSON mode instructions + {action JSON Schema} + environment rules - * (fixed text) + prompt.md body - * user = latest observation + recent actions and results from agentLog - * - * The format matches the training distribution of ollama-family open models (Hermes JSON - * mode; NousResearch/Hermes-Function-Calling). and runtime validation (validateAction) are - * derived from sdk's single action schema (actionSchema.ts). - * - * frontmatter has the same shape as the Agent Skills standard (agentskills.io): name / description - * required, intervalMs / model optional, unknown fields ignored (forward compatible). - */ -import { existsSync, readFileSync } from "node:fs"; -import { join } from "node:path"; -import { parse as parseYaml } from "yaml"; -import { actionJsonSchema } from "@eris/sdk/actionSchema.js"; -import { safeStringify } from "@eris/sdk/logger.js"; -import type { AgentObservation, ProtocolId } from "@eris/sdk/types.js"; - -export type PromptAgent = { - name: string; - description: string; - intervalMs?: number; - model?: string; - body: string; -}; - -export const DEFAULT_PROMPT_INTERVAL_MS = 5000; -export const DEFAULT_PROMPT_MODEL = "gpt-oss:120b"; -// Self-improvement (prompt revision) default = off. Enable via the roster env ERIS_PROMPT_REVISE_EVERY (number of decision cycles). -export const DEFAULT_PROMPT_REVISE_EVERY = 0; - -// Read prompt.md and validate its frontmatter. name/description are required (roster display / log header). -export function loadPromptAgent(agentDir: string): PromptAgent { - const path = join(agentDir, "prompt.md"); - if (!existsSync(path)) throw new Error(`prompt.md not found in ${agentDir}`); - const raw = readFileSync(path, "utf8"); - const m = raw.match(/^---\n([\s\S]*?)\n---\n?([\s\S]*)$/); - if (!m) { - throw new Error( - `${path}: frontmatter (---) is required (name / description mandatory; agentskills.io compatible)`, - ); - } - const fm = parseYaml(m[1]) as Record | null; - if (!fm || typeof fm !== "object") - throw new Error(`${path}: frontmatter must be a YAML mapping`); - if (typeof fm.name !== "string" || fm.name.trim() === "") - throw new Error(`${path}: frontmatter "name" is required`); - if (typeof fm.description !== "string" || fm.description.trim() === "") - throw new Error(`${path}: frontmatter "description" is required`); - const intervalMs = - fm.intervalMs === undefined ? undefined : Number(fm.intervalMs); - if ( - intervalMs !== undefined && - !(Number.isFinite(intervalMs) && intervalMs > 0) - ) - throw new Error(`${path}: intervalMs must be a positive number`); - return { - name: fm.name, - description: fm.description, - intervalMs, - model: typeof fm.model === "string" ? fm.model : undefined, - body: m[2].trim(), - }; -} - -// Fixed text of the environment rules. The action *format* is owned by (derived from sdk -// actionSchema), so this only states environment semantics that can only be expressed in natural -// language (how to read the observation, constraints, costs). A PR that changes the shape of -// observation / limits must update this at the same time (ADR 0015 Risks). -const ENV_RULES = `# Environment rules - -You are one trading agent among several competing on a simulated DeFi market -(Uniswap v3 / Balancer / Curve spot, GMX perp, Aave v3 lending, LST liquid staking — -only the venues listed in observation.enabledProtocols are live this run). - -## Observation (the user message contains the latest one as JSON) -- fairPriceUsdcPerWeth: the environment's fair price for WETH in USDC. Venue prices - that deviate from it tend to revert toward it. -- fairPricesUsd / baseBalances / baseDecimals: per-asset data when more bases (e.g. WBTC) trade. -- protocols.uniswap.pool.priceUsdcPerWeth, protocols.balancer.priceUsdcPerWeth, - protocols.curve.priceUsdcPerWeth: current venue prices. gap = fair/price - 1. -- balances: your wallet (decimal integer strings; WETH wei 1e18, USDC units 1e6). -- limits: hard caps enforced by the validator. Any action beyond them is rejected - before reaching the chain (the cycle is wasted). -- competition: priority-fee auction feedback (maxCompetitorPriorityFeeWei etc.). - Blocks order transactions by priority fee, descending. -- blocksRemaining: blocks left before the run ends. Anything that takes longer than - this to unwind cannot be unwound, and is scored at whatever you could exit it for. -- protocols.lst (when the lst venue is live): a liquid staking token with TWO prices. - redemptionRateWeth is what the vault owes per LST, reachable only through a - withdrawal queue that takes withdrawalDelayBlocks. marketPriceWeth is what the - LST/WETH pool pays right now. discountBps is how far the market sits below - redemption (positive = LST is cheap). You are scored on what you could realize, so - a queued redemption only counts if it finalizes while blocksRemaining lasts. - -## Action rules -- Output exactly ONE action object per decision cycle, as JSON matching the schema. -- Amounts are decimal integer strings in base units (wei / USDC units). Never use floats. -- swap.amountIn must be <= limits.maxWethInWei / maxUsdcInUnits AND <= your balance. -- priority fee (maxPriorityFeePerGasWei) is burned ETH: bid just above - competition.maxCompetitorPriorityFeeWei when you must win ordering, never more than - the trade's expected profit. It must be <= limits.maxPriorityFeePerGasWei. -- Reverted transactions still pay gas. If the edge is small or uncertain, emit - {"type":"noop","reason":"..."} instead. -- Fees/slippage: venue swap fees and slippage come out of your PnL. A gap smaller than - ~2x total costs is usually not worth taking.`; - -// Hermes JSON mode system prompt ( + rules + prompt.md body). -// If jsonSchema is passed it is not regenerated (shares the same one bot.ts built for LLM tool use). -export function buildSystemPrompt( - agent: PromptAgent, - enabledProtocols?: ProtocolId[], - jsonSchema?: Record, -): string { - const schema = safeStringify( - jsonSchema ?? actionJsonSchema(enabledProtocols), - ); - return `You are "${agent.name}" — ${agent.description}. -You are a function-calling trading agent. For each decision cycle, respond with a single -JSON object that conforms to the JSON schema below. Do not output anything else: no prose, -no markdown fences, no explanations outside the JSON. - -${schema} - - -${ENV_RULES} - -# Strategy (written by the participant) -${agent.body}`; -} - -// One line of recent action history (summarized from agentLog and attached to the user message). -export type RecentAction = { - round?: number; - action?: unknown; - note?: string; -}; - -// --------------------------------------------------------------------------- -// Self-improvement (prompt revision): every N decision cycles, attach recent actions/results and -// have the LLM rewrite the prompt body itself (the improvement target = the prompt, matching ADR -// 0015's unit of submission). The revision discipline distills the lessons of the old -// self-improvement mechanism (formerly _archive/llm/prompts.ts; removed, see git history). -// --------------------------------------------------------------------------- - -export type RevisionStats = { - cycles: number; - initialValueUsdc: number | null; - currentValueUsdc: number | null; - recentRevertRate?: number; - recentSampleSize?: number; -}; - -// System prompt for revision. The output is "only the markdown of the new prompt body" (no frontmatter/fences). -export function buildRevisionSystem(agent: PromptAgent): string { - return `You are improving the strategy prompt of the trading agent "${agent.name}" (${agent.description}). -You will receive the current strategy prompt body and the agent's recent decisions and results. -Rewrite the strategy prompt body to make the agent measurably better. - -Revision discipline: -- Preserve the strategy class and proven profitable behavior; improve the measured weakness. -- Ground every change in the evidence given (recent actions, skips, reverts, value trajectory). - Never invent a bug or an opportunity the data does not show. -- Reverts, fees and churn are direct costs. If results show over-trading, prefer higher - thresholds / cooldowns over more aggression. -- Keep the prompt concrete: numeric thresholds, sizing rules, bidding rules, explicit noop rules. -- Total value moves are dominated by price drift (beta); judge changes by trade edge, not equity. - -Output ONLY the new prompt body as plain markdown. No frontmatter, no code fences, no commentary.`; -} - -// User message for revision (current body + recent actions and results + value trajectory). -export function buildRevisionUser( - body: string, - recent: RecentAction[], - stats: RevisionStats, -): string { - const recentText = - recent.length === 0 - ? "(none)" - : recent - .map( - (r) => - `- round=${r.round ?? "?"} action=${safeStringify(r.action ?? null)}${r.note ? ` note=${r.note}` : ""}`, - ) - .join("\n"); - const value = - stats.initialValueUsdc !== null && stats.currentValueUsdc !== null - ? `${stats.initialValueUsdc.toFixed(2)} -> ${stats.currentValueUsdc.toFixed(2)} USDC (includes price drift beta you do NOT control)` - : "(not yet observed)"; - const revert = - stats.recentSampleSize !== undefined && stats.recentSampleSize > 0 - ? `${((stats.recentRevertRate ?? 0) * 100).toFixed(0)}% over last ${stats.recentSampleSize} txs` - : "(no included txs yet)"; - return `## Current strategy prompt body -${body} - -## Evidence (${stats.cycles} decision cycles so far) -- Portfolio value: ${value} -- Recent revert rate: ${revert} -- Recent decisions (most recent last): -${recentText} - -Rewrite the strategy prompt body now. Output only the new body.`; -} - -// user message: latest observation + your own recent actions and results. -export function buildUserMessage( - obs: AgentObservation, - recent: RecentAction[], -): string { - const recentText = - recent.length === 0 - ? "(none yet)" - : recent - .map( - (r) => - `- round=${r.round ?? "?"} action=${safeStringify(r.action ?? null)}${r.note ? ` note=${r.note}` : ""}`, - ) - .join("\n"); - return `## Latest observation (round ${obs.round}) -${safeStringify(obs)} - -## Your recent actions (most recent last) -${recentText} - -Respond with exactly one JSON action.`; -} diff --git a/example/agents/simple-rule/prompt.md b/example/agents/simple-rule/prompt.md deleted file mode 100644 index 46e75e8..0000000 --- a/example/agents/simple-rule/prompt.md +++ /dev/null @@ -1,51 +0,0 @@ ---- -name: simple-rule -description: Small swaps on uniswap pool vs fair-price gaps ---- -# Mission - -You are a plain single-pool (uniswap WETH/USDC) mean-reversion bot. Capture the -pull toward fair price with minimal machinery. No cross-venue comparison -(that belongs to venue-arb / arb-bot). - -## Market view - -The environment's fair price is mean-reverting; flow pushes the pool price away -from fair, and it comes back. Buying cheap / selling rich in small clips has -positive expectancy - but only the part of the gap that survives the 0.3% fee -plus slippage is profit. - -## Decision procedure (every cycle) - -1. pool = protocols.uniswap.pool.priceUsdcPerWeth, fair = fairPriceUsdcPerWeth -2. gap = fair / pool - 1 (positive = pool is cheap = buy WETH) -3. If |gap| < 15bps (0.0015): noop (below half the fee there is no trade) -4. Direction: gap > 0 -> tokenIn="USDC" (buy), gap < 0 -> tokenIn="WETH" (sell) -5. Size: cap = min(balance, per-round cap); - sizeBps = clamp(|gap| x 200000, 250, 2500) (i.e. +250bps per 12.5bps of gap, max 25%) - amountIn = cap x sizeBps / 10000 (integer string) -6. Action: {"type":"swap","tokenIn":...,"amountIn":...,"slippageBps":50, - "maxPriorityFeePerGasWei":""} - -## Worked example - -fair=3000, pool=2994 -> gap=+20.0bps -> sizeBps=clamp(400,250,2500)=400. -With a USDC cap of "5000000000" (5,000 USDC): amountIn = "200000000" (200 USDC). - -## Risk management - -- If the tokenIn-side balance is 0, noop (in USDC-only runs you cannot sell - WETH until you hold inventory) -- After 3 consecutive reverts in the same direction, double the threshold for - the next 3 cycles - -## Explicit noop criteria - -- |gap| < 15bps / uniswap not in enabledProtocols / tokenIn balance 0 / - amountIn rounds to 0 - -## Revision invariants (for self-improvement) - -- Stay single-pool uniswap (multi-venue is a different strategy). Always trade - toward fair, never away from it. -- Tunable: threshold, size gain, slippage, cooldowns. diff --git a/example/agents/stat-arb/agent.ts b/example/agents/stat-arb/agent.ts index c6be452..fe2e94e 100644 --- a/example/agents/stat-arb/agent.ts +++ b/example/agents/stat-arb/agent.ts @@ -25,6 +25,7 @@ */ import type { AgentAction, AgentObservation } from "@eris/sdk"; import { RollingStats } from "../lib/rolling-stats.js"; +import { affordable } from "../lib/affordable.js"; const WINDOW = Math.max( 2, @@ -155,10 +156,13 @@ export function decide(obs: AgentObservation): AgentAction | null { Math.floor(SIZE_FLOOR_BPS + (SIZE_CAP_BPS - SIZE_FLOOR_BPS) * t), ), ); - const amountIn = (max * BigInt(sizeBps)) / 10_000n; + // Capped by the wallet as well as by the rule limit. Under USDC-only funding the sell leg has no + // inventory behind it; proposing it anyway is a self-reject, which is indistinguishable in the + // score from choosing not to trade (issue #54). + const amountIn = affordable(obs, tokenIn, (max * BigInt(sizeBps)) / 10_000n); if (amountIn <= 0n) { - return noop("size rounds to zero"); + return noop(`no ${tokenIn} to fund this side of the spread`); } // EV in USDC ≈ size_usdc * |gap|. Convert to wei via fair price. diff --git a/example/agents/stat-arb/prompt.md b/example/agents/stat-arb/prompt.md deleted file mode 100644 index fad34a7..0000000 --- a/example/agents/stat-arb/prompt.md +++ /dev/null @@ -1,49 +0,0 @@ ---- -name: stat-arb -description: z-score statistical arb (data-driven threshold + EV-proportional bids) ---- -# Mission - -You are a statistical arbitrage bot. You use **no fixed gap threshold**; -entries trigger on the z-score of the gap against its historical distribution. -In low-vol runs you react to small gaps, in high-vol runs you wait for large -ones - the threshold adapts itself. - -## Market view - -"A 10bps gap" means different things in different regimes: a 2-sigma anomaly -when sigma=5bps, background noise when sigma=30bps. A fixed threshold is -always wrong in one of those runs. A distribution-based rule works in both. - -## Decision procedure (every cycle) - -1. gap = fair / uniswap pool price - 1; append to history - (seed the distribution from the 20 points in observation.history at startup) -2. If sample count < 20 (burn-in): noop -3. z = (gap - mean) / stddev -4. If |z| < 1.5: noop -5. Direction: z > 0 (pool cheap) -> buy with USDC; z < 0 -> sell WETH -6. Size: scale linearly from |z|=1.5 to 2.5, saturating at 50% of - min(balance, per-round cap) -7. Bid: expected EV (size USD x |gap|) in wei x 0.3 / 180000 gas, clamped to - [default, max] -8. Action: {"type":"swap","tokenIn":...,"amountIn":...,"slippageBps":75, - "maxPriorityFeePerGasWei":""} - -## Maintaining the statistics - -- Update mean/variance incrementally every cycle (Welford-style running - moments; no need to store full history) -- If an event breaks the distribution (a 5-sigma gap during a stress event), - exclude that point from the update but still use it for the trading decision - (don't let outliers pollute sigma) - -## Explicit noop criteria - -- During burn-in / |z| < 1.5 / sigma ~ 0 (degenerate distribution, common - right after start) / insufficient balance - -## Revision invariants (for self-improvement) - -- Keep distribution-based entry (never degrade to fixed-bps thresholds). -- Tunable: entry/saturation z, burn-in length, EV fraction, outlier rule. diff --git a/example/agents/venue-arb/agent.ts b/example/agents/venue-arb/agent.ts index 5083fb7..22d6a46 100644 --- a/example/agents/venue-arb/agent.ts +++ b/example/agents/venue-arb/agent.ts @@ -1,6 +1,12 @@ // venue-arb: a cross-venue arbitrage agent that swaps toward fair on the pool most deviated from // fairPrice among the active AMM venues (uniswap/balancer/curve). +// +// "Most deviated" is qualified by "and fundable": under USDC-only funding the agent starts with no +// WETH, so a rich pool -- the one it would sell into -- is not a trade it can make. It buys the +// cheap venue instead and only sells once it is holding inventory. Taking the largest gap +// unconditionally is what made this agent self-reject every action it produced (issue #54). import type { AgentAction, AgentObservation } from "@eris/sdk"; +import { affordable, canFund } from "../lib/affordable.js"; type Venue = { id: "uniswap" | "balancer" | "curve"; @@ -33,26 +39,44 @@ export function decide(obs: AgentObservation): AgentAction | null { let best: Venue | undefined; let bestGap = 0; + let skippedUnfundable = false; for (const v of venues) { if (!Number.isFinite(v.price) || v.price <= 0) continue; // exclude broken/uninitialized venues const gap = Math.abs(fair / v.price - 1); - if (gap > bestGap) { - bestGap = gap; - best = v; + if (gap <= bestGap) continue; + // Pool below fair -> WETH is cheap -> buy it with USDC. Pool above fair -> sell WETH. + if (!canFund(obs, v.price < fair ? "USDC" : "WETH")) { + skippedUnfundable = true; + continue; } + bestGap = gap; + best = v; } if (!best || bestGap < 0.001) { - return { type: "noop", reason: "no venue gap" }; + return { + type: "noop", + reason: skippedUnfundable + ? "the widest gaps need inventory this agent does not hold" + : "no venue gap", + }; } - // If pool price < fair, WETH is cheap -> buy WETH with USDC (USDC in) const tokenIn = best.price < fair ? "USDC" : "WETH"; - const max = BigInt( - tokenIn === "WETH" ? obs.limits.maxWethInWei : obs.limits.maxUsdcInUnits, - ); const sizeBps = Math.min(2500, Math.max(250, Math.floor(bestGap * 200_000))); - const amountIn = (max * BigInt(sizeBps)) / 10_000n; + const amountIn = affordable( + obs, + tokenIn, + (BigInt( + tokenIn === "WETH" ? obs.limits.maxWethInWei : obs.limits.maxUsdcInUnits, + ) * + BigInt(sizeBps)) / + 10_000n, + ); + // canFund said the wallet clears the dust floor, but the rule cap or the gap-derived size can + // still land under it. Proposing it anyway would just be rejected. + if (amountIn === 0n) + return { type: "noop", reason: "affordable size is below the dust floor" }; return { type: best.swapType, diff --git a/example/agents/venue-arb/improve.md b/example/agents/venue-arb/improve.md new file mode 100644 index 0000000..3145404 --- /dev/null +++ b/example/agents/venue-arb/improve.md @@ -0,0 +1,46 @@ +--- +name: venue-arb +description: WETH-only cross-venue arbitrage. The LLM tunes the strategy in-run; the strategy itself trades every block. +reviseEveryBlocks: 60 +--- + +You are maintaining a WETH-only cross-venue arbitrage strategy. It runs on every block without you. +Decide whether the code should change, and if so, what to. + +The strategy compares each AMM venue's pool price against the fair price and swaps toward fair on +the venue that has moved furthest — but only in a direction it can fund. Holding no WETH, it can +buy a cheap venue and cannot sell a rich one; it acquires inventory first and sells later. + +## When to leave it alone + +Return `"executorTs": null` unless something specific is wrong. It is up? Leave it. The market fell +and the strategy was holding? That is the market, not the code. Only a handful of decisions since +the last revision? Not enough to tell. + +A rewrite that performs worse than what it replaced is rolled back automatically, so a speculative +change costs you a revision and gains nothing. + +## What is worth changing + +- **Rejected actions or decide errors.** Always a bug. Fix first. +- **The gap threshold.** The strategy ignores gaps under a floor. Too high and it sits out real + opportunities; too low and it pays fees for noise. The recent decisions show which side you are on. +- **Sizing.** Size scales with the gap. If trades are winning but small, the ramp is too flat; if + they win often and still lose money, the round trip costs more than the edge. + +Resist tightening after every losing patch. A strategy that trades nothing scores zero, and zero +loses to anyone who traded. + +## Constraints + +- Only `obs`, `ctx` and standard JavaScript. No `require`, `import`, `process` or `fetch`. +- **Check the balance before choosing a direction.** `obs.balances.wethWei` is zero at the start of + the run. A leg the runtime rejects is indistinguishable from doing nothing. +- Respect `obs.limits`. +- Return one action object or `null`. `ctx.log({ reason })` records why, and you will read it later. + +## Undoing a change + +Nothing reverts automatically. If one of your rewrites made things worse, return +`{"notes": "...", "revertTo": }` — the context lists every version, when it went in, and +what the agent was worth at the time. diff --git a/example/agents/venue-arb/prompt.md b/example/agents/venue-arb/prompt.md deleted file mode 100644 index 8bd106b..0000000 --- a/example/agents/venue-arb/prompt.md +++ /dev/null @@ -1,57 +0,0 @@ ---- -name: venue-arb -description: WETH cross-venue arb (push the most-deviated venue toward fair) ---- -# Mission - -You are a WETH cross-venue arbitrage bot. Pick whichever of uniswap / balancer / -curve deviates most from fair and swap toward fair on that venue. The -multi-venue version of simple-rule. - -## Market view - -Flow pushes each venue independently, so one venue is always the most -distorted. Choosing the maximum-deviation venue buys the largest expected edge -for the same decision cost. A single swap carries temporary direction risk -(beta) - which is why you only ever trade toward fair (reversion works for you). - -## Decision procedure (every cycle) - -1. Collect venue prices: - - uniswap: protocols.uniswap.pool.priceUsdcPerWeth (action type "swap") - - balancer: protocols.balancer.priceUsdcPerWeth ("balancerSwap") - - curve: protocols.curve.priceUsdcPerWeth ("curveSwap") - Exclude venues with missing / zero / non-finite prices -2. dev = |fair / price - 1| per venue; pick the maximum -3. If max dev < 10bps (0.001): noop -4. Direction: price < fair -> tokenIn="USDC" (buy the cheap venue), - price > fair -> tokenIn="WETH" -5. Size: cap = min(balance, per-round cap); - sizeBps = clamp(dev x 200000, 250, 2500); amountIn = cap x sizeBps / 10000 -6. One swap on the chosen venue: - {"type":"balancerSwap","tokenIn":"USDC","amountIn":"...","slippageBps":75, - "maxPriorityFeePerGasWei":""} - -## Bidding - -- Default fee normally. Only for fat opportunities (dev > 30bps), bid - competition.maxCompetitorPriorityFeeWei + 1 gwei as insurance against losing - the ordering race. Never bid more than 10% of expected profit - (size USD x dev). - -## Risk management - -- tokenIn balance 0 -> only look for opportunities in the other direction - (else noop) -- competition.recentRevertRate > 50% (sample >= 4) -> double the threshold and - cool down for 5 cycles - -## Explicit noop criteria - -- All venues dev < 10bps / no valid venue / insufficient balance / amountIn=0 - -## Revision invariants (for self-improvement) - -- "Toward fair only" and "single swap only" (2-leg bundles belong to - cross-venue-arb). -- Tunable: threshold, size gain, bidding rule, cooldowns. diff --git a/scripts/checkStrategyCode.ts b/scripts/checkStrategyCode.ts index 9201e88..f50264d 100644 --- a/scripts/checkStrategyCode.ts +++ b/scripts/checkStrategyCode.ts @@ -10,7 +10,7 @@ import { join } from "node:path"; import { findCheatcodeUsage, type StaticCheckFinding, -} from "../core/src/strategyStaticCheck.js"; +} from "@eris/sdk/strategyStaticCheck.js"; // ADR 0015 §2: 1 agent = 1 directory. runtime/ is a reserved name (not participant code, so excluded); // lib/ holds shared strategy helpers, so it is included. diff --git a/scripts/genStateDump.ts b/scripts/genStateDump.ts index 3052da4..8ef86fe 100644 --- a/scripts/genStateDump.ts +++ b/scripts/genStateDump.ts @@ -159,7 +159,7 @@ async function main(): Promise { ` commit=${manifest.sourceCommit.slice(0, 12)} chainId=${manifest.chainId} genesis=${genesis.hash.slice(0, 12)}…`, ); console.log(` fingerprint=${manifest.deploymentsFingerprint.slice(0, 20)}…`); - console.log(` run: npm run backtest -- --regime calm-01`); + console.log(` run: npm run backtest -- --regime calm --seed 101`); } main().catch((error) => { diff --git a/sdk/src/agent.ts b/sdk/src/agent.ts index 564f5fc..ed95440 100644 --- a/sdk/src/agent.ts +++ b/sdk/src/agent.ts @@ -2,7 +2,8 @@ // The agent.ts in example/agents// exports one of: // - decide(obs, ctx): rule strategy. runtime/bot.ts drives it in a read→decide→send loop // - run(ctx): self-driven (liquidator etc.). bot.ts does not loop and delegates by passing ctx -// A prompt agent that is a single prompt.md has no export; bot.ts produces the action via the LLM (§4). +// A self-improving agent (agent.ts + improve.md) exports decide like any rule agent; the LLM swaps +// that function out of band rather than producing actions itself (ADR 0018). import type { Address, PublicClient, WalletClient } from "viem"; import type { SimConfig } from "./config.js"; import type { AgentAction, AgentObservation } from "./types.js"; diff --git a/sdk/src/config.ts b/sdk/src/config.ts index 90df70d..065a9ba 100644 --- a/sdk/src/config.ts +++ b/sdk/src/config.ts @@ -11,6 +11,7 @@ import { MAX_BUNDLE_ACTIONS, } from "./constants.js"; import type { ProtocolId } from "./types.js"; +import type { OuParams } from "./rng.js"; import { baseTokens } from "./markets.js"; // The lst venue is deliberately not in the default set: it exists only under local deploy (issue @@ -26,6 +27,11 @@ const ALL_PROTOCOLS: ProtocolId[] = [ // Protocols that can be named in run.protocols but are not in the default set. const OPT_IN_PROTOCOLS: ProtocolId[] = ["lst"]; +export type OuConfig = { + global: OuParams; + perBase: Record; +}; + export type SimConfig = { rpcUrl: string; chainId: number; @@ -87,6 +93,16 @@ export type SimConfig = { // protocols' working set (ADR 0006 Risks anvil cold-fetch mitigation). The competition-phase mine // then avoids hitting upstream fetches. 0 disables it (ERIS_PREWARM_BLOCKS). prewarmBlocks: number; + // Fair-price OU parameters (market.* in YAML). `global` is the default and the WETH path; `perBase` + // resolves each registered base, falling back to the global value. A regime sets these to be a + // regime -- e.g. cex-drift is a nonzero `drift` with a weak `kappa` (ADR 0017 regime 1). + ou: OuConfig; + // Reconstruct the value cross-section only every Nth block instead of every block (ERIS_SCORE_EVERY). + // The final score reads only the first and last cross-sections (alphaByAgent = alphaLast - alphaFirst), + // so thinning does not change any agent's score -- it only coarsens the equity curve written to + // events.jsonl. The first and last blocks are always read. 1 (default) = every block. + // Used to cut reconstruction cost when replaying a whole scenario matrix (ADR 0017 §3). + scoreEvery: number; seed: number; runDirRoot: string; agentTimeoutMs: number; @@ -134,6 +150,11 @@ export type SimConfig = { uninformedFlowCount: number; // How many blocks the uninformed direction persists (default 1). >1 mimics order-flow imbalance and naturally produces a spread. uninformedFlowPersistBlocks: number; + // Probability [0,1] that a venue's persisted uninformed direction follows the market-wide one + // instead of its own (UNINFORMED_FLOW_TREND_CORRELATION). 0 (default) = independent per venue, + // which manufactures a cross-venue spread; 1 = the whole market leans the same way, which is what + // the informed-flow regime is (ADR 0017 regime 2). Only bites when uninformedPersistBlocks > 1. + uninformedFlowTrendCorrelation: number; informedFlowMaxWethWei: bigint; enabledProtocols: ProtocolId[]; maxGmxSizeUsd: bigint; @@ -290,6 +311,8 @@ export function loadConfig(env = process.env): SimConfig { localSnapshotFile: env.ERIS_LOCAL_SNAPSHOT_FILE ?? ".local-snapshot", runMode: env.ERIS_RUN_MODE === "backtest" ? "backtest" : "realtime", prewarmBlocks: intEnv(env.ERIS_PREWARM_BLOCKS, 0), + ou: readOuParams(env), + scoreEvery: Math.max(1, intEnv(env.ERIS_SCORE_EVERY, 1)), seed: intEnv(env.SEED, 1), runDirRoot: env.REPORT_DIR ?? "./runs", agentTimeoutMs: intEnv(env.AGENT_TIMEOUT_MS, 5000), @@ -335,6 +358,10 @@ export function loadConfig(env = process.env): SimConfig { ), uninformedFlowCount: intEnv(env.UNINFORMED_FLOW_COUNT, 1), uninformedFlowPersistBlocks: intEnv(env.UNINFORMED_FLOW_PERSIST_BLOCKS, 1), + uninformedFlowTrendCorrelation: Math.min( + 1, + Math.max(0, floatEnv(env.UNINFORMED_FLOW_TREND_CORRELATION, 0)), + ), informedFlowMaxWethWei: bigintEnv( env.INFORMED_FLOW_MAX_WETH_WEI, 2_000_000_000_000_000_000n, @@ -543,6 +570,34 @@ function readBaseAmounts( return out; } +// OU parameters for the fair-price process, per base (ADR 0017 regime 1). +// +// These used to live only in sdk/src/rng.ts as module-level constants read straight from +// process.env, which meant a regime YAML could not set them: the YAML loader builds a source map, it +// does not mutate process.env. Reading them here puts them on the same footing as every other run +// knob -- YAML sets them, and the coordinator passes them explicitly rather than the price model +// reaching for globals. +function readOuParams(env: NodeJS.ProcessEnv): OuConfig { + const global = { + volatility: floatEnv(env.ERIS_PRICE_VOLATILITY, 0.004), + kappa: floatEnv(env.ERIS_PRICE_REVERT_KAPPA, 0.02), + drift: floatEnv(env.ERIS_PRICE_DRIFT, 0), + }; + const perBase: Record = {}; + for (const t of baseTokens()) { + const sfx = t.symbol.toUpperCase(); + perBase[t.symbol] = { + volatility: floatEnv( + env[`ERIS_PRICE_VOLATILITY_${sfx}`], + global.volatility, + ), + kappa: floatEnv(env[`ERIS_PRICE_REVERT_KAPPA_${sfx}`], global.kappa), + drift: floatEnv(env[`ERIS_PRICE_DRIFT_${sfx}`], global.drift), + }; + } + return { global, perBase }; +} + function hexEnv(value: string | undefined, fallback: string): Hex { const result = value && value.length > 0 ? value : fallback; if (!/^0x[0-9a-fA-F]{64}$/.test(result)) diff --git a/sdk/src/rng.ts b/sdk/src/rng.ts index a0490b3..edc5040 100644 --- a/sdk/src/rng.ts +++ b/sdk/src/rng.ts @@ -69,7 +69,15 @@ const PRICE_DRIFT = floatEnv(process.env.ERIS_PRICE_DRIFT, 0); // OU parameters for a single asset (ADR 0013). export type OuParams = { volatility: number; kappa: number; drift: number }; -// Global default (backward compatible: same behavior as the old nextFairPrice). +// Legacy env-driven accessors. The run's parameters now live in SimConfig (`config.ou`, from the +// YAML `market.*` section; see readOuParams in config.ts), because the YAML loader builds a source +// map rather than mutating process.env and so could never reach the constants above. The +// coordinator passes config.ou explicitly; these remain only as the default for a caller that +// passes no params, and they read process.env, which the config path deliberately does not. +// +// Prefer passing params. A caller that forgets gets the module defaults with no error -- e.g. the +// cex-drift regime's drift silently becoming 0. ERIS_PRICE_* is listed in RETIRED_CONFIG_ENV so a +// stale environment at least announces itself. export function globalOuParams(): OuParams { return { volatility: PRICE_VOLATILITY, diff --git a/sdk/src/runConfig.ts b/sdk/src/runConfig.ts index 2f66838..bec82d6 100644 --- a/sdk/src/runConfig.ts +++ b/sdk/src/runConfig.ts @@ -75,12 +75,18 @@ const SCHEMA: Record = { "run.localDeploy": "ERIS_LOCAL_DEPLOY", "run.skipReset": "ERIS_SKIP_RESET", "run.prewarmBlocks": "ERIS_PREWARM_BLOCKS", + "run.scoreEvery": "ERIS_SCORE_EVERY", "run.reportDir": "REPORT_DIR", "run.flashArb": "ERIS_FLASH_ARB", "run.localSnapshotFile": "ERIS_LOCAL_SNAPSHOT_FILE", "run.agentTimeoutMs": "AGENT_TIMEOUT_MS", "run.agentsConfig": "AGENTS_CONFIG", // roster file path when there are no inline agents "run.agentsDir": "ERIS_AGENTS_DIR", // root of the agent directory convention (ADR 0015 §6) + // market (the fair-price OU process. ADR 0017 regime 1 "cex-drift" needs these per regime, and + // until now they were only reachable through raw env, which a regime YAML cannot set) + "market.volatility": "ERIS_PRICE_VOLATILITY", + "market.kappa": "ERIS_PRICE_REVERT_KAPPA", + "market.drift": "ERIS_PRICE_DRIFT", // funding "funding.ethWei": "INITIAL_ETH_WEI", "funding.wethWei": "INITIAL_WETH_WEI", @@ -103,6 +109,7 @@ const SCHEMA: Record = { "flow.uninformedMaxWethWei": "UNINFORMED_FLOW_MAX_WETH_WEI", "flow.uninformedCount": "UNINFORMED_FLOW_COUNT", "flow.uninformedPersistBlocks": "UNINFORMED_FLOW_PERSIST_BLOCKS", + "flow.uninformedTrendCorrelation": "UNINFORMED_FLOW_TREND_CORRELATION", "flow.informedMaxWethWei": "INFORMED_FLOW_MAX_WETH_WEI", "flow.balancerMaxWethWei": "BALANCER_FLOW_MAX_WETH_WEI", "flow.curveMaxWethWei": "CURVE_FLOW_MAX_WETH_WEI", @@ -149,7 +156,23 @@ const BASE_SECTIONS: Record = { "limits.aaveSupplyBase": { prefix: "MAX_AAVE_SUPPLY" }, "flow.baseMax": { prefix: "FLOW_MAX" }, }; -const SECTIONS = ["run", "funding", "limits", "flow", "stress", "vuln", "lst"]; +// Per-base overrides whose env name is `_` with no unit suffix. Distinct from +// BASE_SECTIONS, whose names carry one (`MAX_AGENT_WBTC_IN_SATS`): a volatility has no unit. +const BASE_PLAIN_SECTIONS: Record = { + "market.baseVolatility": "ERIS_PRICE_VOLATILITY", + "market.baseKappa": "ERIS_PRICE_REVERT_KAPPA", + "market.baseDrift": "ERIS_PRICE_DRIFT", +}; +const SECTIONS = [ + "run", + "market", + "funding", + "limits", + "flow", + "stress", + "vuln", + "lst", +]; function baseEnvName(prefix: string, sym: string, infix?: string): string { const unit = unitSuffixFor(tokenInfo(sym).decimals); @@ -173,7 +196,15 @@ function applyDoc( for (const [sk, sv] of Object.entries(v as Record)) { const path = `${k}.${sk}`; const baseDef = BASE_SECTIONS[path]; - if (baseDef) { + const basePlainPrefix = BASE_PLAIN_SECTIONS[path]; + if (basePlainPrefix) { + if (sv && typeof sv === "object" && !Array.isArray(sv)) + for (const [sym, value] of Object.entries( + sv as Record, + )) + source[`${basePlainPrefix}_${sym.toUpperCase()}`] = + toEnvString(value); + } else if (baseDef) { if (sv && typeof sv === "object" && !Array.isArray(sv)) for (const [sym, amt] of Object.entries( sv as Record, diff --git a/core/src/strategyStaticCheck.ts b/sdk/src/strategyStaticCheck.ts similarity index 52% rename from core/src/strategyStaticCheck.ts rename to sdk/src/strategyStaticCheck.ts index 9510747..01c8f82 100644 --- a/core/src/strategyStaticCheck.ts +++ b/sdk/src/strategyStaticCheck.ts @@ -1,10 +1,15 @@ -// Static analysis of strategy code (ADR 0006 §5). -// In direct mode the agent touches the anvil RPC directly, so it can in principle -// cheat via unauthenticated cheatcodes (anvil_setBalance / evm_mine / -// anvil_impersonateAccount, etc.). When an LLM authors strategy code, "self-written -// agent = trusted" no longer holds, so the /strategy-evolve gate includes a -// mechanical check that generated/edited strategy code contains no cheatcode calls, -// as an entry-side defense (paired with post-run auditing). +// Static analysis of strategy code (ADR 0006 §5, ADR 0018 §2). +// +// An agent talks to anvil directly, so it could in principle cheat through the unauthenticated +// cheatcode RPCs (anvil_setBalance / evm_mine / anvil_impersonateAccount, ...). The submission gate +// (`npm run check:strategy`) runs this over participant code as an entry-side defense, paired with +// post-run auditing. +// +// It lives in the sdk rather than in core because both sides need it: core runs it as a gate, and +// the agent runtime (example/agents/runtime) runs it on **LLM-generated executor code before +// installing it** (ADR 0018). Generated code is the case the original comment anticipated -- once an +// LLM authors the strategy, "self-written agent = trusted" stops holding -- and example cannot +// import core (the dependency direction is example -> sdk <- core). export type StaticCheckFinding = { line: number; // 1-based match: string; diff --git a/test/backtestShared.test.ts b/test/backtestShared.test.ts index 3689e64..f5be99a 100644 --- a/test/backtestShared.test.ts +++ b/test/backtestShared.test.ts @@ -179,14 +179,14 @@ describe("resolveRegimePath", () => { it("a name resolves to config/regimes/.yaml, a path spec resolves as-is", () => { const root = join(tmp, "root"); mkdirSync(join(root, "config", "regimes"), { recursive: true }); - writeFileSync(join(root, "config", "regimes", "calm-01.yaml"), "run: {}\n"); + writeFileSync(join(root, "config", "regimes", "calm.yaml"), "run: {}\n"); assert.equal( - resolveRegimePath(root, "calm-01"), - join(root, "config", "regimes", "calm-01.yaml"), + resolveRegimePath(root, "calm"), + join(root, "config", "regimes", "calm.yaml"), ); assert.equal( - resolveRegimePath(root, "config/regimes/calm-01.yaml"), - join(root, "config", "regimes", "calm-01.yaml"), + resolveRegimePath(root, "config/regimes/calm.yaml"), + join(root, "config", "regimes", "calm.yaml"), ); }); @@ -194,7 +194,7 @@ describe("resolveRegimePath", () => { const root = join(tmp, "root"); assert.throws( () => resolveRegimePath(root, "spike-99"), - /regime not found: spike-99.*available: calm-01/s, + /regime not found: spike-99.*available: calm/s, ); }); }); diff --git a/test/flowTrendCorrelation.test.ts b/test/flowTrendCorrelation.test.ts new file mode 100644 index 0000000..19b665d --- /dev/null +++ b/test/flowTrendCorrelation.test.ts @@ -0,0 +1,181 @@ +// The persisted uninformed trend, and its correlated form (ADR 0017 regime 2, informed-flow). +// +// The trend direction has to satisfy three things at once, and the first version of this file only +// checked the third, which let two real bugs through (both found in review): +// +// 1. It must depend on the seed. The direction used to be a function of the block window alone, so +// it was identical on every published seed and on every unpublished one -- an agent could +// hard-code `floor(round/persistBlocks) % 2` and front-run every reversal. The private seed set +// would have provided no protection at all. +// 2. It must actually be distributed. The mixing step used float `*` on values above 2^53, which +// rounded the low bits away; "uniswap" and "balancer" came back even in *every* window, i.e. a +// permanent one-way bias rather than a trend, and no divergence between the two deepest venues. +// 3. correlation=1 must align the venues, correlation=0 must not. +// +// The tests below check all three. The weak assertion that let (1) and (2) through was +// `new Set(directions).size === 2`, which a perfect alternation satisfies. +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { Rng } from "@eris/sdk/rng.js"; +import { buildAmmFlow } from "../core/src/flow/logic.js"; + +const VENUES = ["uniswap", "balancer", "curve"] as const; +const MAX = 1_000_000_000_000_000_000n; +const FEE_WEI = 1_000_000_000n; +const PERSIST = 5; + +// The direction of a venue's uninformed order in a round: "buy" pushes price up (USDC in), +// "sell" pushes it down (WETH in). The informed leg is filtered out -- it chases fair, so it says +// nothing about the trend being injected. +function direction( + venue: (typeof VENUES)[number], + round: number, + correlation: number, + trendSeed = 7, +): "buy" | "sell" | null { + const orders = buildAmmFlow( + // A fresh Rng per call: the trend must come from the deterministic trend stream, not from the + // shared RNG, so the direction has to be stable regardless of RNG position. + new Rng(12345), + venue, + 2000, + 2000, // pool == fair, so informed flow has no gap to close + MAX, + MAX, + FEE_WEI, + undefined, + false, + "WETH", + 1, + round, + PERSIST, + 0, + 0, + 1, + correlation, + trendSeed, + ); + const uninformed = orders.filter((o) => o.kind === "uninformed"); + if (uninformed.length === 0) return null; + const action = uninformed[0].action as { tokenIn?: string }; + return action.tokenIn === "USDC" ? "buy" : "sell"; +} + +// The direction sequence over `count` consecutive windows. +function windowDirections( + venue: (typeof VENUES)[number], + correlation: number, + trendSeed: number, + count = 24, +): string[] { + return Array.from( + { length: count }, + (_, w) => direction(venue, w * PERSIST, correlation, trendSeed) ?? "?", + ); +} + +test("the direction depends on the seed, not just the block window", () => { + // Bug 1. A seed-independent direction makes the regime memorizable, and the private seed set + // stops protecting anything. + const patterns = new Set( + [1, 2, 3, 4, 5, 101, 202, 303].map((seed) => + windowDirections("uniswap", 1, seed).join(""), + ), + ); + assert.ok( + patterns.size >= 6, + `only ${patterns.size} distinct direction sequences across 8 seeds`, + ); +}); + +test("no venue is pinned to one direction, and none is a plain alternation", () => { + // Bug 2. Both failure modes read as "a trend exists" if you only count distinct values. + for (const venue of VENUES) { + for (const correlation of [0, 1]) { + const dirs = windowDirections(venue, correlation, 7, 40); + const buys = dirs.filter((d) => d === "buy").length; + assert.ok( + buys > 6 && buys < 34, + `${venue}@corr=${correlation} is one-way: ${buys}/40 buys`, + ); + const alternating = dirs.every((d, i) => i === 0 || d !== dirs[i - 1]); + assert.ok( + !alternating, + `${venue}@corr=${correlation} alternates on a fixed clock`, + ); + } + } +}); + +test("correlation=0 keeps the venues independent", () => { + // Every pair must disagree sometimes; checking only "some pair disagreed" hid that uniswap and + // balancer were permanently identical. + for (const [a, b] of [ + ["uniswap", "balancer"], + ["uniswap", "curve"], + ["balancer", "curve"], + ] as const) { + const da = windowDirections(a, 0, 7, 40); + const db = windowDirections(b, 0, 7, 40); + const disagreements = da.filter((d, i) => d !== db[i]).length; + assert.ok( + disagreements > 6, + `${a} and ${b} only disagreed ${disagreements}/40 windows`, + ); + } +}); + +test("correlation=1 aligns every venue in each window", () => { + for (let round = 0; round < 20 * PERSIST; round += PERSIST) { + const dirs = VENUES.map((v) => direction(v, round, 1)); + assert.equal( + new Set(dirs).size, + 1, + `venues disagreed at round ${round}: ${dirs.join(",")}`, + ); + } +}); + +test("the direction is held for the whole window", () => { + for (let start = 0; start < 5 * PERSIST; start += PERSIST) { + const first = direction("uniswap", start, 1); + for (let r = start; r < start + PERSIST; r++) + assert.equal( + direction("uniswap", r, 1), + first, + `direction changed inside window starting at ${start}`, + ); + } +}); + +test("the trend does not touch the RNG consumption sequence", () => { + // Drawn off a separate stream on purpose: if it consumed from the shared rng, enabling the trend + // (or changing its correlation) would shift every downstream order in the run and no calibration + // would carry over. + const drain = (correlation: number, persist: number): number => { + const rng = new Rng(999); + buildAmmFlow( + rng, + "uniswap", + 1990, + 2000, + MAX, + MAX, + FEE_WEI, + undefined, + false, + "WETH", + 1, + 3, + persist, + 0, + 0, + 1, + correlation, + 7, + ); + // Whatever the RNG produces next reveals how many draws the call consumed. + return rng.next(); + }; + assert.equal(drain(0, PERSIST), drain(1, PERSIST)); +}); diff --git a/test/improve.test.ts b/test/improve.test.ts new file mode 100644 index 0000000..6345856 --- /dev/null +++ b/test/improve.test.ts @@ -0,0 +1,301 @@ +// Self-improving agent runtime (ADR 0018). +// +// The guards here exist because the deleted src/llm two-layer machinery lacked or under-used them: +// it lost to frozen strategies on multi-seed validation and its rollback never fired in 18 runs. +// So the tests concentrate on the three things that must not fail open -- generated code that +// cheats, generated code that does not compile, and a cadence a participant can declare freely. +import test from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + buildRevisionContext, + buildRevisionSystem, + compileExecutor, + DEFAULT_REVISE_EVERY_BLOCKS, + effectiveReviseInterval, + loadImproveAgent, + EXECUTOR_TIMEOUT_MS, + MAX_REVISIONS_PER_RUN, + parseRevision, +} from "../example/agents/runtime/improve.js"; + +function agentDir(improveMd: string): string { + const dir = mkdtempSync(join(tmpdir(), "eris-improve-")); + writeFileSync(join(dir, "improve.md"), improveMd); + return dir; +} + +const FRONTMATTER = `--- +name: test-agent +description: an agent used by the tests +--- + +Only rewrite the strategy when it is losing money.`; + +test("loadImproveAgent: reads frontmatter and defaults the cadence", () => { + const agent = loadImproveAgent(agentDir(FRONTMATTER)); + assert.equal(agent.name, "test-agent"); + assert.equal(agent.reviseEveryBlocks, DEFAULT_REVISE_EVERY_BLOCKS); + assert.match(agent.body, /Only rewrite the strategy/); +}); + +test("loadImproveAgent: the participant can declare the cadence", () => { + const agent = loadImproveAgent( + agentDir(FRONTMATTER.replace("---\n\n", "reviseEveryBlocks: 30\n---\n\n")), + ); + assert.equal(agent.reviseEveryBlocks, 30); +}); + +test("loadImproveAgent: a missing name or bad cadence is an explicit error", () => { + assert.throws( + () => loadImproveAgent(agentDir("---\ndescription: x\n---\nbody")), + /"name" is required/, + ); + assert.throws( + () => + loadImproveAgent( + agentDir( + FRONTMATTER.replace("---\n\n", "reviseEveryBlocks: 0\n---\n\n"), + ), + ), + /must be a positive number/, + ); +}); + +test("the declared cadence is honored until it would exceed the operator's cap", () => { + // A co-located run shares one LLM budget, so one participant declaring "every block" must not be + // able to starve the field -- but a reasonable declaration has to pass through untouched, or the + // knob is decorative. + assert.deepEqual(effectiveReviseInterval(60, 360), { + blocks: 60, + clamped: false, + }); + const greedy = effectiveReviseInterval(1, 360); + assert.ok(greedy.clamped); + assert.equal(greedy.blocks, Math.ceil(360 / MAX_REVISIONS_PER_RUN)); + // A run with no block target has no total to divide up; the per-run counter caps it instead. + assert.deepEqual(effectiveReviseInterval(1, 0), { + blocks: 1, + clamped: false, + }); +}); + +test("parseRevision: null or omitted executorTs means 'keep the current strategy'", () => { + // Not an error case: declining to touch a working strategy is the behavior ADR 0018 wants, and + // models express it both ways. + for (const raw of [ + { version: 2, notes: "still working", executorTs: null }, + { version: 2, notes: "still working" }, + ]) { + const r = parseRevision(raw); + assert.ok(r.ok); + assert.equal(r.revision.executorTs, null); + assert.equal(r.revision.revertTo, null); + } +}); + +test("parseRevision: revertTo is how the model undoes its own change", () => { + // Reverting is the model's judgment, not an automatic threshold: an automatic rule needs a number + // and there is no defensible one (the previous implementation's never fired; the obvious opposite + // reverts every revision in a losing regime). + const r = parseRevision({ notes: "v1 made it worse", revertTo: 0 }); + assert.ok(r.ok); + assert.equal(r.revision.revertTo, 0); + assert.equal(r.revision.executorTs, null); +}); + +test("parseRevision: asking for both a rewrite and a revert is ambiguous", () => { + // Guessing which one was meant is how a model's intent gets silently overridden. + const r = parseRevision({ + notes: "x", + executorTs: "return null;", + revertTo: 1, + }); + assert.ok(!r.ok); + assert.match(r.reason, /not both/); +}); + +test("parseRevision: a non-integer revertTo is rejected", () => { + const r = parseRevision({ notes: "x", revertTo: "the good one" }); + assert.ok(!r.ok); + assert.match(r.reason, /integer version/); +}); + +test("parseRevision: malformed responses are rejected, not coerced", () => { + const bad: Array<[unknown, RegExp]> = [ + ["not an object", /JSON object/], + [{ executorTs: "return null;" }, /notes/], + [{ notes: "x", executorTs: 42 }, /executorTs must be/], + [{ notes: "x", executorTs: " " }, /use null to keep/], + ]; + for (const [raw, pattern] of bad) { + const r = parseRevision(raw); + assert.ok(!r.ok, `expected rejection for ${JSON.stringify(raw)}`); + assert.match(r.reason, pattern); + } +}); + +test("compileExecutor: the action comes back in this realm, not the sandbox's", async () => { + // An object built inside the vm carries that context's Object.prototype, so it is not + // `instanceof Object` here and deep-equality against a host object fails. Left alone that shows up + // far from its cause -- in validation or logging -- so the boundary normalizes it. + const r = compileExecutor(`return { type: "swap", nested: { a: [1, 2] } };`); + assert.ok(r.ok); + const action = (await r.executor({ round: 1 } as never, {} as never)) as object; + assert.ok(action instanceof Object, "action is not a host-realm object"); + assert.deepEqual(action, { type: "swap", nested: { a: [1, 2] } }); +}); + +test("compileExecutor: a valid body becomes a callable decide", async () => { + const r = compileExecutor( + `if (obs.round % 2 === 0) return null; + return { type: "swap", tokenIn: "USDC", amountIn: "1" };`, + ); + assert.ok(r.ok); + const ctx = {} as never; + assert.equal(await r.executor({ round: 2 } as never, ctx), null); + assert.deepEqual(await r.executor({ round: 3 } as never, ctx), { + type: "swap", + tokenIn: "USDC", + amountIn: "1", + }); +}); + +test("compileExecutor: cheatcode calls are refused before installation", () => { + // An LLM-authored strategy is not trusted code. This is the same check the submission gate runs, + // applied to generated code -- which is the case the gate cannot see. + for (const source of [ + `await ctx.publicClient.request({ method: "anvil_setBalance" }); return null;`, + `await ctx.publicClient.request({ method: "evm_mine" }); return null;`, + `setEthBalance(ctx.publicClient, ctx.address, 1n); return null;`, + ]) { + const r = compileExecutor(source); + assert.ok(!r.ok, `expected refusal for: ${source}`); + assert.match(r.reason, /privileged calls/); + } +}); + +test("compileExecutor: a syntax error is a rejection, not a crash", () => { + const r = compileExecutor("return {{{ oops"); + assert.ok(!r.ok); + assert.match(r.reason, /compile failed/); +}); + +test("compileExecutor: the sandbox has no module system, process or filesystem", () => { + // Not a security boundary against a determined attacker -- a boundary against the model reaching + // for something that is not the trading interface. Reaching for it should fail loudly at call + // time rather than silently succeeding. + for (const source of [ + `return require("node:fs").readFileSync("/etc/passwd");`, + `return process.env.ERIS_AGENT_PRIVATE_KEY;`, + ]) { + const r = compileExecutor(source); + assert.ok(r.ok, "the body compiles; the reference only fails when called"); + assert.rejects( + async () => await r.executor({ round: 1 } as never, {} as never), + /is not defined/, + ); + } +}); + +test("buildRevisionSystem: the model is told it is not trading, and may decline", () => { + const agent = loadImproveAgent(agentDir(FRONTMATTER)); + const system = buildRevisionSystem(agent, "return null;"); + assert.match(system, /You are NOT trading/); + // The participant's own instructions have to reach the model, or improve.md is decorative. + assert.match(system, /Only rewrite the strategy when it is losing money/); + // Declining must read as a legitimate answer, since "do not touch a winner" is the fix for the + // failure mode the previous implementation had. + assert.match(system, /executorTs": null|"executorTs": null/); + assert.match(system, /does not need to be touched/); + assert.match(system, /return null;/); +}); + +test("buildRevisionContext: reports PnL since the run and since the last revision", () => { + const context = buildRevisionContext({ + block: 120, + valueUsdc: 25_500, + initialValueUsdc: 25_000, + sinceLastRevisionUsdc: -80, + currentVersion: 2, + history: [], + recent: [{ round: 118, reason: "no gap" }], + observation: null, + }); + assert.match(context, /block: 120/); + assert.match(context, /strategy version: 2/); + assert.match(context, /since the run started: 500\.00/); + // Without this the model cannot tell whether its own last change helped. + assert.match(context, /since the last revision: -80\.00/); + assert.match(context, /block 118: no action — no gap/); +}); + +test("compileExecutor: a strategy that never returns is bounded, not left to wedge the agent", async () => { + // The Script timeout covers evaluating the function expression, not calling it. Without a bound + // on the call, a generated body that awaits forever holds the caller's `deciding` guard and the + // agent stops deciding for the rest of the run while every log still looks healthy. + const r = compileExecutor(`await new Promise(() => {}); return null;`); + assert.ok(r.ok); + const started = Date.now(); + await assert.rejects( + async () => await r.executor({ round: 1 } as never, {} as never), + /is not returning/, + ); + // Bounded near the limit rather than hanging; generous upper bound so a slow machine cannot flake. + assert.ok( + Date.now() - started < EXECUTOR_TIMEOUT_MS * 3, + "the call was not bounded", + ); +}); + +test("compileExecutor: a fast strategy is not penalised by the timeout", async () => { + const r = compileExecutor(`return { type: "noop" };`); + assert.ok(r.ok); + assert.deepEqual(await r.executor({ round: 1 } as never, {} as never), { + type: "noop", + }); +}); + +test("buildRevisionContext: the history is what makes a revert an informed choice", () => { + // Without it the model can only guess which earlier version to go back to, and `revertTo` becomes + // a coin flip rather than a judgment. + const context = buildRevisionContext({ + block: 200, + valueUsdc: 24_000, + initialValueUsdc: 25_000, + sinceLastRevisionUsdc: -900, + currentVersion: 1, + history: [ + { + version: 0, + source: "return null;", + notes: "the strategy as submitted", + installedAtBlock: 0, + valueAtInstall: 25_000, + }, + { + version: 1, + source: "return {};", + notes: "widened the entry threshold", + installedAtBlock: 140, + valueAtInstall: 24_900, + }, + ], + recent: [], + observation: null, + }); + assert.match(context, /v0 @ block 0/); + assert.match(context, /widened the entry threshold/); + // Values are shown relative to the run start, which is the frame the model reasons in. + assert.match(context, /v1 @ block 140 \(value then: -100\.00 USDC/); +}); + +test("buildRevisionSystem: reverting is offered, and said to be manual", () => { + const agent = loadImproveAgent(agentDir(FRONTMATTER)); + const system = buildRevisionSystem(agent, "return null;"); + assert.match(system, /revertTo/); + // The model must know nothing will undo a bad change for it. + assert.match(system, /Nothing reverts automatically/); +}); diff --git a/test/postRunCheck.test.ts b/test/postRunCheck.test.ts index 3f8368c..912f22f 100644 --- a/test/postRunCheck.test.ts +++ b/test/postRunCheck.test.ts @@ -1,6 +1,9 @@ import test from "node:test"; import assert from "node:assert/strict"; -import { checkFeeViolations } from "../core/src/postRunCheck.js"; +import { + checkFeeViolations, + countRevertedTxs, +} from "../core/src/postRunCheck.js"; const HEADER = "round,blockNumber,txIndex,hash,from,priorityFeeWei,status,ownerId,role,actionType,bundleId,bundleIndex"; @@ -47,3 +50,31 @@ test("checkFeeViolations: skips rows with an invalid fee value", () => { [], ); }); + +// A reverted environment shock must be visible (ADR 0017 regime 3). +// +// A whale order goes out through the ordinary flow relay, which catches *submission* errors. An +// on-chain revert is not one: the tx lands, the schedule logs that the whale fired, and only +// blocks.csv records that it did nothing. A missing token approval once turned the whale regime +// into calm for a whole run with every other log looking healthy. +test("countRevertedTxs: separates reverted from executed for one owner", () => { + const rows = csv([ + "1,100,7,0xaaa,0xw,100000000,reverted,flow-whale:uninformed,uninformed-flow,swap,,", + "2,101,7,0xbbb,0xw,100000000,success,flow-whale:uninformed,uninformed-flow,balancerSwap,,", + "3,102,7,0xccc,0xw,100000000,reverted,flow-whale:uninformed,uninformed-flow,curveSwap,,", + // another owner's revert must not be counted + "4,103,1,0xddd,0xa,100000000,reverted,arb,agent,swap,,", + ]); + assert.deepEqual(countRevertedTxs(rows, "flow-whale:uninformed"), { + total: 3, + reverted: 2, + }); +}); + +test("countRevertedTxs: an owner with no txs is zero, not an error", () => { + // The whale wallet exists only when the schedule has a whale; asking about it otherwise is normal. + assert.deepEqual(countRevertedTxs(csv([]), "flow-whale:uninformed"), { + total: 0, + reverted: 0, + }); +}); diff --git a/test/run-config.test.ts b/test/run-config.test.ts index 60d0f11..3b7a26a 100644 --- a/test/run-config.test.ts +++ b/test/run-config.test.ts @@ -54,6 +54,49 @@ test("buildSource(nested schema) -> loadConfig: reflected in SimConfig", () => { assert.equal(config.initialWethWei, 0n); }); +test("market.*: the OU parameters reach SimConfig from YAML (ADR 0017 regime 1)", () => { + // These used to be module-level constants in sdk/src/rng.ts read straight from process.env, which + // a regime YAML has no way to set -- the loader builds a source map rather than mutating env. The + // cex-drift regime is nothing but these two numbers, so this is the test that it is a regime at all. + const source = buildSource({ + market: { volatility: 0.01, kappa: 0.004, drift: 0.0015 }, + }); + assert.equal(source.ERIS_PRICE_VOLATILITY, "0.01"); + assert.equal(source.ERIS_PRICE_REVERT_KAPPA, "0.004"); + assert.equal(source.ERIS_PRICE_DRIFT, "0.0015"); + + const config = loadConfig(source); + assert.deepEqual(config.ou.global, { + volatility: 0.01, + kappa: 0.004, + drift: 0.0015, + }); + // Every registered base resolves, falling back to the global value when it has no override. + assert.deepEqual(config.ou.perBase.WETH, config.ou.global); +}); + +test("market.*: unset falls back to the calm defaults", () => { + const config = loadConfig(buildSource({ run: { seed: 1 } })); + assert.deepEqual(config.ou.global, { + volatility: 0.004, + kappa: 0.02, + drift: 0, + }); +}); + +test("market.base*: per-base overrides expand to _ with no unit suffix", () => { + // Distinct from the funding/limits per-base maps, whose env names carry a unit (INITIAL_WETH_WEI). + // A volatility has no unit, so appending one would silently produce a key nothing reads. + const source = buildSource({ + market: { volatility: 0.004, baseVolatility: { WETH: 0.02 } }, + }); + assert.equal(source.ERIS_PRICE_VOLATILITY_WETH, "0.02"); + const config = loadConfig(source); + assert.equal(config.ou.perBase.WETH.volatility, 0.02); + // The global is untouched: only that base moved. + assert.equal(config.ou.global.volatility, 0.004); +}); + test("buildSource: expands the per-base map into __ (WETH=WEI)", () => { // WETH is in the fork default registry, so the unit suffix (WEI) can be derived. const source = buildSource({ funding: { base: { WETH: "5" } } }); diff --git a/test/runtimePrompt.test.ts b/test/runtimePrompt.test.ts deleted file mode 100644 index 39a5590..0000000 --- a/test/runtimePrompt.test.ts +++ /dev/null @@ -1,130 +0,0 @@ -import test from "node:test"; -import assert from "node:assert/strict"; -import { mkdtempSync, writeFileSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { - buildRevisionSystem, - buildRevisionUser, - buildSystemPrompt, - buildUserMessage, - loadPromptAgent, -} from "../example/agents/runtime/prompt.js"; -import type { AgentObservation } from "../sdk/src/types.js"; - -function writePrompt(content: string): string { - const dir = mkdtempSync(join(tmpdir(), "eris-prompt-")); - writeFileSync(join(dir, "prompt.md"), content); - return dir; -} - -test("loadPromptAgent: reads frontmatter (name/description required + optional fields)", () => { - const dir = writePrompt( - [ - "---", - "name: my-arb", - "description: cross-venue arbitrage", - "intervalMs: 4000", - "model: gpt-oss:120b", - "unknownField: ignored", // forward compat: unknown fields are ignored - "---", - "You are an arbitrage bot.", - "", - ].join("\n"), - ); - const agent = loadPromptAgent(dir); - assert.equal(agent.name, "my-arb"); - assert.equal(agent.description, "cross-venue arbitrage"); - assert.equal(agent.intervalMs, 4000); - assert.equal(agent.model, "gpt-oss:120b"); - assert.equal(agent.body, "You are an arbitrage bot."); -}); - -test("loadPromptAgent: a missing name is an explicit error", () => { - const dir = writePrompt( - ["---", "description: x", "---", "body", ""].join("\n"), - ); - assert.throws(() => loadPromptAgent(dir), /"name" is required/); -}); - -test("loadPromptAgent: missing frontmatter is an explicit error", () => { - const dir = writePrompt("body only\n"); - assert.throws(() => loadPromptAgent(dir), /frontmatter/); -}); - -test("buildSystemPrompt: composes + environment rules + prompt.md body (Hermes format)", () => { - const dir = writePrompt( - [ - "---", - "name: t", - "description: d", - "---", - "STRATEGY_BODY_MARKER", - "", - ].join("\n"), - ); - const agent = loadPromptAgent(dir); - const system = buildSystemPrompt(agent, ["uniswap"]); - assert.match(system, //); - assert.match(system, /<\/schema>/); - assert.match(system, /Environment rules/); - assert.match(system, /STRATEGY_BODY_MARKER/); - // the enabled-venue narrowing is reflected in - assert.doesNotMatch(system, /balancerSwap/); -}); - -test("buildUserMessage: fills in the observation and recent actions", () => { - const obs = { - kind: "observation", - round: 12, - fairPriceUsdcPerWeth: 3000, - } as unknown as AgentObservation; - const msg = buildUserMessage(obs, [ - { round: 11, action: { type: "noop" }, note: "skipped" }, - ]); - assert.match(msg, /round 12/); - assert.match(msg, /"fairPriceUsdcPerWeth":3000/); - assert.match(msg, /round=11/); - assert.match(msg, /skipped/); -}); - -test("buildRevisionSystem/User: the self-improvement prompt includes discipline, evidence, and the current body", () => { - const dir = writePrompt( - ["---", "name: rev-t", "description: d", "---", "OLD_BODY_MARKER", ""].join( - "\n", - ), - ); - const agent = loadPromptAgent(dir); - const system = buildRevisionSystem(agent); - assert.match(system, /rev-t/); - assert.match(system, /Revision discipline/); - assert.match(system, /Output ONLY the new prompt body/); - - const user = buildRevisionUser( - agent.body, - [{ round: 5, action: { type: "noop" }, note: "skipped" }], - { - cycles: 12, - initialValueUsdc: 1000, - currentValueUsdc: 990.5, - recentRevertRate: 0.25, - recentSampleSize: 8, - }, - ); - assert.match(user, /OLD_BODY_MARKER/); - assert.match(user, /12 decision cycles/); - assert.match(user, /1000\.00 -> 990\.50 USDC/); - assert.match(user, /25% over last 8 txs/); - assert.match(user, /round=5/); -}); - -test("buildRevisionUser: emits placeholders before any observation or fill", () => { - const user = buildRevisionUser("BODY", [], { - cycles: 0, - initialValueUsdc: null, - currentValueUsdc: null, - }); - assert.match(user, /\(not yet observed\)/); - assert.match(user, /\(no included txs yet\)/); - assert.match(user, /\(none\)/); -}); diff --git a/test/scoringBlocks.test.ts b/test/scoringBlocks.test.ts new file mode 100644 index 0000000..d40cbf8 --- /dev/null +++ b/test/scoringBlocks.test.ts @@ -0,0 +1,43 @@ +// Thinning the scoring cross-sections must not move any agent's score (ADR 0017 §3). +// +// The final score is alphaLast - alphaFirst, so the only thing that has to survive thinning is +// "the first block read is fromBlock and the last one read is toBlock". Everything between them +// is equity-curve resolution. These tests pin that invariant, because a thinning bug that drops +// either endpoint changes scores silently rather than failing. +import test from "node:test"; +import assert from "node:assert/strict"; +import { scoringBlocks } from "../core/src/realtime/reconstruct.js"; + +test("scoreEvery=1 reads every block in the window", () => { + assert.deepEqual(scoringBlocks(100, 105, 1), [100, 101, 102, 103, 104, 105]); +}); + +test("scoreEvery thins the interior but keeps both endpoints", () => { + assert.deepEqual(scoringBlocks(100, 108, 4), [100, 104, 108]); + // toBlock is not on the stride here: it still has to be the last cross-section. + assert.deepEqual(scoringBlocks(100, 110, 4), [100, 104, 108, 110]); +}); + +test("endpoints survive every stride, including strides larger than the window", () => { + for (const every of [1, 2, 3, 5, 7, 8, 100]) { + const blocks = scoringBlocks(100, 110, every); + assert.equal(blocks[0], 100, `fromBlock missing for every=${every}`); + assert.equal(blocks.at(-1), 110, `toBlock missing for every=${every}`); + // Strictly increasing, so alphaFirst/alphaLast cannot be assigned out of order. + for (let i = 1; i < blocks.length; i++) + assert.ok(blocks[i] > blocks[i - 1], `not increasing for every=${every}`); + } +}); + +test("a single-block window yields exactly one cross-section", () => { + // fromBlock === toBlock means alphaFirst and alphaLast are the same read, so the score is 0. + // Emitting the block twice would make it 0 as well, but it would double the reads. + assert.deepEqual(scoringBlocks(100, 100, 1), [100]); + assert.deepEqual(scoringBlocks(100, 100, 8), [100]); +}); + +test("a non-positive or fractional stride degrades to every block", () => { + assert.deepEqual(scoringBlocks(10, 13, 0), [10, 11, 12, 13]); + assert.deepEqual(scoringBlocks(10, 13, -5), [10, 11, 12, 13]); + assert.deepEqual(scoringBlocks(10, 14, 2.7), [10, 12, 14]); +}); diff --git a/test/standings.test.ts b/test/standings.test.ts new file mode 100644 index 0000000..6476495 --- /dev/null +++ b/test/standings.test.ts @@ -0,0 +1,175 @@ +// Scenario-matrix aggregation (ADR 0017 §4). +// +// The properties worth pinning are the ones that decide a competition: a regime with a bigger +// opportunity must not dominate the ranking, a crashed agent must not out-rank a finisher, and an +// environment failure must not be charged to the participants. +import test from "node:test"; +import assert from "node:assert/strict"; +import { + computeStandings, + scenarioZScores, + DISQUALIFIED_Z_PENALTY, + type ScenarioResult, +} from "../core/src/backtest/standings.js"; + +const agent = (id: string, netPnlUsdc: number, disqualified?: string) => ({ + id, + netPnlUsdc, + ...(disqualified !== undefined ? { disqualified } : {}), +}); + +test("z-scores are centered on the scenario's finishers", () => { + const { z } = scenarioZScores( + [agent("a", 800), agent("b", 200), agent("c", -100)], + "netPnlUsdc", + ); + // mean 300, population sd sqrt(140000) ~ 374.17 + assert.ok(Math.abs(z.a - 1.336) < 0.01, `z.a=${z.a}`); + assert.ok(Math.abs(z.b - -0.267) < 0.01, `z.b=${z.b}`); + assert.ok(Math.abs(z.c - -1.069) < 0.01, `z.c=${z.c}`); + // Centering means they sum to zero: nobody gains ground without someone losing it. + assert.ok(Math.abs(z.a + z.b + z.c) < 1e-9); +}); + +test("a tie gives everyone zero rather than dividing by zero", () => { + const { z } = scenarioZScores( + [agent("a", 0), agent("b", 0), agent("c", 0)], + "netPnlUsdc", + ); + assert.deepEqual(z, { a: 0, b: 0, c: 0 }); +}); + +test("a lone finisher scores zero (there is nobody to be better than)", () => { + const { z } = scenarioZScores([agent("a", 500)], "netPnlUsdc"); + assert.deepEqual(z, { a: 0 }); +}); + +test("a disqualified agent lands below the worst finisher", () => { + const { z, disqualified } = scenarioZScores( + [agent("a", 800), agent("b", 200), agent("c", 9999, "process died")], + "netPnlUsdc", + ); + assert.equal(disqualified.c, "process died"); + const worstFinisher = Math.min(z.a, z.b); + assert.equal(z.c, worstFinisher - DISQUALIFIED_Z_PENALTY); + // Crucially: a huge raw score does not rescue it. + assert.ok(z.c < z.a && z.c < z.b); +}); + +test("an agent with no readable metric is disqualified, not scored as zero", () => { + // Scoring it as 0 would place it mid-pack in a scenario where everyone lost money. + const { z, disqualified } = scenarioZScores( + [agent("a", -500), agent("b", -300), { id: "c" }], + "netPnlUsdc", + ); + assert.match(disqualified.c, /no netPnlUsdc/); + assert.ok(z.c < z.a && z.c < z.b); +}); + +test("regime equal weighting stops the big-opportunity regime from deciding the ranking", () => { + // crash pays in the hundreds, calm in the tens. On a raw sum a wins; the ranking should say b. + const results: ScenarioResult[] = [ + { + regime: "crash", + seed: 1, + agents: [agent("a", 800), agent("b", 200), agent("c", -100)], + }, + { + regime: "calm", + seed: 1, + agents: [agent("a", 10), agent("b", 40), agent("c", 25)], + }, + ]; + const rawSum = { a: 810, b: 240, c: -75 }; + assert.ok(rawSum.a > rawSum.b, "precondition: a wins on the raw sum"); + + const standings = computeStandings(results, "netPnlUsdc"); + assert.deepEqual( + standings.agents.map((x) => x.id), + ["b", "a", "c"], + ); +}); + +test("seed count inside a regime does not change that regime's weight", () => { + // calm is run three times and crash once. calm must still be worth exactly half the total. + const calm = (seed: number): ScenarioResult => ({ + regime: "calm", + seed, + agents: [agent("a", 10), agent("b", 40)], + }); + const many = computeStandings( + [ + { regime: "crash", seed: 1, agents: [agent("a", 800), agent("b", 200)] }, + calm(1), + calm(2), + calm(3), + ], + "netPnlUsdc", + ); + const one = computeStandings( + [ + { regime: "crash", seed: 1, agents: [agent("a", 800), agent("b", 200)] }, + calm(1), + ], + "netPnlUsdc", + ); + const totalOf = (s: typeof many, id: string) => + s.agents.find((x) => x.id === id)?.total ?? NaN; + assert.ok(Math.abs(totalOf(many, "a") - totalOf(one, "a")) < 1e-9); + assert.ok(Math.abs(totalOf(many, "b") - totalOf(one, "b")) < 1e-9); +}); + +test("a scenario with no summary is excluded, not scored as a row of zeros", () => { + const standings = computeStandings( + [ + { regime: "calm", seed: 1, agents: [agent("a", 100), agent("b", -100)] }, + { regime: "calm", seed: 2, error: "anvil died" }, + ], + "netPnlUsdc", + ); + assert.equal(standings.scenarios.length, 1); + assert.deepEqual(standings.excludedScenarios, [ + { regime: "calm", seed: 2, error: "anvil died" }, + ]); + // The surviving scenario alone decides the ranking; the dead one dilutes nothing. + const a = standings.agents.find((x) => x.id === "a"); + assert.equal(a?.scenariosScored, 1); + assert.ok((a?.total ?? 0) > 0); +}); + +test("disqualifications are counted per agent", () => { + const standings = computeStandings( + [ + { + regime: "crash", + seed: 1, + agents: [agent("a", 100), agent("b", 0, "fee cap violation")], + }, + { + regime: "crash", + seed: 2, + agents: [agent("a", 100), agent("b", 50)], + }, + ], + "netPnlUsdc", + ); + const b = standings.agents.find((x) => x.id === "b"); + assert.equal(b?.disqualifications, 1); + assert.equal(b?.scenariosScored, 2); +}); + +test("the metric is selectable, and the two can disagree on the winner", () => { + // a is up on gross PnL purely by holding a rising asset; b took the edge. + const results: ScenarioResult[] = [ + { + regime: "calm", + seed: 1, + agents: [ + { id: "a", netPnlUsdc: 1000, alphaUsdc: 0 }, + { id: "b", netPnlUsdc: 200, alphaUsdc: 200 }, + ], + }, + ]; + assert.equal(computeStandings(results, "netPnlUsdc").agents[0].id, "a"); + assert.equal(computeStandings(results, "alphaUsdc").agents[0].id, "b"); +}); diff --git a/test/strategyStaticCheck.test.ts b/test/strategyStaticCheck.test.ts index 4d07e85..4cd2b2c 100644 --- a/test/strategyStaticCheck.test.ts +++ b/test/strategyStaticCheck.test.ts @@ -1,6 +1,6 @@ import test from "node:test"; import assert from "node:assert/strict"; -import { findCheatcodeUsage } from "../core/src/strategyStaticCheck.js"; +import { findCheatcodeUsage } from "@eris/sdk/strategyStaticCheck.js"; test("findCheatcodeUsage: detects cheatcode RPC with line numbers", () => { const source = [ diff --git a/test/summaryMultiBaseValuation.test.ts b/test/summaryMultiBaseValuation.test.ts new file mode 100644 index 0000000..1a7b109 --- /dev/null +++ b/test/summaryMultiBaseValuation.test.ts @@ -0,0 +1,84 @@ +// summary.json's netPnlUsdc must price every base, not just WETH. +// +// valueUsdc marks an unlisted base at `p[sym] ?? 0`, so handing it the scalar WETH price silently +// values an agent's WBTC at zero: whoever ends the run holding a non-WETH base has that inventory +// deleted from their PnL. Found in the ADR 0017 §5 pilot -- on a 24-agent calm run the WBTC-trading +// agents reported a reproducible -6,686 USDC loss (to the cent, across four independent addresses +// and two repeats) while the scoring reconstruction, which prices every base since issue #41, put +// the same agents at +13 alpha. +// +// This is the same class of bug as issue #41, on the other scoring path: #41 fixed reconstruct.ts +// and left the coordinator's end-of-run summary untouched. It matters more now that ADR 0017 ranks +// the competition on netPnlUsdc. +// +// The multi-base assertions need a registry that actually has a second base, which only the local +// deploy constants provide (the fork default is WETH-only), so they skip elsewhere. The invariant +// they protect is enforced at the call site in coordinator.ts, which passes ctx.fairPrices. +import test from "node:test"; +import assert from "node:assert/strict"; +import { valueUsdc } from "@eris/sdk/pnl.js"; +import { baseTokens } from "@eris/sdk/markets.js"; +import type { BalanceSnapshot } from "@eris/sdk/types.js"; + +// A registered base that is not WETH, if the active registry has one. +const extraBase = baseTokens().find((t) => t.symbol !== "WETH"); + +test("the per-base map values every base the agent holds", (t) => { + if (!extraBase) { + t.skip( + "registry has no non-WETH base (fork default); needs local constants", + ); + return; + } + const unit = 10n ** BigInt(extraBase.decimals); + const held: BalanceSnapshot = { + ethWei: 0n, + wethWei: 10n ** 18n, + usdcUnits: 1_000_000_000n, // 1,000 USDC + bases: { WETH: 10n ** 18n, [extraBase.symbol]: unit }, + }; + const prices = { WETH: 2000, [extraBase.symbol]: 60_000 }; + assert.equal(valueUsdc(held, prices), 1000 + 2000 + 60_000); + // The scalar form is the bug: the same holding loses the entire extra base. + assert.equal(valueUsdc(held, 2000), 1000 + 2000); +}); + +test("a round trip into a non-WETH base is not a loss", (t) => { + if (!extraBase) { + t.skip( + "registry has no non-WETH base (fork default); needs local constants", + ); + return; + } + // The shape of the pilot failure: start in USDC, end holding the base bought at fair. Priced + // correctly that is flat; priced with the scalar it looks like the position was burned. + const unit = 10n ** BigInt(extraBase.decimals); + const prices = { WETH: 2000, [extraBase.symbol]: 60_000 }; + const before: BalanceSnapshot = { + ethWei: 0n, + wethWei: 0n, + usdcUnits: 60_000_000_000n, // 60,000 USDC + bases: { WETH: 0n }, + }; + const after: BalanceSnapshot = { + ethWei: 0n, + wethWei: 0n, + usdcUnits: 0n, + bases: { WETH: 0n, [extraBase.symbol]: unit }, + }; + assert.equal(valueUsdc(after, prices) - valueUsdc(before, prices), 0); + assert.equal(valueUsdc(after, 2000) - valueUsdc(before, 2000), -60_000); +}); + +test("a WETH-only holding is valued the same either way", () => { + // The scalar form stays correct for the WETH-only case, which is why the bug went unnoticed: + // every regime before multi-asset ended with agents holding only WETH and USDC. + const held: BalanceSnapshot = { + ethWei: 0n, + wethWei: 3n * 10n ** 18n, + usdcUnits: 1_000_000_000n, + bases: { WETH: 3n * 10n ** 18n }, + }; + assert.equal(valueUsdc(held, 2000), valueUsdc(held, { WETH: 2000 })); + assert.equal(valueUsdc(held, 2000), 1000 + 6000); +}); diff --git a/test/whale.test.ts b/test/whale.test.ts new file mode 100644 index 0000000..aa5debf --- /dev/null +++ b/test/whale.test.ts @@ -0,0 +1,210 @@ +// Whale point stress event (ADR 0017 regime 3). +// +// The schedule half (placement, side resolution, determinism from the seed) and the order half +// (what actually gets submitted) are both pure, so both are pinned here. The thing most worth +// pinning is that a whale is a *point* event: if it were treated as a trapezoid it would silently +// become a multi-block price overlay, which is the crash regime, not this one. +import test from "node:test"; +import assert from "node:assert/strict"; +import { + EventSchedule, + parseStressEvents, +} from "../core/src/realtime/events.js"; +import { buildWhaleOrder, whaleFunding } from "../core/src/realtime/whale.js"; +import { baseTokens } from "@eris/sdk/markets.js"; + +const WHALE = { + type: "whale" as const, + magnitudeRange: [30, 60] as [number, number], + windowFrac: [0.3, 0.7] as [number, number], + rampBlocks: 0, + holdBlocks: 0, + decayBlocks: 0, +}; + +test("a whale occupies a single block and never touches the price overlay", () => { + const schedule = new EventSchedule([WHALE], 42, 100); + const [ev] = schedule.events; + assert.equal(ev.endBlock, ev.startBlock + 1); + // Unlike crash, fair price is untouched: the dislocation is the pool moving away from an + // unchanged fair, which is the whole point of the regime. + for (let i = 0; i < 100; i++) assert.equal(schedule.at(i).wethMult, 1); + // It is delivered as a point event instead. + assert.deepEqual(schedule.pointEventsAt(ev.startBlock), [ev]); + assert.deepEqual(schedule.pointEventsAt(ev.startBlock + 1), []); +}); + +test("the same seed resolves the same whale, a different seed can move it", () => { + const a = new EventSchedule([WHALE], 42, 100).events[0]; + const b = new EventSchedule([WHALE], 42, 100).events[0]; + assert.deepEqual(a, b); + + const differs = [1, 2, 3, 4, 5, 6, 7, 8].some((seed) => { + const e = new EventSchedule([WHALE], seed, 100).events[0]; + return e.startBlock !== a.startBlock || e.magnitude !== a.magnitude; + }); + assert.ok( + differs, + "no seed changed the whale (the range is not being sampled)", + ); +}); + +test("side defaults to seed-chosen and both sides occur across seeds", () => { + const sides = new Set( + Array.from( + { length: 24 }, + (_, seed) => new EventSchedule([WHALE], seed, 100).events[0].side, + ), + ); + assert.deepEqual([...sides].sort(), ["buy", "sell"]); +}); + +test("an explicit side pins the direction without moving the schedule", () => { + // The side draw is taken unconditionally so that pinning it cannot shift the placement of this + // event or of any event after it in the list. + const free = new EventSchedule([WHALE], 7, 100).events[0]; + const pinned = new EventSchedule([{ ...WHALE, side: "buy" }], 7, 100) + .events[0]; + assert.equal(pinned.side, "buy"); + assert.equal(pinned.startBlock, free.startBlock); + assert.equal(pinned.magnitude, free.magnitude); +}); + +test("a sell spends the base and a buy spends USDC, at the venue asked for", () => { + const base = { + ...new EventSchedule([WHALE], 42, 100).events[0], + magnitude: 40, + }; + + const sell = buildWhaleOrder({ ...base, side: "sell" }, 2000, 1n); + assert.equal(sell.protocol, "uniswap"); + assert.equal(sell.walletKey, "whale:uninformed"); + const sellAction = sell.action as unknown as Record; + assert.equal(sellAction.type, "swap"); + assert.equal(sellAction.tokenIn, "WETH"); + assert.equal(sellAction.amountIn, (40n * 10n ** 18n).toString()); + + const buy = buildWhaleOrder( + { ...base, side: "buy", venue: "curve" }, + 2000, + 1n, + ); + const buyAction = buy.action as unknown as Record; + assert.equal(buy.protocol, "curve"); + assert.equal(buyAction.type, "curveSwap"); + assert.equal(buyAction.tokenIn, "USDC"); + // 40 WETH at 2000 = 80,000 USDC (6 decimals) + assert.equal(buyAction.amountIn, (80_000n * 10n ** 6n).toString()); +}); + +test("a whale accepts any fill: capping slippage would cap the event itself", () => { + const ev = new EventSchedule([WHALE], 42, 100).events[0]; + const action = buildWhaleOrder(ev, 2000, 1n).action as unknown as Record< + string, + string + >; + assert.equal(action.minAmountOut, "0"); +}); + +const whaleEvent = (over: Record) => ({ + ...new EventSchedule([WHALE], 42, 100).events[0], + ...over, +}); + +test("funding covers the cumulative same-side notional, not just the largest order", () => { + // Sizing on the max only looks sufficient because buys and sells replenish each other. A seed that + // draws every whale on the same side spends the sum, and the last order would revert on balance. + const allSells = [ + whaleEvent({ side: "sell", magnitude: 50 }), + whaleEvent({ side: "sell", magnitude: 50 }), + whaleEvent({ side: "sell", magnitude: 50 }), + ]; + const funding = whaleFunding(allSells, { WETH: 2000 }); + assert.ok( + funding.baseWei.WETH >= 150n * 10n ** 18n, + `baseWei=${funding.baseWei.WETH} does not cover 150 WETH of sells`, + ); + // Nothing was bought, so no USDC is needed. + assert.equal(funding.usdcUnits, 0n); +}); + +test("the two sides are funded separately rather than netted", () => { + // Order matters: a sell that comes before the buy needs its base up front regardless of what the + // buy would have replenished afterwards. + const funding = whaleFunding( + [ + whaleEvent({ side: "sell", magnitude: 40 }), + whaleEvent({ side: "buy", magnitude: 40 }), + ], + { WETH: 2000 }, + ); + assert.ok(funding.baseWei.WETH >= 40n * 10n ** 18n); + assert.ok(funding.usdcUnits >= 80_000n * 10n ** 6n); +}); + +test("a non-WETH whale is funded in its own base at its own price", (t) => { + // Previously hard-coded to WETH: a WBTC whale got WETH it could not sell and USDC priced off the + // WETH feed, so it reverted either way -- silently. + const extra = baseTokens().find((b) => b.symbol !== "WETH"); + if (!extra) { + t.skip( + "registry has no non-WETH base (fork default); needs local constants", + ); + return; + } + const funding = whaleFunding( + [whaleEvent({ side: "sell", base: extra.symbol, magnitude: 5 })], + { WETH: 2000, [extra.symbol]: 60_000 }, + ); + assert.equal(funding.baseWei.WETH, undefined); + assert.ok( + funding.baseWei[extra.symbol] >= 5n * 10n ** BigInt(extra.decimals), + ); +}); + +test("a whale whose base has no fair price fails fast rather than going unfunded", () => { + assert.throws( + () => whaleFunding([whaleEvent({ side: "buy", base: "WETH" })], {}), + /no fair price is available/, + ); +}); + +test("no whale in the schedule means no endowment", () => { + assert.deepEqual(whaleFunding([], { WETH: 2000 }), { + baseWei: {}, + usdcUnits: 0n, + }); +}); + +test("side/venue are rejected on event types they do not apply to", () => { + const crash = { + type: "crash", + magnitudeRange: [0.1, 0.2], + windowFrac: [0.3, 0.7], + rampBlocks: 3, + holdBlocks: 6, + decayBlocks: 8, + }; + assert.throws( + () => parseStressEvents(JSON.stringify([{ ...crash, side: "buy" }])), + /side only applies to type "whale"/, + ); + assert.throws( + () => parseStressEvents(JSON.stringify([{ ...crash, venue: "curve" }])), + /venue only applies to type "whale"/, + ); + assert.throws( + () => parseStressEvents(JSON.stringify([{ ...WHALE, side: "sideways" }])), + /side must be/, + ); +}); + +test("a whale needs no trapezoid fields", () => { + const [parsed] = parseStressEvents( + JSON.stringify([ + { type: "whale", magnitudeRange: [30, 60], windowFrac: [0.3, 0.7] }, + ]), + ); + assert.equal(parsed.type, "whale"); + assert.equal(parsed.rampBlocks, 0); +});