From ea2a7e6579edfc28f58ef96d5a98fc2796f0def0 Mon Sep 17 00:00:00 2001 From: Seth Date: Fri, 18 Sep 2026 02:19:57 -0400 Subject: [PATCH] feat(coding-agent): DAG swarm workflows (swarm harness kind, executor, eval) Declarative task-graph orchestration for subagent swarms, from the "Swarm DAGs: declarative orchestration in Continual Harness" spec. Three components in one PR: 1. Swarm specification (harness entry kind "swarm"): a versioned DAG of subagent nodes with stable ids, subagent references (harness entries or inline specs), task/resident lifecycles, typed input/output ports, per-node and run budgets, retries, fail_fast/continue/escalate failure policies, and bounded foreach fan-out. Write-time dry run: validate_swarm_spec enforces 12 rule groups (ids, cycles over depends_on + data edges, port type matching, resident constraints, foreach bounds, budget sanity, 1024-node cap); an invalid DAG never reaches the store. rlm.harness.create_swarm/update_swarm/delete_swarm; /refine accepts the kind; the harness digest renders the invoke contract. 2. Executor (rlm.swarm.run/status/stop/resume): nonblocking admission (re-validates, resolves references, reports all failures, starts only ready nodes, ends the model turn), then a kernel control loop resumes through bounded collect polls; binds typed outputs into dependent prompts (json ports via fenced blocks), foreach fan-out with max clamp, retries, rate-limit backoff, injectable-clock budgets, failure policies (fail_fast cancels in-flight siblings, escalate pauses with one quiet notice), cancellation cascade via delete_subagent, event ledger with arrived/shown/read stages, and one swarm.progress notice per milestone (mirrors bash.completed). asyncio imports deferred to preserve the repl boot contract. 3. Capability evaluation harness: three reference swarms (review sweep with typed fan-in and escalation, N-wide builder with per-node budgets, resident watcher with teardown) against hand-written baselines with identical inputs/models/budgets; deterministic replay and ledger checks, verdict rules per the spec, metrics report. Never runs in CI (no token spend); offline checks are unit-tested. Review history: PR-G round caught and fixed foreach mixed-instance policy handling and a stop() race (verified by repro re-runs); PR-H round fixed replay seq gaps, retry settle accounting, baseline tautology and fairness (single-listing substitution), and fail-fast --swarms validation. All suites green: 90+122 py, 134+52 TS, tsgo and biome clean; rlm boot contract verified. Known merge-time item: main's test-line budget (check:test-policy) fails by design - the feature is contract-test-heavy (~1.8k net test lines over source); the reviewed-override mechanism lives on the unmerged security/release-hardening branch. Consolidates #2397, #2401, #2402 (closed in favor of this PR). --- .../coding-agent/.changes/swarm-dag-eval.md | 1 + .../coding-agent/scripts/swarm-dag-eval.ts | 1875 +++++++++++++++++ .../coding-agent/src/core/agent-messages.ts | 4 +- .../coding-agent/src/core/agent-session.ts | 31 + packages/coding-agent/src/core/messages.ts | 30 + packages/coding-agent/src/core/prompts/rlm.ts | 4 +- .../src/core/refinement/refinement.ts | 19 +- packages/coding-agent/src/core/rlm-runtime.ts | 51 + packages/coding-agent/test/refinement.test.ts | 121 +- .../6009-opencode-maintenance-session.test.ts | 2 +- .../6010-auxiliary-reasoning.test.ts | 2 +- .../6010-refinement-output-budget.test.ts | 2 +- .../coding-agent/test/swarm-dag-eval.test.ts | 919 ++++++++ .../coding-agent/test/swarm-executor.test.ts | 78 + prime-agent-runtime/src/rlm/__init__.py | 24 + prime-agent-runtime/src/rlm/harness.py | 72 +- prime-agent-runtime/src/rlm/swarm.py | 1456 +++++++++++++ prime-agent-runtime/test/test_harness.py | 123 +- .../test/test_swarm_executor.py | 1038 +++++++++ prime-agent-runtime/test/test_swarm_spec.py | 642 ++++++ tsconfig.json | 7 +- 21 files changed, 6484 insertions(+), 17 deletions(-) create mode 100644 packages/coding-agent/.changes/swarm-dag-eval.md create mode 100644 packages/coding-agent/scripts/swarm-dag-eval.ts create mode 100644 packages/coding-agent/test/swarm-dag-eval.test.ts create mode 100644 packages/coding-agent/test/swarm-executor.test.ts create mode 100644 prime-agent-runtime/src/rlm/swarm.py create mode 100644 prime-agent-runtime/test/test_swarm_executor.py create mode 100644 prime-agent-runtime/test/test_swarm_spec.py diff --git a/packages/coding-agent/.changes/swarm-dag-eval.md b/packages/coding-agent/.changes/swarm-dag-eval.md new file mode 100644 index 0000000000..ee6a62b26a --- /dev/null +++ b/packages/coding-agent/.changes/swarm-dag-eval.md @@ -0,0 +1 @@ +- Added the swarm DAG capability evaluation harness: three reference swarms (review sweep, N-wide builder, resident watcher) against hand-written baselines, with deterministic replay/ledger checks, verdict rules, and a metrics report; never runs in CI (no token spend). diff --git a/packages/coding-agent/scripts/swarm-dag-eval.ts b/packages/coding-agent/scripts/swarm-dag-eval.ts new file mode 100644 index 0000000000..239dd937e7 --- /dev/null +++ b/packages/coding-agent/scripts/swarm-dag-eval.ts @@ -0,0 +1,1875 @@ +#!/usr/bin/env node +/** + * Swarm DAG capability evaluation — PR H of the swarm DAG feature set. + * Spec: "Swarm DAGs: declarative orchestration in Continual Harness" + * https://app.notion.com/p/3da72940136f81a88554e6ec7119270e — section "Proposed evaluation". + * + * Runs the capability layer against REAL sessions with live models: + * - three reference swarms (review-sweep, n-wide-builder, resident-watcher), each + * paired with a hand-written manual-orchestration baseline (rlm.spawn + rlm.collect) + * for the same topology, inputs, model, and declared budget; + * - one escalation trial of review-sweep with a planted failing reviewer (the declared + * escalate policy must pause the run and start nothing further); + * - one dry-run rejection trial (a broken spec must make rlm.swarm.run raise and + * start no node). + * + * Plus the deterministic replay check: --replay re-verifies + * saved run ledgers for stable event identities and complete resource accounting. + * + * This script spends real model tokens. It NEVER runs in CI; the deterministic pieces + * (spec shapes, prompt builders, answer checkers, replay checker, report renderer) are + * unit-tested in test/swarm-dag-eval.test.ts. The live run is the reviewer's call. + * + * Usage: + * npx tsx scripts/swarm-dag-eval.ts \ + * --model internal/glm-5.2-fast --swarms review-sweep,builder,resident-watcher \ + * --width 6 --trials 1 --out ./swarm-dag-eval-reports + * npx tsx scripts/swarm-dag-eval.ts --replay ./swarm-dag-eval-reports/report.json + */ + +import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { AuthStorage } from "../src/core/auth-storage.js"; +import { calculateContextTokens } from "../src/core/compaction/compaction.js"; +import { SWARM_PROGRESS_NOTICE_CUSTOM_TYPE } from "../src/core/messages.js"; +import { ModelRegistry } from "../src/core/model-registry.js"; +import { getSessionArtifactPath, SessionManager } from "../src/core/session-manager.js"; +import { SettingsManager } from "../src/core/settings-manager.js"; +import { createAgentSession } from "../src/core/sdk.js"; +import { getAgentDir } from "../src/config.js"; + +// --------------------------------------------------------------------------- +// Spec types (mirror the kernel-side swarm dag schema; the kernel validates on +// write at run time, so the TS side only builds and shapes-checks them). +// --------------------------------------------------------------------------- + +export type PortType = "text" | "json"; + +export interface SwarmDagPort { + name: string; + type: PortType; + from?: string; +} + +export interface SwarmDagOutputPort { + name: string; + type: PortType; +} + +export interface SwarmDagForeach { + over: string; + max: number; +} + +export type FailurePolicy = "fail_fast" | "continue" | "escalate"; + +export interface InlineSubagent { + prompt: string; + name?: string; + model?: string; + thinking?: string; +} + +export interface SwarmDagNodeSpec { + id: string; + subagent: string | InlineSubagent; + lifecycle?: "task" | "resident"; + depends_on?: string[]; + inputs?: SwarmDagPort[]; + outputs?: SwarmDagOutputPort[]; + budget_ms?: number; + retries?: number; + failure_policy?: FailurePolicy; + foreach?: SwarmDagForeach; +} + +export interface SwarmDagRunSpec { + budget_ms?: number; + failure_policy?: FailurePolicy; + max_parallel?: number; +} + +export interface SwarmDagSpec { + run?: SwarmDagRunSpec; + nodes: SwarmDagNodeSpec[]; +} + +export type ReferenceSwarmKind = + | "review-sweep" + | "builder" + | "resident-watcher" + | "review-sweep-fail" + | "dry-run-reject"; + +export interface ReferenceSwarm { + id: string; + kind: ReferenceSwarmKind; + title: string; + description: string; + dag: SwarmDagSpec; + declaredBudgetMs: number; + declaredFanIn: number; + width?: number; +} + +// --------------------------------------------------------------------------- +// Constants shared by the swarms, their baselines, and the checkers. +// --------------------------------------------------------------------------- + +/** Per-node wall-clock budget (admission to settlement) declared on every task node. */ +export const NODE_BUDGET_MS = 240_000; +/** Whole-run wall-clock budget declared on every reference swarm. */ +export const RUN_BUDGET_MS = 900_000; +export const REVIEW_FOREACH_MAX = 8; +export const DEFAULT_WIDTH = 6; +export const MAX_WIDTH = 12; +export const DEFAULT_MODEL = "internal/glm-5.2-fast"; +export const RESIDENT_WATCHER_SLEEP_SECONDS = 900; + +export interface ReviewFile { + name: string; + code: string; + issueId: string; + audit: string; +} + +/** Four review files, each with one real, checkable planted defect and its audit id. */ +export const REVIEW_FILES: ReviewFile[] = [ + { + name: "fa", + code: "export function clampUpper(value, max) {\n\treturn Math.max(value, max);\n}", + issueId: "AUDIT-A1", + audit: "callers expect the value capped at max, but Math.max returns the larger operand, so values above max pass through unclamped", + }, + { + name: "fb", + code: "export function medianSorted(values) {\n\treturn values[Math.floor(values.length / 2)];\n}", + issueId: "AUDIT-B1", + audit: "the median of an even-length sorted list is the average of the two middle values, but this returns only the upper middle", + }, + { + name: "fc", + code: 'export function isBlank(text) {\n\treturn text === "";\n}', + issueId: "AUDIT-C1", + audit: "whitespace-only strings should count as blank, but the strict equality comparison misses them", + }, + { + name: "fd", + code: "const RETRY_DELAY_SECONDS = 30;\nexport const RETRY_DELAY_MS = RETRY_DELAY_SECONDS;", + issueId: "AUDIT-D1", + audit: "RETRY_DELAY_MS is documented as milliseconds but the constant was meant as seconds; the unit conversion is missing", + }, +]; + +export const REVIEW_FILE_NAMES = REVIEW_FILES.map((file) => file.name); +export const REVIEW_ISSUE_IDS = REVIEW_FILES.map((file) => file.issueId); + +export const BUILDER_MARKER = (index: number) => `swb-marker-${index}`; +export const TASK_MARKER_A = "swt-1"; +export const TASK_MARKER_B = "swt-2"; +export const WATCHER_MARKER = "swr-marker"; + +function miniRepoListing(): string { + return REVIEW_FILES.map((file) => `[${file.name}] ${file.code} // audit ${file.issueId}: ${file.audit}`).join("\n"); +} + +// --------------------------------------------------------------------------- +// Child prompt builders (shared verbatim between swarm nodes and baselines). +// --------------------------------------------------------------------------- + +export function buildFilesNodePrompt(): string { + const json = JSON.stringify({ files: REVIEW_FILE_NAMES }); + return [ + "You are the source node of a pull-request review sweep. Reply with exactly one fenced json block and nothing else:", + "", + "```json", + json, + "```", + ].join("\n"); +} + +/** Reviewer prompt for one file; `{files}` is the foreach placeholder. */ +export function buildReviewerPromptTemplate(): string { + return [ + "You are one code reviewer in a pull-request review sweep.", + "", + "Mini-repo under review (four files, one planted defect each, audit note included):", + "", + miniRepoListing(), + "", + "Your assigned file is {files}. Verify its defect is real, then reply with exactly one line:", + "FOUND ", + "Output that single line and nothing else, then end your turn.", + ].join("\n"); +} + +export function buildReportNodePrompt(): string { + return [ + "You are the aggregation node of a pull-request review sweep.", + "", + "Files under review (json): {file_list}", + "Reviewer reports (one FOUND line per file): {found}", + "", + "Every file has exactly one audit id. Cross-check that every reviewer report carries one, then reply with exactly one fenced json block and nothing else:", + "", + '```json\n{"issues": []}\n```', + ].join("\n"); +} + +export function buildBrokenReviewerPrompt(): string { + return [ + "You are one code reviewer in a pull-request review sweep. Your assigned file is fa. Verify its defect, then reply with exactly one line:", + "FOUND AUDIT-A1", + "Output that single line and nothing else, then end your turn.", + ].join("\n"); +} + +export function buildBuilderNodePrompt(index: number): string { + return [ + `You are builder node ${index} of a wide build. Reply with exactly one line:`, + `BUILT ${BUILDER_MARKER(index)}`, + "Output that single line and nothing else, then end your turn.", + ].join("\n"); +} + +export function buildCollectorPrompt(width: number): string { + const lines = Array.from({ length: width }, (_, i) => `{line-${i + 1}}`).join("\n"); + const markers = Array.from({ length: width }, (_, i) => BUILDER_MARKER(i + 1)).join(" "); + return [ + `You are the collector of a ${width}-wide build. One line per builder node arrived:`, + "", + lines, + "", + "Merge them. Reply with exactly one line:", + `COLLECTED (expected form: COLLECTED ${markers})`, + "Output that single line and nothing else, then end your turn.", + ].join("\n"); +} + +export function buildWatcherPrompt(): string { + return [ + "You are a resident watcher node attached to an orchestration run. Do exactly this, in order:", + "", + '1. In the ipython tool, send your parent one message with exactly this text:', + ` await agent_message.send("WATCHER-UP ${WATCHER_MARKER}", receiver_role="parent")`, + "2. Then, still in the ipython tool, run:", + " import asyncio", + ` await asyncio.sleep(${RESIDENT_WATCHER_SLEEP_SECONDS})`, + " and stay idle. Do not end your turn before the sleep finishes. Do not send more messages. Do nothing else.", + ].join("\n"); +} + +export function buildTaskAPrompt(): string { + return [ + "You are task node A of a tiny two-step chain. Reply with exactly one line:", + `STEP ${TASK_MARKER_A}`, + "Output that single line and nothing else, then end your turn.", + ].join("\n"); +} + +export function buildTaskBPromptTemplate(): string { + return [ + "You are task node B of a tiny two-step chain. The previous step reported: {prev}", + "Reply with exactly one line:", + `STEP ${TASK_MARKER_B}`, + "Output that single line and nothing else, then end your turn.", + ].join("\n"); +} + +// --------------------------------------------------------------------------- +// Reference swarm spec builders. +// --------------------------------------------------------------------------- + +export function buildReviewSweepDag(): SwarmDagSpec { + return { + run: { budget_ms: RUN_BUDGET_MS, failure_policy: "escalate", max_parallel: REVIEW_FOREACH_MAX }, + nodes: [ + { + id: "files", + subagent: { prompt: buildFilesNodePrompt(), name: "files-source" }, + outputs: [{ name: "files", type: "json" }], + budget_ms: NODE_BUDGET_MS, + }, + { + id: "review", + subagent: { prompt: buildReviewerPromptTemplate(), name: "file-reviewer" }, + inputs: [{ name: "files", type: "json", from: "files.files" }], + outputs: [{ name: "found", type: "text" }], + foreach: { over: "files", max: REVIEW_FOREACH_MAX }, + budget_ms: NODE_BUDGET_MS, + }, + { + id: "report", + subagent: { prompt: buildReportNodePrompt(), name: "review-aggregator" }, + inputs: [ + { name: "file_list", type: "json", from: "files.files" }, + { name: "found", type: "text", from: "review.found" }, + ], + outputs: [{ name: "issues", type: "json" }], + budget_ms: NODE_BUDGET_MS, + }, + ], + }; +} + +/** review-sweep plus a planted failing reviewer: admission of an unresolvable model pin. */ +export function buildReviewSweepFailDag(): SwarmDagSpec { + const dag = buildReviewSweepDag(); + const nodes = [...dag.nodes]; + // Insert after the foreach so it starts once the sweep is under way. + nodes.splice(2, 0, { + id: "review-broken", + subagent: { prompt: buildBrokenReviewerPrompt(), model: "internal/no-such-model-for-eval", name: "broken-reviewer" }, + depends_on: ["files"], + budget_ms: NODE_BUDGET_MS, + failure_policy: "escalate", + }); + return { ...dag, nodes }; +} + +export function buildBuilderDag(width: number): SwarmDagSpec { + const nodes: SwarmDagNodeSpec[] = Array.from({ length: width }, (_, i) => ({ + id: `builder-${i + 1}`, + subagent: { prompt: buildBuilderNodePrompt(i + 1), name: `builder-${i + 1}` }, + outputs: [{ name: "line", type: "text" }], + budget_ms: NODE_BUDGET_MS, + })); + nodes.push({ + id: "collector", + subagent: { prompt: buildCollectorPrompt(width), name: "build-collector" }, + inputs: Array.from({ length: width }, (_, i) => ({ + name: `line-${i + 1}`, + type: "text", + from: `builder-${i + 1}.line`, + })), + outputs: [{ name: "merged", type: "text" }], + budget_ms: NODE_BUDGET_MS, + }); + return { run: { budget_ms: RUN_BUDGET_MS, failure_policy: "escalate", max_parallel: 8 }, nodes }; +} + +export function buildResidentWatcherDag(): SwarmDagSpec { + return { + run: { budget_ms: RUN_BUDGET_MS, failure_policy: "escalate", max_parallel: 8 }, + nodes: [ + { + id: "watcher", + subagent: { prompt: buildWatcherPrompt(), name: "resident-watcher" }, + lifecycle: "resident", + }, + { + id: "task-a", + subagent: { prompt: buildTaskAPrompt(), name: "chain-step-a" }, + outputs: [{ name: "step", type: "text" }], + budget_ms: NODE_BUDGET_MS, + }, + { + id: "task-b", + subagent: { prompt: buildTaskBPromptTemplate(), name: "chain-step-b" }, + inputs: [{ name: "prev", type: "text", from: "task-a.step" }], + outputs: [{ name: "step", type: "text" }], + budget_ms: NODE_BUDGET_MS, + }, + ], + }; +} + +/** Structurally valid but unresolvable: references a subagent entry that does not exist. */ +export function buildBrokenDag(): SwarmDagSpec { + return { + run: { budget_ms: RUN_BUDGET_MS, failure_policy: "escalate", max_parallel: 8 }, + nodes: [{ id: "broken-source", subagent: "no-such-subagent-entry" }], + }; +} + +export const SWARM_ENTRY_IDS = { + reviewSweep: "swarm-dag-eval-review-sweep", + reviewSweepFail: "swarm-dag-eval-review-fail", + builder: "swarm-dag-eval-builder", + residentWatcher: "swarm-dag-eval-resident-watcher", + broken: "swarm-dag-eval-broken", +} as const; + +export function buildReferenceSwarms(width: number): ReferenceSwarm[] { + return [ + { + id: SWARM_ENTRY_IDS.reviewSweep, + kind: "review-sweep", + title: "review-sweep", + description: + "Reference swarm: pull-request review sweep with typed fan-in and escalation (capability eval).", + dag: buildReviewSweepDag(), + declaredBudgetMs: RUN_BUDGET_MS, + declaredFanIn: REVIEW_FILES.length, + }, + { + id: SWARM_ENTRY_IDS.builder, + kind: "builder", + title: "builder", + description: `Reference swarm: ${width}-wide builder run with per-node budgets (capability eval).`, + dag: buildBuilderDag(width), + declaredBudgetMs: RUN_BUDGET_MS, + declaredFanIn: width, + width, + }, + { + id: SWARM_ENTRY_IDS.residentWatcher, + kind: "resident-watcher", + title: "resident-watcher", + description: "Reference swarm: resident watcher that starts a bounded task DAG (capability eval).", + dag: buildResidentWatcherDag(), + declaredBudgetMs: RUN_BUDGET_MS, + declaredFanIn: 1, + }, + { + id: SWARM_ENTRY_IDS.reviewSweepFail, + kind: "review-sweep-fail", + title: "review-sweep-fail", + description: + "Escalation probe: review sweep with one planted failing reviewer; the declared escalate policy must pause the run.", + dag: buildReviewSweepFailDag(), + declaredBudgetMs: RUN_BUDGET_MS, + declaredFanIn: REVIEW_FILES.length, + }, + { + id: SWARM_ENTRY_IDS.broken, + kind: "dry-run-reject", + title: "broken", + description: "Dry-run probe: structurally valid spec that references an unknown subagent.", + dag: buildBrokenDag(), + declaredBudgetMs: RUN_BUDGET_MS, + declaredFanIn: 0, + }, + ]; +} + +export function findReferenceSwarm(swarms: ReferenceSwarm[], kind: ReferenceSwarmKind): ReferenceSwarm { + const swarm = swarms.find((entry) => entry.kind === kind); + if (!swarm) throw new Error(`unknown reference swarm kind ${kind}`); + return swarm; +} + +// --------------------------------------------------------------------------- +// Harness-state seeding: write the harness_state.json the kernel will load +// (refinement.ts getLocalHarnessStateDir + agent-session.ts RLM_HARNESS_STATE_DIR). +// --------------------------------------------------------------------------- + +export interface HarnessEntryJson { + id: string; + kind: "swarm"; + title: string; + content: string; + path: string; + scope: "local"; + reference: Record; + arguments: { dag: SwarmDagSpec }; + metadata: Record; + source: string; + created_at: string; + updated_at: string; + version: number; +} + +/** Full harness_state.json file body seeding the given swarm entries. */ +export function buildHarnessStateFile(specs: ReferenceSwarm[], now = new Date()): string { + const entries: Record = {}; + for (const spec of specs) { + entries[spec.id] = { + id: spec.id, + kind: "swarm", + title: spec.title, + content: spec.description, + path: "swarm-dag-eval", + scope: "local", + reference: {}, + arguments: { dag: spec.dag }, + metadata: { evalKind: spec.kind, source: "swarm-dag-eval" }, + source: "agent", + created_at: now.toISOString(), + updated_at: now.toISOString(), + version: 1, + }; + } + return `${JSON.stringify({ schema: 1, entries: { prompt: {}, memory: {}, skill: {}, subagent: {}, swarm: entries }, refinements: [] }, null, 2)}\n`; +} + +/** Seed the local harness dir for this session so rlm.swarm.run sees the specs. */ +export function seedHarnessState(sessionManager: SessionManager, specs: ReferenceSwarm[]): string { + const artifactDir = getSessionArtifactPath(sessionManager.getSessionDir(), sessionManager.getSessionId()); + const harnessDir = join(artifactDir, "harness"); + mkdirSync(harnessDir, { recursive: true }); + const statePath = join(harnessDir, "harness_state.json"); + writeFileSync(statePath, buildHarnessStateFile(specs)); + return statePath; +} + +// --------------------------------------------------------------------------- +// Parent prompts. +// --------------------------------------------------------------------------- + +function pollCode(breakStates: string): string { + return [ + "import asyncio, json", + "started = await rlm.swarm.run('')", + 'run_id = started["run_id"]', + "while True:", + "\tstatus = await rlm.swarm.status(run_id)", + `\tif status["state"] in (${breakStates}):`, + "\t\tbreak", + "\tawait asyncio.sleep(5)", + ].join("\n"); +} + +function fill(prompt: string, swarm: ReferenceSwarm, ledgerPath: string, poll: string): string { + return prompt + .replace("", poll) + .replaceAll("", swarm.id) + .replaceAll("", ledgerPath); +} + +/** + * Parent prompt for a swarm trial. Contains NO spawn/collect instructions: the + * verdict "no task-specific orchestration code in the parent" is checked against + * this builder (see test/swarm-dag-eval.test.ts). + */ +export function buildSwarmParentPrompt(swarm: ReferenceSwarm, ledgerPath: string): string { + const head = `Capability eval: swarm DAG orchestration. The local harness state for this session seeds exactly one swarm specification: "${swarm.id}". Run it with the executor and report the outcome. Do not spawn subagents yourself; the swarm executor owns the children.`; + switch (swarm.kind) { + case "review-sweep": + return fill( + [ + head, + "", + "Step 1 — start the run and poll it to a terminal state in one ipython cell:", + "", + "", + "", + 'Step 2 — in the same or a new ipython cell, save the final status:', + "", + '\tjson.dump(status, open(r"", "w"))', + "", + 'Step 3 — the node with id "report" in status["nodes"] has an answer_preview containing a JSON object like {"issues": [...]}. Output exactly one line and nothing else:', + "", + 'ANSWER: ISSUES: ; STATE: ', + ].join("\n"), + swarm, + ledgerPath, + pollCode('"done", "failed", "stopped", "paused"'), + ); + case "builder": + return fill( + [ + head, + "", + "Step 1 — start the run and poll it to a terminal state in one ipython cell:", + "", + "", + "", + 'Step 2 — in the same or a new ipython cell, save the final status:', + "", + '\tjson.dump(status, open(r"", "w"))', + "", + 'Step 3 — the node with id "collector" in status["nodes"] has an answer_preview starting with COLLECTED and listing every swb-marker. Output exactly one line and nothing else:', + "", + 'ANSWER: MARKERS: ; STATE: ', + ].join("\n"), + swarm, + ledgerPath, + pollCode('"done", "failed", "stopped", "paused"'), + ); + case "resident-watcher": + return fill( + [ + head, + "The watcher node is resident: the run reaches done while the watcher child stays alive; you must then stop the run to tear it down.", + "", + "Step 1 — start the run and poll until the declarative work is done (state done) in one ipython cell:", + "", + "", + "", + "Step 2 — stop the run, save the final status, and report. In the same or a new ipython cell:", + "", + "\tstopped = await rlm.swarm.stop(run_id)", + "\tstatus = await rlm.swarm.status(run_id)", + '\tjson.dump(status, open(r"", "w"))', + "\tstopped", + "", + 'Step 3 — the nodes "task-a" and "task-b" in status["nodes"] have answer_previews listing the step markers, and stopped["cancelled"] lists the torn-down resident. Output exactly one line and nothing else:', + "", + 'ANSWER: MARKERS: ; STOPPED: ; STATE: ', + ].join("\n"), + swarm, + ledgerPath, + pollCode('"done", "failed", "paused"'), + ); + case "review-sweep-fail": + return fill( + [ + head, + "One reviewer node in this swarm is planted to fail (its subagent model reference cannot be resolved), so the declared escalate policy must pause the run.", + "", + "Step 1 — start the run and poll it to a terminal state in one ipython cell:", + "", + "", + "", + 'Step 2 — save the paused status, then stop the run to cancel the in-flight children. In the same or a new ipython cell:', + "", + '\tjson.dump(status, open(r"", "w"))', + "\tawait rlm.swarm.stop(run_id)", + "", + 'Step 3 — from the saved status: STATE is status["state"], FAILED-NODE is the id of the node whose status is "error", and REPORT-STATUS is the status of the node "report". Output exactly one line and nothing else:', + "", + "ANSWER: STATE: ; FAILED-NODE: ; REPORT-STATUS: ", + ].join("\n"), + swarm, + ledgerPath, + pollCode('"done", "failed", "stopped", "paused"'), + ); + case "dry-run-reject": + return fill( + [ + head, + "This specification is intentionally INVALID: it references a harness subagent that does not exist. The run call must raise and start no node.", + "", + "Step 1 — in one ipython cell:", + "", + "\timport json", + '\terror_message = "no error"', + "\ttry:", + "\t\tawait rlm.swarm.run('')", + "\texcept Exception as exc:", + "\t\terror_message = str(exc)", + "\tsubs = await rlm.list_subagents()", + "\tprint(error_message)", + "\tsubs", + "", + "Step 2 — output exactly one line and nothing else:", + "", + "ANSWER: REJECTED: ; CHILDREN: ; MESSAGE: ", + ].join("\n"), + swarm, + ledgerPath, + "", + ); + } +} + +function budgetLine(swarm: ReferenceSwarm): string { + return `Declared budget: complete the whole run within ${Math.round(swarm.declaredBudgetMs / 60000)} minutes; each child within ${Math.round(NODE_BUDGET_MS / 60000)} minutes.`; +} + +/** Baseline prompt: the identical task done with manual orchestration. */ +export function buildBaselinePrompt(swarm: ReferenceSwarm, ledgerPath: string): string { + switch (swarm.kind) { + case "review-sweep": + return [ + "Capability eval: manual multi-agent orchestration (baseline). Do the identical pull-request review sweep by orchestrating the children yourself with rlm.spawn and rlm.collect. Do NOT use the swarm executor.", + budgetLine(swarm), + "", + "Step 1 — spawn one reviewer child per file in one ipython cell. Each child's prompt is the reviewer template below with the placeholder {files} replaced by the file's name; compose the four prompts by substitution. Do not set a model on the spawn; children inherit yours.", + "", + "import asyncio, json", + 'reviewer_template = """', + `\t${buildReviewerPromptTemplate().replaceAll("\n", "\n\t")}`, + '\t"""', + 'reviewer_prompts = {name: reviewer_template.replace("{files}", name) for name in ["fa", "fb", "fc", "fd"]}', + 'handles = {name: await rlm.spawn(prompt, name=f"reviewer-{name}") for name, prompt in reviewer_prompts.items()}', + 'ids = [handle.rlm_child_id for handle in handles.values()]', + "", + "Step 2 — poll until all four children settle, then save their answers:", + "", + "while True:", + "\tresults = await rlm.collect(ids, timeout_ms=2000)", + "\tif all(r.settled for r in results):", + "\t\tbreak", + "\tawait asyncio.sleep(2)", + "answers = {r.session_name: r.answer_preview for r in results}", + `json.dump(answers, open(r"${ledgerPath}", "w"))`, + "results", + "", + "Step 3 — aggregate the found audit ids from the four answer previews yourself and output exactly one line and nothing else:", + "", + "ANSWER: ISSUES: ", + ].join("\n"); case "builder": { + const width = swarm.width ?? DEFAULT_WIDTH; + const prompts = Array.from({ length: width }, (_, i) => { + const prompt = buildBuilderNodePrompt(i + 1); + return `\t"${i + 1}": """\n${prompt.replaceAll("\n", "\n\t")}""",`; + }); + return [ + "Capability eval: manual multi-agent orchestration (baseline). Do the identical wide build by orchestrating the children yourself with rlm.spawn and rlm.collect. Do NOT use the swarm executor.", + budgetLine(swarm), + "", + `Step 1 — spawn ${width} builder children in one ipython cell, using the exact child prompts below. Do not set a model on the spawn; children inherit yours.`, + "", + "import asyncio, json", + "builder_prompts = {", + ...prompts, + "}", + 'handles = {i: await rlm.spawn(prompt, name=f"builder-{i}") for i, prompt in builder_prompts.items()}', + "ids = [handle.rlm_child_id for handle in handles.values()]", + "", + "Step 2 — poll until every child settles, then save their answers:", + "", + "while True:", + "\tresults = await rlm.collect(ids, timeout_ms=2000)", + "\tif all(r.settled for r in results):", + "\t\tbreak", + "\tawait asyncio.sleep(2)", + "answers = {r.session_name: r.answer_preview for r in results}", + `json.dump(answers, open(r"${ledgerPath}", "w"))`, + "results", + "", + "Step 3 — merge the builder markers from the answer previews yourself and output exactly one line and nothing else:", + "", + "ANSWER: MARKERS: ", + ].join("\n"); + } + case "resident-watcher": + return [ + "Capability eval: manual multi-agent orchestration (baseline). Run the identical resident-watcher topology by orchestrating the children yourself with rlm.spawn and rlm.collect. Do NOT use the swarm executor.", + budgetLine(swarm), + "", + "Step 1 — spawn the watcher child and the task-a child in one ipython cell, using the exact child prompts below. Do not set a model on the spawn; children inherit yours.", + "", + "import asyncio, json", + 'watcher = await rlm.spawn("""', + `\t${buildWatcherPrompt().replaceAll("\n", "\n\t")}`, + '\t""", name="watcher")', + 'task_a = await rlm.spawn("""', + `\t${buildTaskAPrompt().replaceAll("\n", "\n\t")}`, + '\t""", name="task-a")', + "", + "Step 2 — poll until task-a settles and read its answer_preview:", + "", + "while True:", + "\tresults = await rlm.collect([task_a.rlm_child_id], timeout_ms=2000)", + "\tif all(r.settled for r in results):", + "\t\tbreak", + "\tawait asyncio.sleep(2)", + "task_a_answer = results[0].answer_preview", + "", + "Step 3 — spawn task-b with the task-b prompt below, replacing the line that reads `The previous step reported: {prev}` with task_a_answer:", + "", + 'task_b = await rlm.spawn("""', + `\t${buildTaskBPromptTemplate().replaceAll("\n", "\n\t")}`, + '\t""", name="task-b")', + "", + "Step 4 — poll until task-b settles, then save the answers and tear the watcher down:", + "", + "while True:", + "\tresults = await rlm.collect([task_a.rlm_child_id, task_b.rlm_child_id], timeout_ms=2000)", + "\tif all(r.settled for r in results):", + "\t\tbreak", + "\tawait asyncio.sleep(2)", + 'answers = {r.session_name: r.answer_preview for r in results}', + `json.dump(answers, open(r"${ledgerPath}", "w"))`, + "await rlm.delete_subagent(watcher.rlm_child_id)", + "", + "Step 5 — output exactly one line and nothing else:", + "", + "ANSWER: MARKERS: ; STOPPED: watcher", + ].join("\n"); + default: + throw new Error(`no baseline exists for reference swarm kind ${swarm.kind}`); + } +} + +// --------------------------------------------------------------------------- +// Answer parsing and task-success checking. +// --------------------------------------------------------------------------- + +export interface ParsedAnswer { + issues: string[]; + markers: string[]; + state: string | null; + stopped: string[]; + failedNode: string | null; + reportStatus: string | null; + rejected: boolean | null; + children: number | null; + message: string | null; +} + +/** Parse the single ANSWER line from the parent's final assistant text. */ +export function parseAnswerLine(text: string | undefined): ParsedAnswer | null { + const match = /ANSWER:\s*(.+?)(?:\r?\n|$)/i.exec(text ?? ""); + if (!match) return null; + const parsed: ParsedAnswer = { + issues: [], + markers: [], + state: null, + stopped: [], + failedNode: null, + reportStatus: null, + rejected: null, + children: null, + message: null, + }; + const list = (value: string): string[] => + value + .split(",") + .map((entry) => entry.trim()) + .filter((entry) => entry.length > 0); + for (const part of match[1].split(";")) { + const separator = part.indexOf(":"); + if (separator < 0) continue; + const key = part.slice(0, separator).trim().toUpperCase(); + const value = part.slice(separator + 1).trim(); + if (!key || !value) continue; + switch (key) { + case "ISSUES": + parsed.issues = list(value); + break; + case "MARKERS": + parsed.markers = list(value); + break; + case "STATE": + parsed.state = value; + break; + case "STOPPED": + parsed.stopped = list(value); + break; + case "FAILED-NODE": + parsed.failedNode = value; + break; + case "REPORT-STATUS": + parsed.reportStatus = value; + break; + case "REJECTED": + parsed.rejected = value.toLowerCase() === "yes"; + break; + case "CHILDREN": + parsed.children = Number(value); + break; + case "MESSAGE": + parsed.message = value; + break; + } + } + return parsed; +} + +// --------------------------------------------------------------------------- +// Ledger shapes (the executor's status() payload) and the replay checker. +// --------------------------------------------------------------------------- + +export interface SwarmLedgerInstance { + index: number; + status: string; + attempt: number; + child?: string | null; + duration_ms?: number | null; + error?: string | null; +} + +export interface SwarmLedgerNode { + id: string; + status: string; + lifecycle: string; + attempts: number; + instances: SwarmLedgerInstance[]; + answer_preview?: string; + error?: string; +} + +export interface SwarmLedgerEvent { + seq: number; + kind: string; + stage?: string; + node?: string; + instance?: number; + detail?: string; + duration_ms?: number | null; + status?: string; + error?: string; + milestone?: string; +} + +export interface SwarmLedgerUsage { + spawns: number; + settled: number; + tool_uses: number; + max_parallel: number; + running: number; +} + +export interface SwarmStatusLedger { + run_id: string; + spec_id: string; + name: string | null; + state: string; + nodes: SwarmLedgerNode[]; + events: SwarmLedgerEvent[]; + elapsed_ms: number; + usage: SwarmLedgerUsage; +} + +export const KNOWN_SWARM_EVENT_KINDS = [ + "run_started", + "node_ready", + "spawned", + "spawn_backoff", + "spawn_deferred", + "settled", + "answer_captured", + "retry", + "node_error", + "node_cancelled", + "cancelled", + "cancel_failed", + "milestone", + "run_stopped", + "resumed", + "executor_error", +] as const; + +const KNOWN_STAGES = ["recorded", "arrived", "shown", "delivered"]; +const LEDGER_STATES = ["running", "stopping", "paused", "done", "failed", "stopped"]; +const INSTANCE_INSTANCE_PATTERN = /^[a-z0-9][a-z0-9-]{0,63}$/; + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function isLedgerShape(value: unknown): value is SwarmStatusLedger { + if (!isRecord(value)) return false; + if (typeof value.run_id !== "string" || !value.run_id) return false; + if (typeof value.spec_id !== "string" || !value.spec_id) return false; + if (typeof value.state !== "string" || !LEDGER_STATES.includes(value.state)) return false; + if (!Array.isArray(value.nodes) || value.nodes.length === 0) return false; + if (!Array.isArray(value.events)) return false; + if (typeof value.elapsed_ms !== "number" || value.elapsed_ms < 0) return false; + if (!isRecord(value.usage)) return false; + return true; +} + +export interface LedgerCheckResult { + ok: boolean; + problems: string[]; +} + +/** + * Deterministic replay check over one saved status ledger: stable event + * identities (contiguous seq, closed kind/stage vocabularies, valid node refs) + * and complete resource accounting (every spawned instance settled with a + * duration, cancelled, or still in flight on a paused run; usage counts match + * the event stream). + */ +export function checkReplayLedger(ledger: unknown): LedgerCheckResult { + const problems: string[] = []; + if (!isLedgerShape(ledger)) { + return { ok: false, problems: ["ledger does not match the swarm status shape"] }; + } + const nodes = ledger.nodes; + const nodeIds = new Set(); + for (const node of nodes) { + if (typeof node.id !== "string" || !INSTANCE_INSTANCE_PATTERN.test(node.id)) { + problems.push(`ledger node has an invalid id: ${JSON.stringify(node.id)}`); + } else if (nodeIds.has(node.id)) { + problems.push(`ledger duplicates node id ${node.id}`); + } + nodeIds.add(node.id); + } + + let truncated = false; + let lastSeq = 0; + const spawned = new Map(); + const settled = new Map(); + const cancelled = new Set(); + const milestones: string[] = []; + let runStopped = false; + for (const [index, event] of ledger.events.entries()) { + if (!isRecord(event)) { + problems.push(`events[${index}] is not an object`); + continue; + } + const seq = event.seq; + if (typeof seq !== "number" || !Number.isInteger(seq) || seq <= lastSeq) { + problems.push(`events[${index}] has a non-increasing seq: ${JSON.stringify(seq)}`); + continue; + } + if (seq !== lastSeq + 1) { + problems.push(`events[${index}] has a seq gap: expected ${lastSeq + 1}, got ${seq} (dropped event)`); + } + if (index === 0 && seq !== 1) truncated = true; + lastSeq = seq; + if (!KNOWN_SWARM_EVENT_KINDS.includes(event.kind as (typeof KNOWN_SWARM_EVENT_KINDS)[number])) { + problems.push(`events[${index}] has an unknown kind: ${JSON.stringify(event.kind)}`); + } + if (event.stage !== undefined && !KNOWN_STAGES.includes(event.stage)) { + problems.push(`events[${index}] has an unknown stage: ${JSON.stringify(event.stage)}`); + } + if (event.node !== undefined && !nodeIds.has(event.node)) { + problems.push(`events[${index}] references unknown node ${JSON.stringify(event.node)}`); + } + const key = event.node !== undefined ? `${event.node}#${event.instance ?? -1}` : ""; + switch (event.kind) { + case "spawned": + if (key) spawned.set(key, (spawned.get(key) ?? 0) + 1); + break; + case "settled": + if (key) settled.set(key, (settled.get(key) ?? 0) + 1); + if (event.status === "done" && (typeof event.duration_ms !== "number" || event.duration_ms < 0)) { + problems.push(`events[${index}] settles done without a duration_ms`); + } + if (event.status === "error" && typeof event.error !== "string") { + problems.push(`events[${index}] settles error without an error message`); + } + if (event.status !== undefined && !["done", "error"].includes(event.status)) { + problems.push(`events[${index}] has an invalid settled status: ${JSON.stringify(event.status)}`); + } + break; + case "cancelled": + if (key) cancelled.add(key); + break; + case "milestone": + if (typeof event.milestone === "string") milestones.push(event.milestone); + break; + case "run_stopped": + runStopped = true; + break; + } + } + if (truncated) { + problems.push("event window is truncated (first seq is not 1); count assertions skipped"); + } + + if (!truncated) { + // Complete resource accounting per instance. + for (const node of nodes) { + for (const instance of node.instances ?? []) { + const key = `${node.id}#${instance.index ?? -1}`; + const hasSpawn = (spawned.get(key) ?? 0) > 0; + const settleCount = settled.get(key) ?? 0; + if (instance.status === "done") { + if (settleCount === 0) problems.push(`node ${node.id} instance ${instance.index} is done without a settle event`); + else if (typeof instance.duration_ms !== "number" || instance.duration_ms < 0) { + problems.push(`node ${node.id} instance ${instance.index} is done without a duration_ms`); + } + } else if (instance.status === "error") { + if (settleCount === 0) problems.push(`node ${node.id} instance ${instance.index} errored without a settle event`); + else if (typeof instance.error !== "string" || !instance.error) { + problems.push(`node ${node.id} instance ${instance.index} errored without an error message`); + } + if (hasSpawn && instance.duration_ms == null) { + problems.push(`node ${node.id} instance ${instance.index} errored after spawn without a duration_ms`); + } + } else if (instance.status === "cancelled") { + if (!cancelled.has(key)) problems.push(`node ${node.id} instance ${instance.index} is cancelled without a cancel event`); + } else if (instance.status === "running" || instance.status === "pending") { + if (!["paused", "running", "stopping"].includes(ledger.state)) { + problems.push(`node ${node.id} instance ${instance.index} is ${instance.status} in a ${ledger.state} ledger`); + } + } + } + } + for (const [key, count] of spawned) { + const settledCount = settled.get(key) ?? 0; + if (settledCount === 0 && !cancelled.has(key)) { + const node = nodes.find((entry) => key.startsWith(`${entry.id}#`)); + const stillInFlight = + node !== undefined && + ["paused", "running", "stopping"].includes(ledger.state) && + node.instances.some( + (instance) => `${node.id}#${instance.index}` === key && ["pending", "running"].includes(instance.status), + ); + if (!stillInFlight) problems.push(`spawned instance ${key} never settled or cancelled (${count} spawn event(s))`); + } + } + // Admission failures settle without a spawn; every other settle must follow a spawn. + for (const [key, count] of settled) { + if ((spawned.get(key) ?? 0) === 0 && count > 0) { + const node = nodes.find((entry) => key.startsWith(`${entry.id}#`)); + const admissionFailure = + node !== undefined && + node.instances.some( + (instance) => `${node.id}#${instance.index}` === key && instance.status === "error", + ); + if (!admissionFailure) { + problems.push(`instance ${key} settled without ever being spawned`); + } + } + } + const spawnEvents = [...spawned.values()].reduce((sum, count) => sum + count, 0); + // Count settled EVENTS on spawned keys, not distinct keys: a retried instance + // settles twice on the same key and the executor's settle_count increments per + // settlement (including retries), so the ledger must match event counts. + const collectSettles = [...settled.entries()] + .filter(([key]) => (spawned.get(key) ?? 0) > 0) + .reduce((sum, [, count]) => sum + count, 0); + if (ledger.usage.spawns !== spawnEvents) { + problems.push(`usage.spawns ${ledger.usage.spawns} does not match ${spawnEvents} spawned event(s)`); + } + if (ledger.usage.settled !== collectSettles) { + problems.push(`usage.settled ${ledger.usage.settled} does not match ${collectSettles} collect settlement(s)`); + } + } + + // Run-state milestone identities. + if (!truncated) { + if (ledger.state === "done" && !milestones.includes("finished")) { + problems.push("run state done without a finished milestone"); + } + if (ledger.state === "failed" && !milestones.includes("failed")) { + problems.push("run state failed without a failed milestone"); + } + if ( + ledger.state === "paused" && + !milestones.some((milestone) => milestone === "paused" || milestone === "budget_exceeded") + ) { + problems.push("run state paused without a paused or budget_exceeded milestone"); + } + if (ledger.state === "stopped" && !runStopped) { + problems.push("run state stopped without a run_stopped event"); + } + } + return { ok: problems.length === 0, problems }; +} + +/** Run the replay check over a saved report.json or a single status ledger. */ +export function runReplayChecks(data: unknown): { ok: boolean; ledgers: { id: string; result: LedgerCheckResult }[] } { + const ledgers: { id: string; result: LedgerCheckResult }[] = []; + if (isRecord(data) && Array.isArray(data.trials)) { + for (const trial of data.trials) { + if (!isRecord(trial)) continue; + if (trial.ledger === null || trial.ledger === undefined) continue; + ledgers.push({ + id: `${String(trial.swarm)}/${String(trial.arm)}/trial-${String(trial.trial)}`, + result: checkReplayLedger(trial.ledger), + }); + } + } else { + ledgers.push({ id: "ledger", result: checkReplayLedger(data) }); + } + return { ok: ledgers.length > 0 && ledgers.every((entry) => entry.result.ok), ledgers }; +} + +// --------------------------------------------------------------------------- +// Task-success checking (checkable answers + ledger cross-checks). +// --------------------------------------------------------------------------- + +export interface TaskCheckOptions { + /** Which arm is being checked; defaults to the swarm arm. */ + arm?: "swarm" | "baseline"; + /** The baseline parent's collect dump ({ child name: answer preview }); swarm arms ignore it. */ + baselineLedger?: Record | null; +} + +/** Join the baseline collect dump's answer previews into one searchable text; null when absent. */ +function baselineLedgerText(baselineLedger: Record | null | undefined): string | null { + if (!baselineLedger || typeof baselineLedger !== "object") return null; + return Object.values(baselineLedger) + .filter((value): value is string => typeof value === "string") + .join("\n"); +} + +/** + * Check the parent's ANSWER against the swarm's checkable answer. The swarm arm + * cross-checks the saved status ledger; the baseline arm instead cross-checks the + * parent's own collect dump against the ANSWER ids, so a baseline trial cannot pass + * on a self-reported ANSWER the children never produced. + */ +export function checkTaskSuccess( + swarm: ReferenceSwarm, + answer: ParsedAnswer | null, + ledger: SwarmStatusLedger | null, + options: TaskCheckOptions = {}, +): { ok: boolean; problems: string[] } { + const problems: string[] = []; + if (answer === null) return { ok: false, problems: ["no ANSWER line in the parent's final text"] }; + const baseline = options.arm === "baseline"; + const width = swarm.width ?? DEFAULT_WIDTH; + const nodeStatus = (id: string): SwarmLedgerNode | undefined => ledger?.nodes.find((node) => node.id === id); + const ledgerText = baseline ? baselineLedgerText(options.baselineLedger) : null; + switch (swarm.kind) { + case "review-sweep": { + for (const issueId of REVIEW_ISSUE_IDS) { + if (!answer.issues.includes(issueId)) problems.push(`planted issue ${issueId} missing from the ANSWER line`); + } + if (baseline) { + if (ledgerText === null) { + problems.push("baseline collect ledger missing (cannot verify the ANSWER against the children)"); + } else { + for (const issueId of REVIEW_ISSUE_IDS) { + if (!ledgerText.includes(issueId)) { + problems.push(`planted issue ${issueId} missing from the baseline collect ledger`); + } + } + for (const issueId of answer.issues) { + if (!ledgerText.includes(issueId)) { + problems.push(`ANSWER issue ${issueId} is not present in the baseline collect ledger`); + } + } + } + } else { + if (answer.state !== "done") problems.push(`ANSWER state is ${answer.state ?? "unset"}, expected done`); + if (ledger !== null) { + if (ledger.state !== "done") problems.push(`ledger state is ${ledger.state}, expected done`); + const report = nodeStatus("report"); + if (report?.status !== "done") problems.push("ledger report node is not done"); + for (const issueId of REVIEW_ISSUE_IDS) { + if (!report?.answer_preview?.includes(issueId)) { + problems.push(`planted issue ${issueId} missing from the report node answer preview`); + } + } + } + } + break; + } + case "builder": { + const markers = Array.from({ length: width }, (_, index) => BUILDER_MARKER(index + 1)); + for (const marker of markers) { + if (!answer.markers.includes(marker)) problems.push(`${marker} missing from the ANSWER line`); + } + if (baseline) { + if (ledgerText === null) { + problems.push("baseline collect ledger missing (cannot verify the ANSWER against the children)"); + } else { + for (const marker of markers) { + if (!ledgerText.includes(marker)) { + problems.push(`${marker} missing from the baseline collect ledger`); + } + } + for (const marker of answer.markers) { + if (!ledgerText.includes(marker)) { + problems.push(`ANSWER marker ${marker} is not present in the baseline collect ledger`); + } + } + } + } else { + if (answer.state !== "done") problems.push(`ANSWER state is ${answer.state ?? "unset"}, expected done`); + if (ledger !== null) { + if (ledger.state !== "done") problems.push(`ledger state is ${ledger.state}, expected done`); + const collector = nodeStatus("collector"); + if (collector?.status !== "done") problems.push("ledger collector node is not done"); + for (const marker of markers) { + if (!collector?.answer_preview?.includes(marker)) { + problems.push(`${marker} missing from the collector answer preview`); + } + } + } + } + break; + } + case "resident-watcher": { + for (const marker of [TASK_MARKER_A, TASK_MARKER_B]) { + if (!answer.markers.includes(marker)) problems.push(`${marker} missing from the ANSWER line`); + } + if (!answer.stopped.includes("watcher")) problems.push("ANSWER does not report the resident watcher as stopped"); + if (baseline) { + if (ledgerText === null) { + problems.push("baseline collect ledger missing (cannot verify the ANSWER against the children)"); + } else { + for (const marker of [TASK_MARKER_A, TASK_MARKER_B]) { + if (!ledgerText.includes(marker)) { + problems.push(`${marker} missing from the baseline collect ledger`); + } + } + for (const marker of answer.markers) { + if (!ledgerText.includes(marker)) { + problems.push(`ANSWER marker ${marker} is not present in the baseline collect ledger`); + } + } + } + } else if (ledger !== null) { + if (ledger.state !== "stopped") problems.push(`ledger state is ${ledger.state}, expected stopped`); + for (const id of ["task-a", "task-b"]) { + if (nodeStatus(id)?.status !== "done") problems.push(`ledger ${id} node is not done`); + } + const watcher = nodeStatus("watcher"); + if (watcher?.status !== "cancelled") problems.push("ledger watcher node is not cancelled"); + const watcherInstance = watcher?.instances.find((instance) => instance.status === "cancelled"); + if (!watcherInstance) problems.push("ledger watcher instance is not cancelled"); + } + break; + } + case "review-sweep-fail": { + if (answer.state !== "paused") problems.push(`ANSWER state is ${answer.state ?? "unset"}, expected paused`); + if (answer.failedNode !== "review-broken") { + problems.push(`ANSWER failed node is ${answer.failedNode ?? "unset"}, expected review-broken`); + } + if (answer.reportStatus !== "pending") { + problems.push(`ANSWER report status is ${answer.reportStatus ?? "unset"}, expected pending`); + } + if (ledger !== null) { + if (ledger.state !== "paused") problems.push(`ledger state is ${ledger.state}, expected paused`); + const broken = nodeStatus("review-broken"); + if (broken?.status !== "error") problems.push("ledger review-broken node is not error"); + if (!broken?.error) problems.push("ledger review-broken node has no error message"); + if (nodeStatus("report")?.status !== "pending") problems.push("ledger report node is not pending"); + if (ledger.events.some((event) => event.kind === "spawned" && event.node === "report")) { + problems.push("report node started despite the escalation pause"); + } + if (!ledger.events.some((event) => event.kind === "milestone" && event.milestone === "paused")) { + problems.push("ledger has no paused milestone"); + } + } + break; + } + case "dry-run-reject": { + if (answer.rejected !== true) problems.push("ANSWER does not report the run call as rejected"); + if (answer.children !== 0) problems.push(`ANSWER children count is ${answer.children ?? "unset"}, expected 0`); + if (!answer.message?.includes("no-such-subagent-entry")) { + problems.push("ANSWER message does not mention the unknown subagent reference"); + } + break; + } + } + return { ok: problems.length === 0, problems }; +} + +// --------------------------------------------------------------------------- +// Trial result, verdicts, and report rendering. +// --------------------------------------------------------------------------- + +export interface SwarmDagEvalTrialResult { + swarm: string; + arm: "swarm" | "baseline"; + trial: number; + model: string; + taskSuccess: boolean; + problems: string[]; + state: string | null; + wallMs: number; + contextTokens: number | null; + totalTokens: number | null; + declaredFanIn: number; + queueLatencyMs: null; + teardownLatencyMs: number | null; + declaredBudgetMs: number; + budgetOvershootMs: number; + elapsedMs: number | null; + spawns: number | null; + settled: number | null; + replayOk: boolean | null; + replayProblems: string[]; + answer: ParsedAnswer | null; + ledger: SwarmStatusLedger | null; + verdict: "pass" | "fail"; +} + +export interface EvalVerdicts { + noOrchestrationCode: boolean; + failurePolicyMatched: boolean | null; + dryRunRejected: boolean | null; + budgetOvershootMs: number; + budgetOvershootZero: boolean | null; + contextPairs: { + swarm: string; + swarmContextTokens: number | null; + baselineContextTokens: number | null; + lower: boolean | null; + bothCorrect: boolean; + }[]; +} + +export function computeVerdicts(results: SwarmDagEvalTrialResult[]): EvalVerdicts { + const swarmArms = results.filter((row) => row.arm === "swarm" && row.swarm !== "review-sweep-fail" && row.swarm !== "dry-run-reject"); + const escalation = results.find((row) => row.swarm === "review-sweep-fail" && row.arm === "swarm"); + const dryRun = results.find((row) => row.swarm === "dry-run-reject" && row.arm === "swarm"); + const budgetOvershootMs = swarmArms.reduce((sum, row) => sum + row.budgetOvershootMs, 0); + const contextPairs: EvalVerdicts["contextPairs"] = []; + for (const swarm of ["review-sweep", "builder", "resident-watcher"]) { + const swarmRows = swarmArms.filter((row) => row.swarm === swarm); + const baselineRows = results.filter((row) => row.arm === "baseline" && row.swarm === swarm); + if (swarmRows.length === 0 && baselineRows.length === 0) continue; + const swarmContext = averageOrNull(swarmRows.map((row) => row.contextTokens)); + const baselineContext = averageOrNull(baselineRows.map((row) => row.contextTokens)); + contextPairs.push({ + swarm, + swarmContextTokens: swarmContext, + baselineContextTokens: baselineContext, + lower: + swarmContext === null || baselineContext === null ? null : swarmContext < baselineContext, + bothCorrect: + swarmRows.every((row) => row.taskSuccess) && baselineRows.every((row) => row.taskSuccess), + }); + } + return { + noOrchestrationCode: true, // Asserted statically: buildSwarmParentPrompt emits no spawn/collect calls (prompt invariant, unit-tested). + failurePolicyMatched: escalation ? escalation.verdict === "pass" : null, + dryRunRejected: dryRun ? dryRun.verdict === "pass" : null, + budgetOvershootMs, + budgetOvershootZero: swarmArms.length === 0 ? null : budgetOvershootMs === 0, + contextPairs, + }; +} + +function averageOrNull(values: (number | null)[]): number | null { + const present = values.filter((value): value is number => value !== null); + if (present.length === 0) return null; + return Math.round(present.reduce((sum, value) => sum + value, 0) / present.length); +} + +export function renderMarkdownReport(results: SwarmDagEvalTrialResult[], config: EvalConfig): string { + const header = [ + "# Swarm DAG capability eval report", + "", + `- model: ${config.model}`, + `- swarms: ${config.swarms.join(", ")} | width: ${config.width} | trials per pair: ${config.trials}`, + `- declared budgets: run ${RUN_BUDGET_MS} ms, per task node ${NODE_BUDGET_MS} ms (same for each swarm/baseline pair)`, + "- queue latency: omitted (the executor event ledger carries no timestamps)", + "- teardown latency: resident trial, finished-notice to final answer", + "- budget overshoot is measured per arm and is not directly comparable*: swarm arms use the run ledger's elapsed_ms against the declared run budget; baseline arms use full wall clock (an upper bound) against the same budget", + "", + "| swarm | arm | trial | task | state | wall s | ctx tokens | total tokens | fan-in | teardown ms | over budget ms* | replay | verdict |", + "| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |", + ]; + const rows = results.map((row) => + [ + row.swarm, + row.arm, + row.trial, + row.taskSuccess ? "ok" : "failed", + row.state ?? "n/a", + (row.wallMs / 1000).toFixed(1), + row.contextTokens ?? "n/a", + row.totalTokens ?? "n/a", + row.declaredFanIn, + row.teardownLatencyMs ?? "n/a", + row.budgetOvershootMs, + row.replayOk === null ? "n/a" : row.replayOk ? "ok" : "failed", + row.verdict, + ].join(" | "), + ); + const verdicts = computeVerdicts(results); + const pairs = verdicts.contextPairs.map((pair) => + `| ${[ + pair.swarm, + pair.swarmContextTokens ?? "n/a", + pair.baselineContextTokens ?? "n/a", + pair.lower === null ? "n/a" : pair.lower ? "yes" : "no", + pair.bothCorrect ? "yes" : "no", + ].join(" | ")} |`, + ); + const summary = [ + "", + "## Swarm vs hand-written baseline (parent context tokens)", + "", + "| swarm | swarm avg | baseline avg | lower | both task-correct |", + "| --- | --- | --- | --- | --- |", + ...pairs, + "", + "## Verdict rules (Notion spec, Proposed evaluation)", + "", + `- no task-specific orchestration code in swarm prompts (asserted statically, prompt invariant): ${ + verdicts.noOrchestrationCode ? "PASS" : "FAIL" + }`, + `- declared failure policy matches observed behavior (escalation): ${renderVerdict(verdicts.failurePolicyMatched)}`, + `- no node starts after a failed dry run: ${renderVerdict(verdicts.dryRunRejected)}`, + `- total budget overshoot (swarm arms only): ${verdicts.budgetOvershootMs} ms ${renderVerdict(verdicts.budgetOvershootZero)}`, + "- parent context lower than baseline: reported per pair above (informational, not asserted)", + "", + "## Problems", + "", + ...(results.flatMap((row) => [...row.problems, ...row.replayProblems].map((problem) => `- ${row.swarm}/${row.arm}/${row.trial}: ${problem}`)).length > 0 + ? results.flatMap((row) => + [...row.problems, ...row.replayProblems].map((problem) => `- ${row.swarm}/${row.arm}/trial ${row.trial}: ${problem}`), + ) + : ["- none"]), + "", + ]; + return [...header, ...rows.map((row) => `| ${row} |`), ...summary].join("\n"); +} + +function renderVerdict(value: boolean | null): string { + if (value === null) return "(not run)"; + return value ? "(PASS)" : "(FAIL)"; +} + +// --------------------------------------------------------------------------- +// Driver. +// --------------------------------------------------------------------------- + +export type SwarmSelection = "review-sweep" | "builder" | "resident-watcher"; + +export interface EvalConfig { + model: string; + swarms: SwarmSelection[]; + width: number; + trials: number; + timeoutMinutes: number; + outDir: string; +} + +export const DEFAULT_EVAL_CONFIG: Pick = { + swarms: ["review-sweep", "builder", "resident-watcher"], + width: DEFAULT_WIDTH, + trials: 1, + timeoutMinutes: 20, +}; + +export function parseEvalArgs(argv: string[], defaults = DEFAULT_EVAL_CONFIG): EvalConfig | { error: string } { + const args: EvalConfig = { + model: DEFAULT_MODEL, + swarms: [...defaults.swarms], + width: defaults.width, + trials: defaults.trials, + timeoutMinutes: defaults.timeoutMinutes, + outDir: "", + }; + const rest = [...argv]; + while (rest.length > 0) { + const arg = rest.shift() as string; + const value = (flag: string): string => { + const next = rest.shift(); + if (next === undefined) throw new Error(`Missing value for ${flag}`); + return next; + }; + switch (arg) { + case "--model": + args.model = value(arg); + break; + case "--swarms": { + const known: SwarmSelection[] = ["review-sweep", "builder", "resident-watcher"]; + const names = value(arg) + .split(",") + .map((raw) => raw.trim()) + .filter((raw) => raw.length > 0); + // A typo must fail before any token is spent, not silently run the defaults. + const unknown = names.filter((raw) => !known.includes(raw as SwarmSelection)); + if (unknown.length > 0) { + return { error: `Unknown swarm in --swarms: ${unknown.join(", ")} (known: ${known.join(", ")})` }; + } + const selected = names as SwarmSelection[]; + if (selected.length > 0) args.swarms = selected; + break; + } + case "--width": + args.width = Math.min(MAX_WIDTH, Math.max(2, Number(value(arg)))); + break; + case "--trials": + args.trials = Math.max(1, Number(value(arg))); + break; + case "--timeout-minutes": + args.timeoutMinutes = Math.max(1, Number(value(arg))); + break; + case "--out": + args.outDir = value(arg); + break; + case "--help": + case "-h": + return { error: "help" }; + default: + return { error: `Unknown argument: ${arg}` }; + } + } + if (!args.outDir) args.outDir = `swarm-dag-eval-reports/${new Date().toISOString().replace(/[:.]/g, "-")}`; + return args; +} + +interface SessionBundle { + session: Awaited>["session"]; + sessionManager: SessionManager; + tempRoot: string; +} + +async function createEvalSession(config: EvalConfig, label: string): Promise { + const realAgentDir = getAgentDir(); + const tempRoot = join( + tmpdir(), + `swarm-dag-eval-${Date.now()}-${label}-${Math.random().toString(36).slice(2)}`, + ); + mkdirSync(tempRoot, { recursive: true }); + const authStorage = AuthStorage.create(join(realAgentDir, "auth.json")); + const modelRegistry = ModelRegistry.create(authStorage, join(realAgentDir, "models.json")); + const settingsManager = SettingsManager.create(tempRoot, tempRoot); + const sessionManager = SessionManager.create(tempRoot, join(tempRoot, "sessions")); + const [provider, ...modelIdParts] = config.model.split("/"); + const model = modelRegistry.find(provider, modelIdParts.join("/")); + if (!model) throw new Error(`Model ${config.model} not found in the registry`); + const { session } = await createAgentSession({ + cwd: tempRoot, + authStorage, + modelRegistry, + settingsManager, + sessionManager, + model, + includeGoals: false, + }); + return { session, sessionManager, tempRoot }; +} + +function lastAssistantContextTokens(session: SessionBundle["session"]): number | null { + const messages = session.agent.state.messages as unknown as Array>; + for (let index = messages.length - 1; index >= 0; index--) { + const message = messages[index]; + if (message.role !== "assistant") continue; + if (message.stopReason === "aborted" || message.stopReason === "error") continue; + const usage = message.usage as Parameters[0] | undefined; + if (usage) return calculateContextTokens(usage); + } + return null; +} + +function messageTimestampMs(message: Record): number | null { + const timestamp = message.timestamp; + if (typeof timestamp === "number") return timestamp; + if (typeof timestamp === "string") { + const parsed = Date.parse(timestamp); + return Number.isNaN(parsed) ? null : parsed; + } + if (timestamp instanceof Date) return timestamp.getTime(); + return null; +} + +/** Resident teardown latency: the finished notice to the parent's final answer. */ +function measureTeardownLatency(session: SessionBundle["session"], runId: string | null): number | null { + if (!runId) return null; + const messages = session.agent.state.messages as unknown as Array>; + let finishedAt: number | null = null; + for (const message of messages) { + if (message.customType !== SWARM_PROGRESS_NOTICE_CUSTOM_TYPE) continue; + const content = typeof message.content === "string" ? message.content : ""; + if (!content.includes("[swarm-progress") || !content.includes("finished")) continue; + if (runId && !content.includes(runId)) continue; + finishedAt = messageTimestampMs(message); + } + let answerAt: number | null = null; + for (let index = messages.length - 1; index >= 0; index--) { + if (messages[index].role !== "assistant") continue; + answerAt = messageTimestampMs(messages[index]); + break; + } + return finishedAt !== null && answerAt !== null ? Math.max(0, answerAt - finishedAt) : null; +} + +function readLedger(ledgerPath: string): SwarmStatusLedger | null { + if (!existsSync(ledgerPath)) return null; + try { + return JSON.parse(readFileSync(ledgerPath, "utf8")) as SwarmStatusLedger; + } catch { + return null; + } +} + +function readBaselineLedger(ledgerPath: string): Record | null { + if (!existsSync(ledgerPath)) return null; + try { + return JSON.parse(readFileSync(ledgerPath, "utf8")) as Record; + } catch { + return null; + } +} + +async function promptWithTimeout(session: SessionBundle["session"], prompt: string, timeoutMs: number): Promise { + let timer: ReturnType | undefined; + try { + await Promise.race([ + session.prompt(prompt), + new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error(`trial prompt timed out after ${timeoutMs} ms`)), timeoutMs); + }), + ]); + } finally { + if (timer !== undefined) clearTimeout(timer); + } +} + +async function runSwarmTrial( + config: EvalConfig, + swarm: ReferenceSwarm, + trial: number, +): Promise { + const startedAt = Date.now(); + const bundle = await createEvalSession(config, swarm.kind); + const ledgerPath = join(bundle.tempRoot, "ledger.json"); + seedHarnessState(bundle.sessionManager, [swarm]); + const prompt = buildSwarmParentPrompt(swarm, ledgerPath); + try { + const problems: string[] = []; + try { + await promptWithTimeout(bundle.session, prompt, config.timeoutMinutes * 60_000); + } catch (error) { + problems.push(`parent turn failed: ${error instanceof Error ? error.message : String(error)}`); + } + const answer = parseAnswerLine(bundle.session.getLastAssistantText()); + const ledger = readLedger(ledgerPath); + const check = checkTaskSuccess(swarm, answer, ledger); + const replay = ledger !== null ? checkReplayLedger(ledger) : null; + problems.push(...check.problems); + if (ledger === null && swarm.kind !== "dry-run-reject") { + problems.push("status ledger was not written"); + } + const contextTokens = lastAssistantContextTokens(bundle.session); + const stats = bundle.session.getSessionStats(); + const elapsedMs = ledger?.elapsed_ms ?? null; + const teardownLatencyMs = + swarm.kind === "resident-watcher" ? measureTeardownLatency(bundle.session, ledger?.run_id ?? null) : null; + const budgetOvershootMs = elapsedMs !== null ? Math.max(0, elapsedMs - swarm.declaredBudgetMs) : 0; + const taskSuccess = check.ok; + return { + swarm: swarm.kind, + arm: "swarm", + trial, + model: config.model, + taskSuccess, + problems, + state: ledger?.state ?? answer?.state ?? null, + wallMs: Date.now() - startedAt, + contextTokens, + totalTokens: stats.tokens.total, + declaredFanIn: swarm.declaredFanIn, + queueLatencyMs: null, + teardownLatencyMs, + declaredBudgetMs: swarm.declaredBudgetMs, + budgetOvershootMs, + elapsedMs, + spawns: ledger?.usage.spawns ?? null, + settled: ledger?.usage.settled ?? null, + replayOk: replay?.ok ?? null, + replayProblems: replay?.problems ?? [], + answer, + ledger, + verdict: taskSuccess && problems.length === 0 && (replay?.ok ?? true) ? "pass" : "fail", + }; + } finally { + await bundle.session.dispose(); + rmSync(bundle.tempRoot, { recursive: true, force: true }); + } +} + +async function runBaselineTrial( + config: EvalConfig, + swarm: ReferenceSwarm, + trial: number, +): Promise { + const startedAt = Date.now(); + const bundle = await createEvalSession(config, `${swarm.kind}-baseline`); + const ledgerPath = join(bundle.tempRoot, "ledger.json"); + const prompt = buildBaselinePrompt(swarm, ledgerPath); + try { + const problems: string[] = []; + try { + await promptWithTimeout(bundle.session, prompt, config.timeoutMinutes * 60_000); + } catch (error) { + problems.push(`parent turn failed: ${error instanceof Error ? error.message : String(error)}`); + } + const answer = parseAnswerLine(bundle.session.getLastAssistantText()); + const baselineLedger = readBaselineLedger(ledgerPath); + const check = checkTaskSuccess(swarm, answer, null, { arm: "baseline", baselineLedger }); + problems.push(...check.problems); + const contextTokens = lastAssistantContextTokens(bundle.session); + const stats = bundle.session.getSessionStats(); + const wallMs = Date.now() - startedAt; + // Wall clock vs declared run budget: an upper-bound overshoot for the arm. + const budgetOvershootMs = Math.max(0, wallMs - swarm.declaredBudgetMs); + return { + swarm: swarm.kind, + arm: "baseline", + trial, + model: config.model, + taskSuccess: check.ok, + problems, + state: answer?.state ?? null, + wallMs, + contextTokens, + totalTokens: stats.tokens.total, + declaredFanIn: swarm.declaredFanIn, + queueLatencyMs: null, + teardownLatencyMs: null, + declaredBudgetMs: swarm.declaredBudgetMs, + budgetOvershootMs, + elapsedMs: null, + spawns: null, + settled: null, + replayOk: null, + replayProblems: [], + answer, + ledger: null, + verdict: check.ok ? "pass" : "fail", + }; + } finally { + await bundle.session.dispose(); + rmSync(bundle.tempRoot, { recursive: true, force: true }); + } +} + +async function main(argv: string[] = process.argv.slice(2)): Promise { + const replayIndex = argv.indexOf("--replay"); + if (replayIndex !== -1) { + const replayPath = argv[replayIndex + 1]; + if (!replayPath) { + console.error("--replay requires a path to a report.json or a single status ledger"); + process.exit(1); + } + let data: unknown; + try { + data = JSON.parse(readFileSync(replayPath, "utf8")); + } catch (error) { + console.error(`cannot read replay input: ${error instanceof Error ? error.message : String(error)}`); + process.exit(1); + } + const replay = runReplayChecks(data); + for (const entry of replay.ledgers) { + console.log(`replay ${entry.id}: ${entry.result.ok ? "ok" : "failed"}`); + for (const problem of entry.result.problems) console.log(` - ${problem}`); + } + console.log(replay.ok ? "all ledgers replay cleanly" : "replay check failed"); + process.exit(replay.ok ? 0 : 1); + } + const config = parseEvalArgs(argv); + if ("error" in config) { + console.error(config.error === "help" ? "See the header of this file for usage." : config.error); + process.exit(config.error === "help" ? 0 : 1); + } + const referenceSwarms = buildReferenceSwarms(config.width); + const results: SwarmDagEvalTrialResult[] = []; + for (const selection of config.swarms) { + const swarm = findReferenceSwarm(referenceSwarms, selection); + for (let trial = 1; trial <= config.trials; trial++) { + console.log(`running ${swarm.kind} swarm trial ${trial}/${config.trials} on ${config.model}`); + try { + results.push(await runSwarmTrial(config, swarm, trial)); + } catch (error) { + console.error(`swarm trial failed: ${error instanceof Error ? error.message : String(error)}`); + } + console.log(`running ${swarm.kind} baseline trial ${trial}/${config.trials} on ${config.model}`); + try { + results.push(await runBaselineTrial(config, swarm, trial)); + } catch (error) { + console.error(`baseline trial failed: ${error instanceof Error ? error.message : String(error)}`); + } + } + } + const escalationSwarm = findReferenceSwarm(referenceSwarms, "review-sweep-fail"); + console.log("running review-sweep escalation probe (one trial)"); + try { + results.push(await runSwarmTrial(config, escalationSwarm, 1)); + } catch (error) { + console.error(`escalation trial failed: ${error instanceof Error ? error.message : String(error)}`); + } + const brokenSwarm = findReferenceSwarm(referenceSwarms, "dry-run-reject"); + console.log("running dry-run rejection probe (one trial)"); + try { + results.push(await runSwarmTrial(config, brokenSwarm, 1)); + } catch (error) { + console.error(`dry-run trial failed: ${error instanceof Error ? error.message : String(error)}`); + } + if (results.length === 0) { + console.error("no trials completed"); + process.exit(1); + } + const markdown = renderMarkdownReport(results, config); + mkdirSync(config.outDir, { recursive: true }); + writeFileSync(join(config.outDir, "report.md"), markdown); + writeFileSync( + join(config.outDir, "report.json"), + JSON.stringify( + { config, generatedAt: new Date().toISOString(), results, verdicts: computeVerdicts(results) }, + null, + 2, + ), + ); + console.log(markdown); + console.log(`reports written to ${config.outDir}`); +} + +if (import.meta.url === `file://${process.argv[1]}`) { + void main().catch((error: unknown) => { + console.error(error); + process.exit(1); + }); +} diff --git a/packages/coding-agent/src/core/agent-messages.ts b/packages/coding-agent/src/core/agent-messages.ts index 62bf08e600..98d2ea30d5 100644 --- a/packages/coding-agent/src/core/agent-messages.ts +++ b/packages/coding-agent/src/core/agent-messages.ts @@ -5,6 +5,7 @@ import type { CustomMessage } from "./messages.js"; import { ASYNC_BASH_COMPLETION_CUSTOM_TYPE, HEARTBEAT_PROMPT_CUSTOM_TYPE, + SWARM_PROGRESS_NOTICE_CUSTOM_TYPE, sanitizeMessageHeaderValue, } from "./messages.js"; import { canonicalSessionPath } from "./session-lease.js"; @@ -438,7 +439,8 @@ export function startsAgentRun(message: AgentMessage): boolean { isAgentSessionMessage(message) || (message.role === "custom" && (message.customType === HEARTBEAT_PROMPT_CUSTOM_TYPE || - message.customType === ASYNC_BASH_COMPLETION_CUSTOM_TYPE)) + message.customType === ASYNC_BASH_COMPLETION_CUSTOM_TYPE || + message.customType === SWARM_PROGRESS_NOTICE_CUSTOM_TYPE)) ); } diff --git a/packages/coding-agent/src/core/agent-session.ts b/packages/coding-agent/src/core/agent-session.ts index 712b0cf32e..634269ecc7 100644 --- a/packages/coding-agent/src/core/agent-session.ts +++ b/packages/coding-agent/src/core/agent-session.ts @@ -194,6 +194,7 @@ import { createRlmChildTerminalNoticeMessage, createSessionSlashCommandMessage, createSessionSlashCommandResultMessage, + createSwarmProgressMessage, HARNESS_DIGEST_CUSTOM_TYPE, type HarnessDigestDetails, HEARTBEAT_PROMPT_CUSTOM_TYPE, @@ -203,6 +204,8 @@ import { type RefinementSource, RLM_CHILD_FAILURE_CUSTOM_TYPE, RLM_CHILD_TERMINAL_NOTICE_CUSTOM_TYPE, + SWARM_PROGRESS_NOTICE_CUSTOM_TYPE, + SWARM_PROGRESS_PREVIEW_LABEL, } from "./messages.js"; import type { ModelRegistry } from "./model-registry.js"; import { findExactModelReferenceMatch } from "./model-resolver.js"; @@ -263,6 +266,7 @@ import { createRlmListSubagentsHostHandler, createRlmProgressNoteHostHandler, createRlmRunHostHandler, + createSwarmProgressHostHandler, findRlmModelMatches, findUniqueRlmShortFormModelMatch, formatRlmModelUnavailableError, @@ -932,6 +936,8 @@ function injectedMessagePreviewLabel(message: CustomMessage): string | undefined return HEARTBEAT_PROMPT_PREVIEW_LABEL; case ASYNC_BASH_COMPLETION_CUSTOM_TYPE: return ASYNC_BASH_COMPLETION_PREVIEW_LABEL; + case SWARM_PROGRESS_NOTICE_CUSTOM_TYPE: + return SWARM_PROGRESS_PREVIEW_LABEL; case GOAL_CONTEXT_CUSTOM_TYPE: return GOAL_CONTEXT_PREVIEW_LABEL; default: @@ -10530,6 +10536,31 @@ export class AgentSession { ), "rlm.progress.note": createRlmProgressNoteHostHandler((message) => this.noteRlmProgress(message)), "rlm.delete_subagent": createRlmDeleteSubagentHostHandler((target) => this.deleteRlmSubagent(target)), + "swarm.progress": createSwarmProgressHostHandler(async (details) => { + const message = createSwarmProgressMessage(details); + const disposeSignal = this._sessionActionCommitDisposeAbortController.signal; + while (true) { + let admissionCommitted = false; + try { + await this._promptInjectedMessage(message.content, message, { + streamingBehavior: "steer", + queueIfBusy: true, + resumeIfIdle: true, + returnAfterAccepted: true, + suppressAutonomousContinuation: true, + admissionCommitted: () => { + admissionCommitted = true; + }, + }); + return; + } catch (error) { + if (admissionCommitted || !(error instanceof SessionInputAdmissionPausedError)) throw error; + while (this._sessionInputAdmissionPauses.size > 0 && !disposeSignal.aborted) { + await this._waitForSessionActivityChange(disposeSignal); + } + } + } + }), "model.info": async () => ({ id: this.model?.id ?? null, provider: this.model?.provider ?? null, diff --git a/packages/coding-agent/src/core/messages.ts b/packages/coding-agent/src/core/messages.ts index 50270e113d..1f806b0283 100644 --- a/packages/coding-agent/src/core/messages.ts +++ b/packages/coding-agent/src/core/messages.ts @@ -49,6 +49,8 @@ export const RLM_CHILD_FAILURE_CUSTOM_TYPE = "rlm_child_failure"; export const RLM_CHILD_TERMINAL_NOTICE_CUSTOM_TYPE = "rlm_child_terminal_notice"; export const ASYNC_BASH_COMPLETION_CUSTOM_TYPE = "async_bash_completion"; export const ASYNC_BASH_COMPLETION_PREVIEW_LABEL = "Background command finished"; +export const SWARM_PROGRESS_NOTICE_CUSTOM_TYPE = "swarm_progress_notice"; +export const SWARM_PROGRESS_PREVIEW_LABEL = "Swarm progress"; /** * Names and other metadata interpolated into a `[ ...]` header line must not @@ -265,6 +267,34 @@ Command: ${JSON.stringify(details.command)}`, }; } +export interface SwarmProgressDetails { + runId: string; + kind: "finished" | "failed" | "paused" | "budget_exceeded"; + node?: string; + detail: string; +} + +interface SwarmProgressMessage extends CustomMessage { + customType: typeof SWARM_PROGRESS_NOTICE_CUSTOM_TYPE; + content: string; +} + +export function createSwarmProgressMessage( + details: SwarmProgressDetails, + timestamp = Date.now(), +): SwarmProgressMessage { + // The wire kind budget_exceeded renders as the friendlier budget-exceeded label. + const kind = details.kind === "budget_exceeded" ? "budget-exceeded" : details.kind; + return { + role: "custom", + customType: SWARM_PROGRESS_NOTICE_CUSTOM_TYPE, + content: `[swarm-progress run:${sanitizeMessageHeaderValue(details.runId)}] ${kind}: ${details.detail}`, + display: true, + details, + timestamp, + }; +} + export function createRlmChildFailureMessage( details: RlmChildFailureDetails, timestamp = Date.now(), diff --git a/packages/coding-agent/src/core/prompts/rlm.ts b/packages/coding-agent/src/core/prompts/rlm.ts index 6a669324cf..1caca60f9f 100644 --- a/packages/coding-agent/src/core/prompts/rlm.ts +++ b/packages/coding-agent/src/core/prompts/rlm.ts @@ -44,7 +44,9 @@ const REPL_CONTROL_PROMPT = [ "", "Python state in the kernel persists across cells: named variables, helper functions, classes, imports, notes, parsed outputs, and helper data structures all remain available in every later turn. Tool calls are themselves Python `await` expressions, so their return values can be bound to variables and composed into program logic just like any other call.", "", - "Continual harness state is available as `rlm.harness` and `rlm.get_harness_state()`. CRUD calls are local to this Prime Agent session by default: `rlm.harness.create_memory(...)`, `rlm.harness.update_memory(...)`, `rlm.harness.delete_memory(...)`, `rlm.harness.create_skill(...)`, `rlm.harness.update_skill(...)`, `rlm.harness.delete_skill(...)`, `rlm.harness.create_subagent(...)`, `rlm.harness.update_subagent(...)`, `rlm.harness.delete_subagent(...)`, `rlm.harness.create_prompt_note(...)`, `rlm.harness.update_prompt_note(...)`, `rlm.harness.delete_prompt_note(...)`, plus `rlm.harness.record_refinement(...)` and `rlm.harness.overview()`. Use `global_=True` only for stable cross-session lessons; Python reserves `global`, so literal `global=True` is invalid syntax.", + "Continual harness state is available as `rlm.harness` and `rlm.get_harness_state()`. CRUD calls are local to this Prime Agent session by default: `rlm.harness.create_memory(...)`, `rlm.harness.update_memory(...)`, `rlm.harness.delete_memory(...)`, `rlm.harness.create_skill(...)`, `rlm.harness.update_skill(...)`, `rlm.harness.delete_skill(...)`, `rlm.harness.create_subagent(...)`, `rlm.harness.update_subagent(...)`, `rlm.harness.delete_subagent(...)`, `rlm.harness.create_swarm(...)`, `rlm.harness.update_swarm(...)`, `rlm.harness.delete_swarm(...)`, `rlm.harness.create_prompt_note(...)`, `rlm.harness.update_prompt_note(...)`, `rlm.harness.delete_prompt_note(...)`, plus `rlm.harness.record_refinement(...)` and `rlm.harness.overview()`. Use `global_=True` only for stable cross-session lessons; Python reserves `global`, so literal `global=True` is invalid syntax.", + "", + "Swarm entries declare a validated DAG of subagent nodes in arguments['dag']: run a stored swarm with await rlm.swarm.run(''), watch it with rlm.swarm.status(run_id), stop it with rlm.swarm.stop(run_id), and resume an escalate-paused run with rlm.swarm.resume(run_id).", "", "Terminology: continual harness names the persisted prompt, memory, skill, and subagent layer; RLM names the runtime, Python REPL kernel, and native call interface exposed to the model.", "", diff --git a/packages/coding-agent/src/core/refinement/refinement.ts b/packages/coding-agent/src/core/refinement/refinement.ts index 28f5badf05..60584d9517 100644 --- a/packages/coding-agent/src/core/refinement/refinement.ts +++ b/packages/coding-agent/src/core/refinement/refinement.ts @@ -32,7 +32,7 @@ const DEFAULT_OVERVIEW_CONTENT_LIMIT = 180; */ const HARNESS_DIGEST_FINGERPRINT_VERSION = 1; -export type RefinementKind = "prompt" | "memory" | "skill" | "subagent"; +export type RefinementKind = "prompt" | "memory" | "skill" | "subagent" | "swarm"; export type RefinementAction = "create" | "update" | "delete"; export type HarnessScope = "local" | "global"; @@ -165,7 +165,7 @@ JSON only with this exact shape: "edits": [ { "action": "create|update|delete", - "kind": "prompt|memory|skill|subagent", + "kind": "prompt|memory|skill|subagent|swarm", "id": "stable id for update/delete, optional for create", "title": "required for create/update except delete", "content": "required for create/update except delete", @@ -257,6 +257,7 @@ function emptyHarnessState(): HarnessState { memory: {}, skill: {}, subagent: {}, + swarm: {}, }, refinements: [], }; @@ -665,6 +666,10 @@ export function formatHarnessStateForPrompt( lines.push( `${kind}: ${entries.length} (invoke a spec by turning it into a concise task prompt and spawning with \`await rlm.spawn('', name='')\`; admission returns a child handle, never the answer)`, ); + } else if (kind === "swarm" && entries.length > 0 && includeIpythonExamples) { + lines.push( + `${kind}: ${entries.length} (run it with \`await rlm.swarm.run('')\`; watch with \`rlm.swarm.status(run_id)\`, stop with \`rlm.swarm.stop(run_id)\`)`, + ); } else { lines.push(`${kind}: ${entries.length}`); } @@ -931,7 +936,7 @@ function validateEdit(edit: RefinementEdit, computedId?: string): string | undef if (!["create", "update", "delete"].includes(edit.action)) { return `unsupported action ${String(edit.action)}`; } - if (!["prompt", "memory", "skill", "subagent"].includes(edit.kind)) { + if (!["prompt", "memory", "skill", "subagent", "swarm"].includes(edit.kind)) { return `unsupported kind ${String(edit.kind)}`; } if (edit.kind === "prompt" && (edit.id === "base_system_prompt" || computedId === "base_system_prompt")) { @@ -967,6 +972,14 @@ function validateEdit(edit: RefinementEdit, computedId?: string): string | undef return `${edit.action} skill requires callable or call_pattern`; } } + if (edit.action !== "delete" && edit.kind === "swarm") { + // Structural check only: the kernel validator (rlm.swarm) enforces the full + // DAG semantics at write time; do not reimplement it here. + const dag = edit.arguments?.dag; + if (typeof dag !== "object" || dag === null || Array.isArray(dag)) { + return "swarm entry requires a dag object in arguments"; + } + } return undefined; } diff --git a/packages/coding-agent/src/core/rlm-runtime.ts b/packages/coding-agent/src/core/rlm-runtime.ts index 4455b00a5a..c43f27bbe8 100644 --- a/packages/coding-agent/src/core/rlm-runtime.ts +++ b/packages/coding-agent/src/core/rlm-runtime.ts @@ -125,6 +125,24 @@ interface AsyncBashConsumedRequest { } type AsyncBashConsumedHandler = (request: AsyncBashConsumedRequest) => void | Promise; + +export type SwarmProgressKind = "finished" | "failed" | "paused" | "budget_exceeded"; + +export interface SwarmProgressRequest { + runId: string; + kind: SwarmProgressKind; + node?: string; + detail: string; +} + +export type SwarmProgressHandler = (request: SwarmProgressRequest) => void | Promise; + +const SWARM_PROGRESS_KINDS: readonly SwarmProgressKind[] = ["finished", "failed", "paused", "budget_exceeded"]; + +function isSwarmProgressKind(value: unknown): value is SwarmProgressKind { + return typeof value === "string" && (SWARM_PROGRESS_KINDS as readonly string[]).includes(value); +} + export type RlmListSubagentsHandler = () => RlmListSubagentsResult | Promise; export type RlmDeleteSubagentHandler = (target: string) => Promise; export type RlmFindModelsHandler = (query: string, limit: number) => RlmFindModelsResult | Promise; @@ -331,6 +349,39 @@ export function createAsyncBashCompletionHostHandler(handler: AsyncBashCompletio }; } +/** Adapt a swarm executor milestone into a validated `swarm.progress` host notification. */ +export function createSwarmProgressHostHandler(handler: SwarmProgressHandler): HostRequestHandler { + return async (payload) => { + const runId = payload.run_id; + if (typeof runId !== "string" || !runId.trim()) { + throw new Error("swarm.progress run_id must be a non-empty string"); + } + const kind = payload.kind; + if (!isSwarmProgressKind(kind)) { + throw new Error( + `swarm.progress kind must be one of ${SWARM_PROGRESS_KINDS.join(", ")}, got ${JSON.stringify(kind)}`, + ); + } + const detail = payload.detail; + if (typeof detail !== "string" || !detail.trim()) { + throw new Error("swarm.progress detail must be a non-empty string"); + } + const node = payload.node; + if (node !== undefined && typeof node !== "string") { + throw new Error("swarm.progress node must be a string when provided"); + } + if (node !== undefined && !node.trim()) { + throw new Error("swarm.progress node must be a non-empty string when provided"); + } + const request: SwarmProgressRequest = { runId: runId.trim(), kind, detail }; + if (typeof node === "string" && node.trim()) { + request.node = node.trim(); + } + await handler(request); + return {}; + }; +} + /** The kernel read a finished command's result, so its completion notice is stale. */ export function createAsyncBashConsumedHostHandler(handler: AsyncBashConsumedHandler): HostRequestHandler { return async (payload) => { diff --git a/packages/coding-agent/test/refinement.test.ts b/packages/coding-agent/test/refinement.test.ts index def8b95c34..2fcfc79ef9 100644 --- a/packages/coding-agent/test/refinement.test.ts +++ b/packages/coding-agent/test/refinement.test.ts @@ -64,7 +64,7 @@ function makeTempDir(): string { return tempDir; } -const kinds = ["prompt", "memory", "skill", "subagent"] as const satisfies readonly RefinementKind[]; +const kinds = ["prompt", "memory", "skill", "subagent", "swarm"] as const satisfies readonly RefinementKind[]; const skillReference = { type: "python", import: "agent_skills.example", @@ -75,6 +75,21 @@ const skillContract = { reference: skillReference, arguments: { input: { type: "string", required: true, description: "Task input" } }, }; +const swarmDag = { + nodes: [ + { + id: "collect", + subagent: "researcher", + outputs: [{ name: "findings", type: "text" }], + }, + { + id: "review", + subagent: { prompt: "Review the findings." }, + depends_on: ["collect"], + inputs: [{ name: "draft", type: "text", from: "collect.findings" }], + }, + ], +}; function proposal(summary: string, edits: RefinementProposal["edits"]): RefinementProposal { return { @@ -131,7 +146,7 @@ function seedEntry(state: HarnessState, kind: RefinementKind, id = `${kind}_entr title: `${kind} title`, content: `${kind} content`, path: `${kind}/path`, - ...(kind === "skill" ? skillContract : {}), + ...(kind === "skill" ? skillContract : kind === "swarm" ? { arguments: { dag: swarmDag } } : {}), metadata: { seeded: true }, }, ]), @@ -205,7 +220,7 @@ describe("harness refinement", () => { it.each(kinds)("applies the create/update/delete lifecycle for %s entries", (kind) => { const state = loadHarnessState(makeTempDir()); const id = `${kind}_entry`; - const skillFields = kind === "skill" ? skillContract : {}; + const skillFields = kind === "skill" ? skillContract : kind === "swarm" ? { arguments: { dag: swarmDag } } : {}; const apply = (edits: RefinementProposal["edits"], refinementId: string) => applyRefinementProposal(state, proposal(`${refinementId} ${kind}`, edits), { id: refinementId }); @@ -272,6 +287,101 @@ describe("harness refinement", () => { expect(state.refinements.at(-1)?.changes).toEqual([`delete ${kind}:${id}`]); }); + it("requires a dag object in arguments for swarm creates and updates", () => { + const state = loadHarnessState(makeTempDir()); + + const missingDag = applyRefinementProposal( + state, + proposal("Create swarm without a dag", [ + { + action: "create", + kind: "swarm", + id: "swarm_entry", + title: "Swarm title", + content: "Swarm content", + }, + ]), + { id: "refine_swarm_missing_dag" }, + ); + + expect(missingDag.appliedEdits[0]).toMatchObject({ + applied: false, + error: "swarm entry requires a dag object in arguments", + }); + expect(state.entries.swarm.swarm_entry).toBeUndefined(); + expect(state.refinements.at(-1)?.changes).toEqual([]); + + const nonObjectDag = applyRefinementProposal( + state, + proposal("Create swarm with a non-object dag", [ + { + action: "create", + kind: "swarm", + id: "swarm_entry", + title: "Swarm title", + content: "Swarm content", + arguments: { dag: ["not", "an", "object"] }, + }, + ]), + { id: "refine_swarm_non_object_dag" }, + ); + + expect(nonObjectDag.appliedEdits[0]).toMatchObject({ + applied: false, + error: "swarm entry requires a dag object in arguments", + }); + + const created = applyRefinementProposal( + state, + proposal("Create swarm with a dag", [ + { + action: "create", + kind: "swarm", + id: "swarm_entry", + title: "Swarm title", + content: "Swarm content", + path: "swarm/created", + arguments: { dag: swarmDag }, + metadata: { kind: "swarm" }, + }, + ]), + { id: "refine_swarm_valid" }, + ); + + expect(created.appliedEdits[0].applied).toBe(true); + expect(state.entries.swarm.swarm_entry.arguments).toEqual({ dag: swarmDag }); + + const updateWithoutDag = applyRefinementProposal( + state, + proposal("Update swarm without a dag", [ + { + action: "update", + kind: "swarm", + id: "swarm_entry", + title: "Swarm title updated", + content: "Swarm content updated", + }, + ]), + { id: "refine_swarm_update_missing_dag" }, + ); + + expect(updateWithoutDag.appliedEdits[0]).toMatchObject({ + applied: false, + error: "swarm entry requires a dag object in arguments", + }); + expect(state.entries.swarm.swarm_entry.title).toBe("Swarm title"); + }); + + it("renders the swarm invoke contract in the harness digest", () => { + const state = loadHarnessState(makeTempDir()); + seedEntry(state, "swarm", "sweep"); + + const digest = formatHarnessStateForPrompt(state); + + expect(digest).toContain("swarm: 1"); + expect(digest).toContain("await rlm.swarm.run('')"); + }); + it("creates ids from titles and uses default path and metadata when omitted", () => { const state = loadHarnessState(makeTempDir()); @@ -297,7 +407,8 @@ describe("harness refinement", () => { error: string; seed?: RefinementKind; }; - const skillFieldsFor = (kind: RefinementKind) => (kind === "skill" ? skillContract : {}); + const skillFieldsFor = (kind: RefinementKind) => + kind === "skill" ? skillContract : kind === "swarm" ? { arguments: { dag: swarmDag } } : {}; it.each([ ...kinds.map( (kind): InvalidCase => ({ @@ -477,7 +588,7 @@ describe("harness refinement", () => { const state = loadHarnessState(dir); - expect(state.entries).toEqual({ prompt: {}, memory: {}, skill: {}, subagent: {} }); + expect(state.entries).toEqual({ prompt: {}, memory: {}, skill: {}, subagent: {}, swarm: {} }); expect(state.refinements).toEqual([]); applyRefinementProposal( state, diff --git a/packages/coding-agent/test/suite/regressions/6009-opencode-maintenance-session.test.ts b/packages/coding-agent/test/suite/regressions/6009-opencode-maintenance-session.test.ts index b7ec64f6dd..0cb790f60c 100644 --- a/packages/coding-agent/test/suite/regressions/6009-opencode-maintenance-session.test.ts +++ b/packages/coding-agent/test/suite/regressions/6009-opencode-maintenance-session.test.ts @@ -233,7 +233,7 @@ describe.each(["opencode", "opencode-go"])("ENG-6009 %s maintenance identity", ( responseText = JSON.stringify(proposal); const state: HarnessState = { schema: 1, - entries: { prompt: {}, memory: {}, skill: {}, subagent: {} }, + entries: { prompt: {}, memory: {}, skill: {}, subagent: {}, swarm: {} }, refinements: [], }; const headers: Record = { "X-Fixture": "retained" }; diff --git a/packages/coding-agent/test/suite/regressions/6010-auxiliary-reasoning.test.ts b/packages/coding-agent/test/suite/regressions/6010-auxiliary-reasoning.test.ts index 9f0b3fad19..c85dcd4ec7 100644 --- a/packages/coding-agent/test/suite/regressions/6010-auxiliary-reasoning.test.ts +++ b/packages/coding-agent/test/suite/regressions/6010-auxiliary-reasoning.test.ts @@ -90,7 +90,7 @@ describe("auxiliary reasoning settings", () => { ); const state: HarnessState = { schema: 1, - entries: { prompt: {}, memory: {}, skill: {}, subagent: {} }, + entries: { prompt: {}, memory: {}, skill: {}, subagent: {}, swarm: {} }, refinements: [], }; const messages = [{ role: "user" as const, content: "Remember the result", timestamp: 1 }]; diff --git a/packages/coding-agent/test/suite/regressions/6010-refinement-output-budget.test.ts b/packages/coding-agent/test/suite/regressions/6010-refinement-output-budget.test.ts index d3ec65b32b..0f076adcd9 100644 --- a/packages/coding-agent/test/suite/regressions/6010-refinement-output-budget.test.ts +++ b/packages/coding-agent/test/suite/regressions/6010-refinement-output-budget.test.ts @@ -30,7 +30,7 @@ async function requestRefinement( ) { const state: HarnessState = { schema: 1, - entries: { prompt: {}, memory: {}, skill: {}, subagent: {} }, + entries: { prompt: {}, memory: {}, skill: {}, subagent: {}, swarm: {} }, refinements: [], }; const messages = content ? [{ role: "user" as const, content, timestamp: 1 }] : []; diff --git a/packages/coding-agent/test/swarm-dag-eval.test.ts b/packages/coding-agent/test/swarm-dag-eval.test.ts new file mode 100644 index 0000000000..96eee01dcb --- /dev/null +++ b/packages/coding-agent/test/swarm-dag-eval.test.ts @@ -0,0 +1,919 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { + BUILDER_MARKER, + buildBaselinePrompt, + buildBrokenDag, + buildBuilderDag, + buildHarnessStateFile, + buildReferenceSwarms, + buildResidentWatcherDag, + buildReviewSweepDag, + buildReviewSweepFailDag, + buildSwarmParentPrompt, + checkReplayLedger, + checkTaskSuccess, + computeVerdicts, + type EvalConfig, + type ParsedAnswer, + parseAnswerLine, + parseEvalArgs, + REVIEW_FILES, + REVIEW_ISSUE_IDS, + renderMarkdownReport, + runReplayChecks, + type SwarmDagEvalTrialResult, + type SwarmLedgerEvent, + type SwarmLedgerInstance, + type SwarmLedgerNode, + type SwarmStatusLedger, +} from "../scripts/swarm-dag-eval.js"; +import { loadHarnessState } from "../src/core/refinement/refinement.js"; + +const WIDTH = 4; +const referenceSwarms = buildReferenceSwarms(WIDTH); +const byKind = (kind: string) => { + const swarm = referenceSwarms.find((entry) => entry.kind === kind); + if (!swarm) throw new Error(`missing reference swarm ${kind}`); + return swarm; +}; + +describe("reference swarm shapes", () => { + it("builds the review sweep with typed fan-in and a bounded foreach", () => { + const dag = buildReviewSweepDag(); + expect(dag.nodes.map((node) => node.id)).toEqual(["files", "review", "report"]); + expect(dag.run).toEqual({ budget_ms: 900_000, failure_policy: "escalate", max_parallel: 8 }); + const [files, review, report] = dag.nodes; + expect(files.outputs).toEqual([{ name: "files", type: "json" }]); + expect(files.budget_ms).toBe(240_000); + expect(review.inputs).toEqual([{ name: "files", type: "json", from: "files.files" }]); + expect(review.foreach).toEqual({ over: "files", max: 8 }); + expect(review.outputs).toEqual([{ name: "found", type: "text" }]); + expect(report.inputs).toEqual([ + { name: "file_list", type: "json", from: "files.files" }, + { name: "found", type: "text", from: "review.found" }, + ]); + expect(report.outputs).toEqual([{ name: "issues", type: "json" }]); + for (const node of dag.nodes) { + expect(node.id).toMatch(/^[a-z0-9][a-z0-9-]{0,63}$/); + expect(typeof node.subagent).toBe("object"); + } + }); + + it("plants one real, checkable issue per review file", () => { + expect(REVIEW_FILES).toHaveLength(4); + expect(REVIEW_ISSUE_IDS).toEqual(["AUDIT-A1", "AUDIT-B1", "AUDIT-C1", "AUDIT-D1"]); + for (const file of REVIEW_FILES) { + expect(file.code.length).toBeGreaterThan(20); + expect(file.audit.length).toBeGreaterThan(20); + expect(file.code).not.toContain(file.issueId); + } + }); + + it("plants a failing reviewer that cannot be admitted (escalation variant)", () => { + const dag = buildReviewSweepFailDag(); + const broken = dag.nodes.find((node) => node.id === "review-broken"); + expect(broken).toBeDefined(); + expect(broken?.depends_on).toEqual(["files"]); + expect(broken?.failure_policy).toBe("escalate"); + if (typeof broken?.subagent === "object") { + expect(broken.subagent.model).toBe("internal/no-such-model-for-eval"); + expect(broken.subagent.prompt).toContain("AUDIT-A1"); + } else { + throw new Error("review-broken must use an inline subagent"); + } + expect(buildReviewSweepDag().nodes.map((node) => node.id)).not.toContain("review-broken"); + }); + + it("builds the N-wide builder with per-node budgets and a typed fan-in collector", () => { + const dag = buildBuilderDag(WIDTH); + expect(dag.nodes.map((node) => node.id)).toEqual([ + ...Array.from({ length: WIDTH }, (_, i) => `builder-${i + 1}`), + "collector", + ]); + const collector = dag.nodes[dag.nodes.length - 1]; + expect(collector.inputs).toEqual( + Array.from({ length: WIDTH }, (_, i) => ({ + name: `line-${i + 1}`, + type: "text", + from: `builder-${i + 1}.line`, + })), + ); + for (const node of dag.nodes) expect(node.budget_ms).toBe(240_000); + for (let i = 1; i <= WIDTH; i++) { + const builder = dag.nodes[i - 1]; + expect(builder?.subagent).toBeTypeOf("object"); + if (typeof builder?.subagent === "object") { + expect(builder.subagent.prompt).toContain(`BUILT ${BUILDER_MARKER(i)}`); + } + } + }); + + it("builds the resident watcher with a resident head and a task chain", () => { + const dag = buildResidentWatcherDag(); + const [watcher, taskA, taskB] = dag.nodes; + expect(watcher?.lifecycle).toBe("resident"); + expect(watcher?.outputs).toBeUndefined(); + expect(watcher?.foreach).toBeUndefined(); + expect(watcher?.depends_on).toBeUndefined(); + expect(watcher?.budget_ms).toBeUndefined(); + expect(taskA?.outputs).toEqual([{ name: "step", type: "text" }]); + expect(taskB?.inputs).toEqual([{ name: "prev", type: "text", from: "task-a.step" }]); + expect(taskB?.lifecycle ?? "task").toBe("task"); + expect(taskA?.budget_ms).toBe(240_000); + }); + + it("builds the broken spec as a structurally valid but unresolvable reference", () => { + const dag = buildBrokenDag(); + expect(dag.nodes).toHaveLength(1); + expect(dag.nodes[0]?.subagent).toBe("no-such-subagent-entry"); + }); +}); + +describe("prompt invariants", () => { + it("swarm parent prompts contain no task-specific orchestration code", () => { + for (const swarm of referenceSwarms) { + const prompt = buildSwarmParentPrompt(swarm, "/tmp/ledger.json"); + expect(prompt).toContain(`rlm.swarm.run('${swarm.id}')`); + expect(prompt).not.toMatch(/rlm\.spawn/); + expect(prompt).not.toMatch(/rlm\.collect/); + } + }); + + it("baseline prompts orchestrate manually and never touch rlm.swarm", () => { + for (const kind of ["review-sweep", "builder", "resident-watcher"] as const) { + const prompt = buildBaselinePrompt(byKind(kind), "/tmp/ledger.json"); + expect(prompt).toMatch(/rlm\.spawn/); + expect(prompt).toMatch(/rlm\.collect/); + expect(prompt).not.toMatch(/rlm\.swarm/); + expect(prompt).toContain("Declared budget"); + } + }); + + it("the reviewer template carries the foreach placeholder and every audit id", () => { + const swarm = byKind("review-sweep"); + const review = swarm.dag.nodes.find((node) => node.id === "review"); + if (typeof review?.subagent !== "object") throw new Error("review must be inline"); + expect(review.subagent.prompt).toContain("{files}"); + for (const file of REVIEW_FILES) { + expect(review.subagent.prompt).toContain(file.code); + expect(review.subagent.prompt).toContain(file.issueId); + } + }); + + it("the review-sweep baseline embeds the mini-repo exactly once (honest lower bound)", () => { + const uniqueSnippet = "Math.max(value, max)"; // unique to file fa's listing line + const baseline = buildBaselinePrompt(byKind("review-sweep"), "/tmp/ledger.json"); + expect(baseline.split(uniqueSnippet).length - 1).toBe(1); + // The baseline composes reviewer prompts by substitution from the shared template. + expect(baseline).toContain('reviewer_template.replace("{files}", name)'); + expect(baseline).not.toContain("STATE: done"); + const swarmParent = buildSwarmParentPrompt(byKind("review-sweep"), "/tmp/ledger.json"); + expect(swarmParent.split(uniqueSnippet).length - 1).toBe(0); + // The other baseline arms also dropped the baked-in STATE. + expect(buildBaselinePrompt(byKind("builder"), "/tmp/ledger.json")).not.toContain("STATE: done"); + expect(buildBaselinePrompt(byKind("resident-watcher"), "/tmp/ledger.json")).not.toContain("STATE: done"); + }); + + it("the resident watcher prompt replies once and holds its turn open", () => { + const swarm = byKind("resident-watcher"); + const watcher = swarm.dag.nodes.find((node) => node.id === "watcher"); + if (typeof watcher?.subagent !== "object") throw new Error("watcher must be inline"); + expect(watcher.subagent.prompt).toContain("agent_message.send"); + expect(watcher.subagent.prompt).toContain("asyncio.sleep(900)"); + expect(watcher.subagent.prompt).toContain("Do not end your turn"); + }); +}); + +describe("harness state seeding", () => { + it("seeds swarm entries the TS host can load back", () => { + const tempDir = mkdtempSync(join(tmpdir(), "swarm-dag-eval-seed-")); + try { + const stateDir = join(tempDir, "harness"); + mkdirSync(stateDir, { recursive: true }); + writeFileSync(join(stateDir, "harness_state.json"), buildHarnessStateFile([byKind("review-sweep")])); + const state = loadHarnessState(stateDir, "local"); + const entry = state.entries.swarm["swarm-dag-eval-review-sweep"]; + expect(entry).toBeDefined(); + expect(entry?.kind).toBe("swarm"); + expect(entry?.scope).toBe("local"); + expect((entry?.arguments.dag as { nodes: { id: string }[] }).nodes.map((node) => node.id)).toEqual([ + "files", + "review", + "report", + ]); + expect(state.entries.prompt).toEqual({}); + expect(state.entries.subagent).toEqual({}); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + }); + + it("writes one entry per seeded spec with the local schema", () => { + const file = JSON.parse(buildHarnessStateFile(referenceSwarms)); + expect(file.schema).toBe(1); + expect(Object.keys(file.entries.swarm)).toHaveLength(referenceSwarms.length); + expect(file.refinements).toEqual([]); + for (const spec of referenceSwarms) { + expect(file.entries.swarm[spec.id]?.kind).toBe("swarm"); + expect(file.entries.swarm[spec.id]?.arguments.dag).toEqual(spec.dag); + } + }); +}); + +describe("answer parsing and task-success checks", () => { + const answer = (overrides: Partial): ParsedAnswer => ({ + issues: [], + markers: [], + state: null, + stopped: [], + failedNode: null, + reportStatus: null, + rejected: null, + children: null, + message: null, + ...overrides, + }); + + it("parses every ANSWER format", () => { + expect(parseAnswerLine("junk\nANSWER: ISSUES: AUDIT-A1, AUDIT-B1, AUDIT-C1, AUDIT-D1; STATE: done")).toEqual( + answer({ issues: REVIEW_ISSUE_IDS, state: "done" }), + ); + expect(parseAnswerLine("ANSWER: MARKERS: swb-marker-1,swb-marker-2; STOPPED: watcher; STATE: stopped")).toEqual( + answer({ markers: ["swb-marker-1", "swb-marker-2"], stopped: ["watcher"], state: "stopped" }), + ); + expect(parseAnswerLine("ANSWER: STATE: paused; FAILED-NODE: review-broken; REPORT-STATUS: pending")).toEqual( + answer({ state: "paused", failedNode: "review-broken", reportStatus: "pending" }), + ); + expect( + parseAnswerLine( + "ANSWER: REJECTED: yes; CHILDREN: 0; MESSAGE: node 'broken-source' references unknown subagent 'no-such-subagent-entry'", + ), + ).toEqual( + answer({ + rejected: true, + children: 0, + message: "node 'broken-source' references unknown subagent 'no-such-subagent-entry'", + }), + ); + expect(parseAnswerLine("no answer here")).toBeNull(); + expect(parseAnswerLine(undefined)).toBeNull(); + }); + + it("accepts a complete review sweep and rejects a missing planted issue", () => { + const good = answer({ issues: REVIEW_ISSUE_IDS, state: "done" }); + expect(checkTaskSuccess(byKind("review-sweep"), good, null).ok).toBe(true); + const missing = answer({ issues: REVIEW_ISSUE_IDS.slice(1), state: "done" }); + const check = checkTaskSuccess(byKind("review-sweep"), missing, null); + expect(check.ok).toBe(false); + expect(check.problems[0]).toContain("AUDIT-A1 missing"); + }); + + it("cross-checks the review sweep ledger against the aggregator preview", () => { + const ledger = statusLedgerFixture({ + state: "done", + nodes: [ + nodeFixture("files", "done"), + nodeFixture("review", "done"), + nodeFixture("report", "done", { + answer_preview: '```json\n{"issues": ["AUDIT-A1","AUDIT-B2"]}\n```', + }), + ], + }); + const good = answer({ issues: REVIEW_ISSUE_IDS, state: "done" }); + const check = checkTaskSuccess(byKind("review-sweep"), good, ledger); + expect(check.ok).toBe(false); + expect(check.problems.some((problem) => problem.includes("AUDIT-B1"))).toBe(true); + }); + + it("checks the builder markers against the collector preview", () => { + const swarm = byKind("builder"); + const markers = Array.from({ length: WIDTH }, (_, i) => BUILDER_MARKER(i + 1)); + expect(checkTaskSuccess(swarm, answer({ markers, state: "done" }), null).ok).toBe(true); + const ledger = statusLedgerFixture({ + state: "done", + nodes: [nodeFixture("collector", "done", { answer_preview: "COLLECTED swb-marker-1" })], + }); + const check = checkTaskSuccess(swarm, answer({ markers, state: "done" }), ledger); + expect(check.ok).toBe(false); + expect(check.problems.some((problem) => problem.includes("swb-marker-2"))).toBe(true); + }); + + it("checks the resident teardown: tasks settled, watcher cancelled", () => { + const swarm = byKind("resident-watcher"); + const good = answer({ markers: ["swt-1", "swt-2"], stopped: ["watcher"], state: "stopped" }); + expect(checkTaskSuccess(swarm, good, null).ok).toBe(true); + expect( + checkTaskSuccess(swarm, answer({ markers: ["swt-1", "swt-2"], stopped: [], state: "stopped" }), null).ok, + ).toBe(false); + const ledger = statusLedgerFixture({ + spec_id: "swarm-dag-eval-resident-watcher", + state: "stopped", + nodes: [ + nodeFixture("watcher", "cancelled", { + lifecycle: "resident", + instances: [instanceFixture(-1, "cancelled")], + }), + nodeFixture("task-a", "done", { instances: [instanceFixture(-1, "done", { duration_ms: 5_000 })] }), + nodeFixture("task-b", "done", { instances: [instanceFixture(-1, "done", { duration_ms: 4_000 })] }), + ], + }); + expect(checkTaskSuccess(swarm, good, ledger).ok).toBe(true); + const badLedger = statusLedgerFixture({ + state: "done", + nodes: [ + nodeFixture("watcher", "running", { lifecycle: "resident" }), + nodeFixture("task-a", "done"), + nodeFixture("task-b", "done"), + ], + }); + const check = checkTaskSuccess(swarm, good, badLedger); + expect(check.ok).toBe(false); + expect(check.problems.some((problem) => problem.includes("expected stopped"))).toBe(true); + expect(check.problems.some((problem) => problem.includes("watcher node is not cancelled"))).toBe(true); + }); + + it("checks the escalation probe answer and ledger", () => { + const swarm = byKind("review-sweep-fail"); + const good = answer({ state: "paused", failedNode: "review-broken", reportStatus: "pending" }); + expect(checkTaskSuccess(swarm, good, null).ok).toBe(true); + const ledger = statusLedgerFixture({ + state: "paused", + nodes: [ + nodeFixture("files", "done"), + nodeFixture("review", "running", { + instances: [instanceFixture(0, "running"), instanceFixture(1, "running")], + }), + nodeFixture("review-broken", "error", { error: "spawn admission failed: no such model" }), + nodeFixture("report", "pending", { instances: [] }), + ], + events: [ + eventFixture(1, "run_started"), + eventFixture(2, "milestone", { milestone: "paused" }), + eventFixture(3, "spawned", { node: "report" }), + ], + }); + const check = checkTaskSuccess(swarm, good, ledger); + expect(check.ok).toBe(false); + expect(check.problems).toContain("report node started despite the escalation pause"); + }); + + it("checks the dry-run rejection answer", () => { + const swarm = byKind("dry-run-reject"); + const good = answer({ + rejected: true, + children: 0, + message: "node 'broken-source' references unknown subagent 'no-such-subagent-entry'", + }); + expect(checkTaskSuccess(swarm, good, null).ok).toBe(true); + const bad = answer({ rejected: false, children: 2, message: "no error" }); + expect(checkTaskSuccess(swarm, bad, null).ok).toBe(false); + expect(checkTaskSuccess(swarm, null, null).problems).toContain("no ANSWER line in the parent's final text"); + }); + + it("baseline arms cross-check the collect ledger against the ANSWER ids", () => { + const swarm = byKind("review-sweep"); + const baselineLedger = Object.fromEntries( + REVIEW_FILES.map((file) => [`reviewer-${file.name}`, `FOUND ${file.issueId}`]), + ); + const good = checkTaskSuccess(swarm, answer({ issues: REVIEW_ISSUE_IDS }), null, { + arm: "baseline", + baselineLedger, + }); + expect(good.ok).toBe(true); + // No STATE field is required for baselines (the template no longer bakes one in). + expect(good.problems).toEqual([]); + + // A planted id the children never reported must fail. + const shortLedger = { "reviewer-fa": "FOUND AUDIT-A1" }; + const missingId = checkTaskSuccess(swarm, answer({ issues: REVIEW_ISSUE_IDS }), null, { + arm: "baseline", + baselineLedger: shortLedger, + }); + expect(missingId.ok).toBe(false); + expect( + missingId.problems.some((problem) => problem.includes("AUDIT-B1 missing from the baseline collect ledger")), + ).toBe(true); + + // An ANSWER id absent from the ledger must fail (the parent cannot invent it). + const invented = checkTaskSuccess(swarm, answer({ issues: [...REVIEW_ISSUE_IDS, "AUDIT-Z9"] }), null, { + arm: "baseline", + baselineLedger, + }); + expect(invented.ok).toBe(false); + expect( + invented.problems.some((problem) => + problem.includes("AUDIT-Z9 is not present in the baseline collect ledger"), + ), + ).toBe(true); + + // A missing ledger fails completion instead of passing on the self-reported ANSWER. + const noLedger = checkTaskSuccess(swarm, answer({ issues: REVIEW_ISSUE_IDS }), null, { + arm: "baseline", + baselineLedger: null, + }); + expect(noLedger.ok).toBe(false); + expect(noLedger.problems.some((problem) => problem.includes("baseline collect ledger missing"))).toBe(true); + }); + + it("baseline arms check builder markers and the resident chain against the collect ledger", () => { + const builder = byKind("builder"); + const markers = Array.from({ length: WIDTH }, (_, i) => BUILDER_MARKER(i + 1)); + const builderLedger = Object.fromEntries(markers.map((marker) => [`builder-${marker}`, `BUILT ${marker}`])); + expect( + checkTaskSuccess(builder, answer({ markers }), null, { arm: "baseline", baselineLedger: builderLedger }).ok, + ).toBe(true); + const missingMarker = checkTaskSuccess(builder, answer({ markers }), null, { + arm: "baseline", + baselineLedger: { "builder-1": "BUILT swb-marker-1" }, + }); + expect(missingMarker.ok).toBe(false); + expect( + missingMarker.problems.some((problem) => + problem.includes("swb-marker-2 missing from the baseline collect ledger"), + ), + ).toBe(true); + + const resident = byKind("resident-watcher"); + const residentLedger = { "task-a": "STEP swt-1", "task-b": "STEP swt-2" }; + expect( + checkTaskSuccess(resident, answer({ markers: ["swt-1", "swt-2"], stopped: ["watcher"] }), null, { + arm: "baseline", + baselineLedger: residentLedger, + }).ok, + ).toBe(true); + const missingTask = checkTaskSuccess(resident, answer({ markers: ["swt-1"], stopped: ["watcher"] }), null, { + arm: "baseline", + baselineLedger: residentLedger, + }); + expect(missingTask.ok).toBe(false); + expect(missingTask.problems.some((problem) => problem.includes("swt-2 missing from the ANSWER line"))).toBe(true); + }); +}); + +// --------------------------------------------------------------------------- +// Replay checker fixtures. +// --------------------------------------------------------------------------- + +function eventFixture(seq: number, kind: string, extra: Partial = {}): SwarmLedgerEvent { + return { seq, kind, stage: "delivered", ...extra }; +} + +function instanceFixture(index: number, status: string, extra: Partial = {}): SwarmLedgerInstance { + return { index, status, attempt: 1, child: "child-1", duration_ms: null, ...extra }; +} + +function nodeFixture(id: string, status: string, extra: Partial = {}): SwarmLedgerNode { + return { id, status, lifecycle: "task", attempts: 1, instances: [instanceFixture(-1, status)], ...extra }; +} + +function statusLedgerFixture(overrides: Partial = {}): SwarmStatusLedger { + return { + run_id: "run-1", + spec_id: "swarm-dag-eval-review-sweep", + name: null, + state: "done", + nodes: [], + events: [], + elapsed_ms: 12_345, + usage: { spawns: 0, settled: 0, tool_uses: 0, max_parallel: 8, running: 0 }, + ...overrides, + }; +} + +function reviewSweepLedger(): SwarmStatusLedger { + return statusLedgerFixture({ + state: "done", + nodes: [ + nodeFixture("files", "done", { instances: [instanceFixture(-1, "done", { duration_ms: 4_000 })] }), + nodeFixture("review", "done", { + instances: REVIEW_FILES.map((_, index) => instanceFixture(index, "done", { duration_ms: 8_000 })), + }), + nodeFixture("report", "done", { + instances: [instanceFixture(-1, "done", { duration_ms: 3_000 })], + answer_preview: '```json\n{"issues": ["AUDIT-A1","AUDIT-B1","AUDIT-C1","AUDIT-D1"]}\n```', + }), + ], + events: [ + eventFixture(1, "run_started", { detail: "3 nodes, max_parallel 8" }), + eventFixture(2, "node_ready", { node: "files" }), + eventFixture(3, "spawned", { node: "files", instance: -1 }), + eventFixture(4, "settled", { node: "files", instance: -1, status: "done", duration_ms: 4_000 }), + eventFixture(5, "answer_captured", { node: "files", instance: -1 }), + eventFixture(6, "node_ready", { node: "review" }), + ...REVIEW_FILES.flatMap((_, index) => [ + eventFixture(7 + index * 3, "spawned", { node: "review", instance: index }), + eventFixture(8 + index * 3, "settled", { + node: "review", + instance: index, + status: "done", + duration_ms: 8_000, + }), + eventFixture(9 + index * 3, "answer_captured", { node: "review", instance: index }), + ]), + eventFixture(19, "node_ready", { node: "report" }), + eventFixture(20, "spawned", { node: "report", instance: -1 }), + eventFixture(21, "settled", { node: "report", instance: -1, status: "done", duration_ms: 3_000 }), + eventFixture(22, "answer_captured", { node: "report", instance: -1 }), + eventFixture(23, "milestone", { milestone: "finished" }), + ], + usage: { spawns: 6, settled: 6, tool_uses: 6, max_parallel: 8, running: 0 }, + }); +} + +function residentLedger(): SwarmStatusLedger { + return statusLedgerFixture({ + spec_id: "swarm-dag-eval-resident-watcher", + state: "stopped", + nodes: [ + nodeFixture("watcher", "cancelled", { + lifecycle: "resident", + instances: [instanceFixture(-1, "cancelled")], + }), + nodeFixture("task-a", "done", { instances: [instanceFixture(-1, "done", { duration_ms: 5_000 })] }), + nodeFixture("task-b", "done", { instances: [instanceFixture(-1, "done", { duration_ms: 4_000 })] }), + ], + events: [ + eventFixture(1, "run_started"), + eventFixture(2, "node_ready", { node: "watcher" }), + eventFixture(3, "spawned", { node: "watcher", instance: -1 }), + eventFixture(4, "node_ready", { node: "task-a" }), + eventFixture(5, "spawned", { node: "task-a", instance: -1 }), + eventFixture(6, "settled", { node: "task-a", instance: -1, status: "done", duration_ms: 5_000 }), + eventFixture(7, "answer_captured", { node: "task-a", instance: -1 }), + eventFixture(8, "node_ready", { node: "task-b" }), + eventFixture(9, "spawned", { node: "task-b", instance: -1 }), + eventFixture(10, "settled", { node: "task-b", instance: -1, status: "done", duration_ms: 4_000 }), + eventFixture(11, "answer_captured", { node: "task-b", instance: -1 }), + eventFixture(12, "milestone", { milestone: "finished" }), + eventFixture(13, "node_cancelled", { node: "watcher" }), + eventFixture(14, "cancelled", { node: "watcher", instance: -1 }), + eventFixture(15, "run_stopped", { detail: "stopped; 1 node(s) cancelled" }), + ], + usage: { spawns: 3, settled: 2, tool_uses: 2, max_parallel: 8, running: 0 }, + }); +} + +function escalationLedger(): SwarmStatusLedger { + return statusLedgerFixture({ + spec_id: "swarm-dag-eval-review-fail", + state: "paused", + nodes: [ + nodeFixture("files", "done", { instances: [instanceFixture(-1, "done", { duration_ms: 4_000 })] }), + nodeFixture("review", "running", { + instances: [ + instanceFixture(0, "running"), + instanceFixture(1, "running"), + instanceFixture(2, "running"), + instanceFixture(3, "running"), + ], + }), + nodeFixture("review-broken", "error", { + instances: [instanceFixture(-1, "error", { error: "spawn admission failed: no such model" })], + error: "spawn admission failed: no such model", + }), + nodeFixture("report", "pending", { instances: [] }), + ], + events: [ + eventFixture(1, "run_started"), + eventFixture(2, "node_ready", { node: "files" }), + eventFixture(3, "spawned", { node: "files", instance: -1 }), + eventFixture(4, "settled", { node: "files", instance: -1, status: "done", duration_ms: 4_000 }), + eventFixture(5, "node_ready", { node: "review" }), + eventFixture(6, "spawned", { node: "review", instance: 0 }), + eventFixture(7, "spawned", { node: "review", instance: 1 }), + eventFixture(8, "spawned", { node: "review", instance: 2 }), + eventFixture(9, "spawned", { node: "review", instance: 3 }), + eventFixture(10, "node_ready", { node: "review-broken" }), + eventFixture(11, "settled", { + node: "review-broken", + instance: -1, + status: "error", + error: "spawn admission failed: no such model", + }), + eventFixture(12, "node_error", { node: "review-broken", error: "spawn admission failed" }), + eventFixture(13, "milestone", { milestone: "paused" }), + ], + usage: { spawns: 5, settled: 1, tool_uses: 1, max_parallel: 8, running: 4 }, + }); +} + +describe("replay checker", () => { + it("accepts the three reference ledgers with stable identities and full accounting", () => { + for (const ledger of [reviewSweepLedger(), residentLedger(), escalationLedger()]) { + const result = checkReplayLedger(ledger); + expect(result.problems).toEqual([]); + expect(result.ok).toBe(true); + } + }); + + it("rejects a ledger that does not match the status shape", () => { + const result = checkReplayLedger({ hello: "world" }); + expect(result.ok).toBe(false); + expect(result.problems[0]).toContain("shape"); + }); + + it("flags a settled-done event without a duration", () => { + const ledger = reviewSweepLedger(); + const settled = ledger.events.find((event) => event.kind === "settled"); + if (!settled) throw new Error("fixture lost its settled event"); + settled.duration_ms = undefined; + const result = checkReplayLedger(ledger); + expect(result.ok).toBe(false); + expect(result.problems.some((problem) => problem.includes("without a duration_ms"))).toBe(true); + }); + + it("flags non-increasing and duplicate event seqs", () => { + const ledger = reviewSweepLedger(); + ledger.events[1]!.seq = 1; + const result = checkReplayLedger(ledger); + expect(result.ok).toBe(false); + expect(result.problems.some((problem) => problem.includes("non-increasing seq"))).toBe(true); + }); + + it("flags a dropped event as a seq gap within the window (1, 2, 4)", () => { + const ledger = reviewSweepLedger(); + // Drop event 3 (a spawned) and renumber nothing: the gap 2 -> 4 must fail even + // though every remaining seq is strictly increasing and unique. + const dropped = ledger.events.filter((event) => event.seq !== 3); + expect(dropped.map((event) => event.seq).slice(0, 4)).toEqual([1, 2, 4, 5]); + const result = checkReplayLedger({ ...ledger, events: dropped }); + expect(result.ok).toBe(false); + expect(result.problems.some((problem) => problem.includes("seq gap"))).toBe(true); + expect(result.problems.some((problem) => problem.includes("expected 3, got 4"))).toBe(true); + }); + + it("accepts a retry ledger: two settled events on one spawned key match usage.settled", () => { + // A retried builder node: first attempt fails (settled error with duration), + // the retry event re-spawns, the second attempt settles done. The executor's + // settle_count increments per settlement, so usage.settled is 2 — the checker + // must count settled EVENTS, not distinct keys. + const ledger = statusLedgerFixture({ + spec_id: "swarm-dag-eval-builder", + state: "done", + nodes: [ + nodeFixture("builder-1", "done", { + attempts: 2, + instances: [instanceFixture(-1, "done", { attempt: 2, duration_ms: 9_000 })], + }), + ], + events: [ + eventFixture(1, "run_started"), + eventFixture(2, "node_ready", { node: "builder-1" }), + eventFixture(3, "spawned", { node: "builder-1", instance: -1 }), + eventFixture(4, "settled", { + node: "builder-1", + instance: -1, + status: "error", + error: "child error", + duration_ms: 8_000, + }), + eventFixture(5, "retry", { node: "builder-1", instance: -1 }), + eventFixture(6, "spawned", { node: "builder-1", instance: -1 }), + eventFixture(7, "settled", { node: "builder-1", instance: -1, status: "done", duration_ms: 9_000 }), + eventFixture(8, "answer_captured", { node: "builder-1", instance: -1 }), + eventFixture(9, "milestone", { milestone: "finished" }), + ], + usage: { spawns: 2, settled: 2, tool_uses: 2, max_parallel: 8, running: 0 }, + }); + const result = checkReplayLedger(ledger); + expect(result.problems).toEqual([]); + expect(result.ok).toBe(true); + // The old distinct-key count would have reported settled 1 != usage.settled 2. + const distinctKeyLedger = { ...ledger, usage: { ...ledger.usage, settled: 1 } }; + const mismatch = checkReplayLedger(distinctKeyLedger); + expect(mismatch.problems.some((problem) => problem.includes("usage.settled"))).toBe(true); + }); + + it("flags unknown event kinds and stages", () => { + const ledger = reviewSweepLedger(); + ledger.events[0] = eventFixture(1, "teleported"); + const kindResult = checkReplayLedger(ledger); + expect(kindResult.problems.some((problem) => problem.includes("unknown kind"))).toBe(true); + const staged = reviewSweepLedger(); + staged.events[0] = eventFixture(1, "run_started", { stage: "vaporized" }); + const stageResult = checkReplayLedger(staged); + expect(stageResult.problems.some((problem) => problem.includes("unknown stage"))).toBe(true); + }); + + it("flags usage counts that disagree with the event stream", () => { + const ledger = reviewSweepLedger(); + ledger.usage = { spawns: 99, settled: 6, tool_uses: 6, max_parallel: 8, running: 0 }; + const result = checkReplayLedger(ledger); + expect(result.problems.some((problem) => problem.includes("usage.spawns"))).toBe(true); + }); + + it("flags a done run without a finished milestone", () => { + const ledger = reviewSweepLedger(); + ledger.events = ledger.events.filter((event) => event.kind !== "milestone"); + const result = checkReplayLedger(ledger); + expect(result.problems.some((problem) => problem.includes("finished milestone"))).toBe(true); + }); + + it("flags a spawned instance that never settles on a completed run", () => { + const ledger = reviewSweepLedger(); + // Drop the report node's settle+answer events and mark it done: unaccounted spawn. + ledger.events = ledger.events.filter( + (event) => !(event.node === "report" && ["settled", "answer_captured"].includes(event.kind)), + ); + const result = checkReplayLedger(ledger); + expect(result.problems.some((problem) => problem.includes("never settled or cancelled"))).toBe(true); + }); + + it("notes a truncated event window instead of asserting counts", () => { + const ledger = reviewSweepLedger(); + ledger.events = ledger.events.slice(10); + ledger.events = ledger.events.map((event, index) => ({ ...event, seq: 11 + index })); + const result = checkReplayLedger(ledger); + expect(result.problems.some((problem) => problem.includes("truncated"))).toBe(true); + expect(result.problems.some((problem) => problem.includes("usage.spawns"))).toBe(false); + }); + + it("runs over a saved report.json and a bare ledger", () => { + const trial = (ledger: SwarmStatusLedger | null): SwarmDagEvalTrialResult => + ({ + swarm: "review-sweep", + arm: "swarm", + trial: 1, + ledger, + }) as SwarmDagEvalTrialResult; + const report = runReplayChecks({ trials: [trial(reviewSweepLedger()), trial(null)] }); + expect(report.ok).toBe(true); + expect(report.ledgers).toHaveLength(1); + const bare = runReplayChecks(residentLedger()); + expect(bare.ok).toBe(true); + expect(bare.ledgers[0]?.id).toBe("ledger"); + }); +}); + +describe("computeVerdicts", () => { + const trial = (overrides: Partial): SwarmDagEvalTrialResult => ({ + swarm: "review-sweep", + arm: "swarm", + trial: 1, + model: "internal/glm-5.2-fast", + taskSuccess: true, + problems: [], + state: "done", + wallMs: 1_000, + contextTokens: null, + totalTokens: 1, + declaredFanIn: 4, + queueLatencyMs: null, + teardownLatencyMs: null, + declaredBudgetMs: 900_000, + budgetOvershootMs: 0, + elapsedMs: null, + spawns: null, + settled: null, + replayOk: null, + replayProblems: [], + answer: null, + ledger: null, + verdict: "pass", + ...overrides, + }); + + it("excludes the escalation and dry-run probes from the budget sum", () => { + const verdicts = computeVerdicts([ + trial({ swarm: "review-sweep", budgetOvershootMs: 1_000 }), + trial({ swarm: "review-sweep-fail", budgetOvershootMs: 50_000, verdict: "pass" }), + trial({ swarm: "dry-run-reject", budgetOvershootMs: 70_000, verdict: "pass" }), + ]); + expect(verdicts.budgetOvershootMs).toBe(1_000); + expect(verdicts.budgetOvershootZero).toBe(false); + expect(verdicts.failurePolicyMatched).toBe(true); + expect(verdicts.dryRunRejected).toBe(true); + const clean = computeVerdicts([ + trial({ swarm: "review-sweep", budgetOvershootMs: 0 }), + trial({ swarm: "review-sweep-fail", budgetOvershootMs: 123, verdict: "fail" }), + trial({ swarm: "dry-run-reject", budgetOvershootMs: 456, verdict: "fail" }), + ]); + expect(clean.budgetOvershootMs).toBe(0); + expect(clean.budgetOvershootZero).toBe(true); + expect(clean.failurePolicyMatched).toBe(false); + expect(clean.dryRunRejected).toBe(false); + }); + + it("reports null probe verdicts when the probes did not run", () => { + const verdicts = computeVerdicts([trial({ swarm: "review-sweep" })]); + expect(verdicts.failurePolicyMatched).toBeNull(); + expect(verdicts.dryRunRejected).toBeNull(); + }); + + it("averages context tokens per pair and flags lower only when measured", () => { + const verdicts = computeVerdicts([ + trial({ swarm: "review-sweep", trial: 1, contextTokens: 100 }), + trial({ swarm: "review-sweep", trial: 2, contextTokens: 200 }), + trial({ swarm: "review-sweep", arm: "baseline", trial: 1, contextTokens: 300 }), + trial({ swarm: "review-sweep", arm: "baseline", trial: 2, contextTokens: 500 }), + ]); + expect(verdicts.contextPairs).toHaveLength(1); + const pair = verdicts.contextPairs[0]!; + expect(pair.swarmContextTokens).toBe(150); + expect(pair.baselineContextTokens).toBe(400); + expect(pair.lower).toBe(true); + expect(pair.bothCorrect).toBe(true); + + const higher = computeVerdicts([ + trial({ swarm: "review-sweep", contextTokens: 900 }), + trial({ swarm: "review-sweep", arm: "baseline", contextTokens: 800 }), + ]); + expect(higher.contextPairs[0]!.lower).toBe(false); + + const unknown = computeVerdicts([ + trial({ swarm: "review-sweep", contextTokens: null }), + trial({ swarm: "review-sweep", arm: "baseline", contextTokens: 800 }), + ]); + expect(unknown.contextPairs[0]!.lower).toBeNull(); + + const incorrect = computeVerdicts([ + trial({ swarm: "review-sweep", contextTokens: 100, taskSuccess: false, verdict: "fail" }), + trial({ swarm: "review-sweep", arm: "baseline", contextTokens: 800 }), + ]); + expect(incorrect.contextPairs[0]!.bothCorrect).toBe(false); + }); + + it("keeps the static no-orchestration verdict asserted", () => { + expect(computeVerdicts([trial({ swarm: "review-sweep" })]).noOrchestrationCode).toBe(true); + }); +}); + +describe("args and report rendering", () => { + it("parses args with defaults and clamps the width", () => { + const defaults = parseEvalArgs([]); + expect(defaults).not.toHaveProperty("error"); + if ("error" in defaults) throw new Error("unreachable"); + expect(defaults.model).toBe("internal/glm-5.2-fast"); + expect(defaults.swarms).toEqual(["review-sweep", "builder", "resident-watcher"]); + expect(defaults.width).toBe(6); + expect(defaults.outDir).toContain("swarm-dag-eval-reports/"); + const clamped = parseEvalArgs(["--width", "99", "--swarms", "builder,review-sweep", "--trials", "3"]); + if ("error" in clamped) throw new Error("unreachable"); + expect(clamped.width).toBe(12); + expect(clamped.swarms).toEqual(["builder", "review-sweep"]); + expect(clamped.trials).toBe(3); + expect(parseEvalArgs(["--nope"])).toEqual({ error: "Unknown argument: --nope" }); + // A typo in --swarms must fail before any token is spent. + const typo = parseEvalArgs(["--swarms", "review-swep"]); + expect("error" in typo).toBe(true); + if ("error" in typo) expect(typo.error).toContain("Unknown swarm in --swarms: review-swep"); + const mixed = parseEvalArgs(["--swarms", "builder,review-swep"]); + expect("error" in mixed).toBe(true); + }); + + it("renders the markdown table, pair comparison, and verdict rules", () => { + const config: EvalConfig = { + model: "internal/glm-5.2-fast", + swarms: ["review-sweep"], + width: 4, + trials: 1, + timeoutMinutes: 20, + outDir: "out", + }; + const result = ( + arm: "swarm" | "baseline", + contextTokens: number | null, + ok: boolean, + ): SwarmDagEvalTrialResult => ({ + swarm: "review-sweep", + arm, + trial: 1, + model: config.model, + taskSuccess: ok, + problems: ok ? [] : ["planted issue AUDIT-A1 missing from the ANSWER line"], + state: "done", + wallMs: 12_345, + contextTokens, + totalTokens: 9_999, + declaredFanIn: 4, + queueLatencyMs: null, + teardownLatencyMs: null, + declaredBudgetMs: 900_000, + budgetOvershootMs: 0, + elapsedMs: 100_000, + spawns: 6, + settled: 6, + replayOk: arm === "swarm" ? true : null, + replayProblems: [], + answer: null, + ledger: null, + verdict: ok ? "pass" : "fail", + }); + const markdown = renderMarkdownReport([result("swarm", 10_000, true), result("baseline", 20_000, false)], config); + expect(markdown).toContain("# Swarm DAG capability eval report"); + expect(markdown).toContain("over budget ms*"); + expect(markdown).toContain("budget overshoot is measured per arm and is not directly comparable*"); + expect(markdown).toContain("| review-sweep | swarm | 1 | ok | done |"); + expect(markdown).toContain("| review-sweep | baseline | 1 | failed | done |"); + expect(markdown).toContain("| review-sweep | 10000 | 20000 | yes | no |"); + expect(markdown).toContain( + "no task-specific orchestration code in swarm prompts (asserted statically, prompt invariant): PASS", + ); + expect(markdown).toContain("declared failure policy matches observed behavior (escalation): (not run)"); + expect(markdown).toContain("total budget overshoot (swarm arms only): 0 ms (PASS)"); + expect(markdown).toContain("- review-sweep/baseline/trial 1: planted issue AUDIT-A1 missing"); + }); +}); diff --git a/packages/coding-agent/test/swarm-executor.test.ts b/packages/coding-agent/test/swarm-executor.test.ts new file mode 100644 index 0000000000..8b44162f46 --- /dev/null +++ b/packages/coding-agent/test/swarm-executor.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, it, vi } from "vitest"; +import { startsAgentRun } from "../src/core/agent-messages.js"; +import { convertToLlm, createSwarmProgressMessage, SWARM_PROGRESS_NOTICE_CUSTOM_TYPE } from "../src/core/messages.js"; +import { createSwarmProgressHostHandler } from "../src/core/rlm-runtime.js"; + +describe("swarm progress", () => { + it("creates a bracket-grammar milestone notice and keeps it model-visible", () => { + const message = createSwarmProgressMessage({ + runId: "run-1", + kind: "budget_exceeded", + detail: "run budget_ms 600000 exceeded after 612000ms", + }); + + expect(message.customType).toBe(SWARM_PROGRESS_NOTICE_CUSTOM_TYPE); + expect(message.content).toBe( + "[swarm-progress run:run-1] budget-exceeded: run budget_ms 600000 exceeded after 612000ms", + ); + expect(convertToLlm([message])).toEqual([ + { + role: "user", + content: [{ type: "text", text: message.content }], + timestamp: message.timestamp, + }, + ]); + }); + + it("renders the finished, failed, and paused milestone kinds verbatim", () => { + expect(createSwarmProgressMessage({ runId: "r", kind: "finished", detail: "all 3 nodes done" }).content).toBe( + "[swarm-progress run:r] finished: all 3 nodes done", + ); + expect(createSwarmProgressMessage({ runId: "r", kind: "failed", detail: "node review failed" }).content).toBe( + "[swarm-progress run:r] failed: node review failed", + ); + expect( + createSwarmProgressMessage({ runId: "r", kind: "paused", detail: "escalate: awaiting resume" }).content, + ).toBe("[swarm-progress run:r] paused: escalate: awaiting resume"); + }); + + it("starts a new agent run for a swarm milestone follow-up", () => { + const message = createSwarmProgressMessage({ runId: "r", kind: "finished", detail: "done" }); + expect(startsAgentRun(message)).toBe(true); + }); + + it("validates and forwards kernel milestone payloads", async () => { + const milestone = vi.fn(); + const handler = createSwarmProgressHostHandler(milestone); + const payload = { run_id: " run-1 ", kind: "paused", node: "review", detail: "node review failed" }; + + await expect(handler(payload)).resolves.toEqual({}); + expect(milestone).toHaveBeenCalledWith({ + runId: "run-1", + kind: "paused", + node: "review", + detail: "node review failed", + }); + }); + + it("omits the node field when the kernel does not provide one", async () => { + const milestone = vi.fn(); + const handler = createSwarmProgressHostHandler(milestone); + + await handler({ run_id: "run-1", kind: "finished", detail: "all nodes done" }); + expect(milestone).toHaveBeenCalledWith({ runId: "run-1", kind: "finished", detail: "all nodes done" }); + }); + + it.each([ + [{ kind: "finished", detail: "done" }, "run_id must be a non-empty string"], + [{ run_id: "", kind: "finished", detail: "done" }, "run_id must be a non-empty string"], + [{ run_id: "r", kind: "weird", detail: "done" }, "kind must be one of"], + [{ run_id: "r", kind: "finished" }, "detail must be a non-empty string"], + [{ run_id: "r", kind: "finished", detail: " " }, "detail must be a non-empty string"], + [{ run_id: "r", kind: "finished", detail: "done", node: 5 }, "node must be a string when provided"], + [{ run_id: "r", kind: "finished", detail: "done", node: "" }, "node must be a non-empty string"], + ])("rejects an invalid payload %#", async (payload, error) => { + const handler = createSwarmProgressHostHandler(() => undefined); + await expect(handler(payload)).rejects.toThrow(error); + }); +}); diff --git a/prime-agent-runtime/src/rlm/__init__.py b/prime-agent-runtime/src/rlm/__init__.py index 3df8252275..c1e9ec42dd 100644 --- a/prime-agent-runtime/src/rlm/__init__.py +++ b/prime-agent-runtime/src/rlm/__init__.py @@ -10,6 +10,7 @@ from .bash import BashHandle, BashResult, bash from .harness import HarnessEntry, HarnessScope, HarnessState, RefinementEvent, get_harness_state +from .swarm import resume_swarm, run_swarm, status_swarm, stop_swarm _NOT_CALLABLE_MESSAGE = "'rlm' is not callable; spawn a child with: handle = await rlm.spawn('sub-task', name='worker')" _RENAMED_RUN_MESSAGE = "rlm.run was renamed; spawn a child with: handle = await rlm.spawn('sub-task', name='worker')" @@ -527,8 +528,31 @@ def __repr__(self) -> str: _harness_state = _HarnessProxy() +class _RLMSwarmNamespace: + """Run stored swarm DAGs: rlm.swarm.run/status/stop/resume. + + ``run('')`` validates a stored swarm entry, starts every + ready node up to the spec's max_parallel, and returns immediately; a + kernel asyncio task continues the run (nonblocking control loop). + Runs live in kernel memory only; children stay supervisor-owned. + """ + + async def run(self, spec_id: str, *, name: str | None = None) -> dict[str, Any]: + return await run_swarm(spec_id, name=name) + + async def status(self, run_id: str) -> dict[str, Any]: + return await status_swarm(run_id) + + async def stop(self, run_id: str) -> dict[str, Any]: + return await stop_swarm(run_id) + + async def resume(self, run_id: str) -> dict[str, Any]: + return await resume_swarm(run_id) + + class _RLMNamespace: harness = _harness_state + swarm = _RLMSwarmNamespace() get_harness_state = staticmethod(get_harness_state) async def spawn( diff --git a/prime-agent-runtime/src/rlm/harness.py b/prime-agent-runtime/src/rlm/harness.py index 5040a2fcff..24fbe4a9b3 100644 --- a/prime-agent-runtime/src/rlm/harness.py +++ b/prime-agent-runtime/src/rlm/harness.py @@ -21,12 +21,14 @@ from uuid import uuid4 from typing import Any, Literal -HarnessKind = Literal["prompt", "memory", "skill", "subagent"] +from .swarm import validate_swarm_spec + +HarnessKind = Literal["prompt", "memory", "skill", "subagent", "swarm"] HarnessScope = Literal["local", "global"] _DEFAULT_FILE_NAME = "harness_state.json" _DEFAULT_HARNESS_DIR_NAME = "harness" -_KINDS: tuple[HarnessKind, ...] = ("prompt", "memory", "skill", "subagent") +_KINDS: tuple[HarnessKind, ...] = ("prompt", "memory", "skill", "subagent", "swarm") _state_cache: dict[tuple[Path, HarnessScope], "HarnessState"] = {} @@ -765,6 +767,68 @@ def update_subagent( def delete_subagent(self, id: str, *, global_: bool = False, **kwargs: Any) -> bool: return self.delete("subagent", id, global_=global_, **kwargs) + def create_swarm( + self, + title: str, + content: str, + *, + id: str | None = None, + path: str = "general", + dag: dict[str, Any] | None = None, + metadata: dict[str, Any] | None = None, + global_: bool = False, + **kwargs: Any, + ) -> HarnessEntry: + # Write-time dry run: an invalid DAG never reaches the store. + errors = validate_swarm_spec(dag) + if errors: + raise ValueError("; ".join(errors)) + return self.create( + "swarm", + title, + content, + id=id, + path=path, + arguments={"dag": dag}, + metadata=metadata, + global_=global_, + **kwargs, + ) + + def update_swarm( + self, + id: str, + title: str, + content: str, + *, + path: str | None = None, + dag: dict[str, Any] | None = None, + metadata: dict[str, Any] | None = None, + global_: bool = False, + **kwargs: Any, + ) -> HarnessEntry: + # Only validate a DAG when one is supplied; omitting it preserves the + # stored arguments (see _upsert) rather than forcing every title/content + # update to re-send the full DAG, exactly like update_skill treats reference. + if dag is not None: + errors = validate_swarm_spec(dag) + if errors: + raise ValueError("; ".join(errors)) + return self.update( + "swarm", + id, + title, + content, + path=path, + arguments={"dag": dag} if dag is not None else None, + metadata=metadata, + global_=global_, + **kwargs, + ) + + def delete_swarm(self, id: str, *, global_: bool = False, **kwargs: Any) -> bool: + return self.delete("swarm", id, global_=global_, **kwargs) + def record_refinement( self, trigger: str, @@ -824,6 +888,10 @@ def overview(self, *, max_entries_per_kind: int = 20, global_: bool = False, **k "files; children reply with await agent_message.send(message, receiver_role='parent'). Use " "await rlm.list_subagents() to recover direct child handles and await agent_message.send(..., " "receiver_role='child', receiver_name=handle.name) for follow-ups.", + "Swarm entries declare a validated DAG of subagent nodes in arguments['dag']: manage them with " + "create_swarm/update_swarm/delete_swarm (create_swarm validates the DAG at write time); run them " + "with await rlm.swarm.run(\"\"), watch with rlm.swarm.status(run_id), stop with " + "rlm.swarm.stop(run_id), and resume a paused run with rlm.swarm.resume(run_id).", ] for kind in _KINDS: records = self.list(kind)[:max_entries_per_kind] diff --git a/prime-agent-runtime/src/rlm/swarm.py b/prime-agent-runtime/src/rlm/swarm.py new file mode 100644 index 0000000000..6bbd4315bd --- /dev/null +++ b/prime-agent-runtime/src/rlm/swarm.py @@ -0,0 +1,1456 @@ +"""Swarm DAG specification helpers and executor. + +A continual-harness ``swarm`` entry stores a declarative DAG of subagent +nodes in ``arguments["dag"]``. The first half of this module implements +the write-time dry run: a validator that checks the whole graph before +anything is stored, a canonicalizer that applies defaults, and a stable +topological sort used for cycle detection. + +The second half implements the executor (``SwarmExecutor`` and the +``rlm.swarm`` namespace: run/status/stop/resume). The executor runs a +canonicalized DAG through the existing RLM supervisor: nodes are admitted +with ``rlm.spawn``, settled through ``rlm.collect``, and cancelled with +``rlm.delete_subagent``. The supervisor owns the children; the executor +owns the run state in kernel memory. Runs do not survive a kernel restart +(the registry lives in this module's state); children are supervisor-owned +and keep running, so ``rlm.list_subagents`` can still see them after a +restart. +""" + +from __future__ import annotations + +import copy +import heapq +import json +import re +import time +from dataclasses import dataclass, field +from typing import Any, Callable +from uuid import uuid4 + +FAILURE_POLICIES: tuple[str, ...] = ("fail_fast", "continue", "escalate") +PORT_TYPES: tuple[str, ...] = ("text", "json") +LIFECYCLES: tuple[str, ...] = ("task", "resident") +MAX_NODES = 1024 +MAX_RETRIES = 10 +MAX_PARALLEL_MIN = 1 +MAX_PARALLEL_MAX = 64 +FOREACH_MAX_MIN = 1 +FOREACH_MAX_MAX = 256 +RUN_FAILURE_POLICY_DEFAULT = "escalate" +RUN_MAX_PARALLEL_DEFAULT = 8 +NODE_LIFECYCLE_DEFAULT = "task" +NODE_RETRIES_DEFAULT = 0 + +_NODE_ID_PATTERN = re.compile(r"[a-z0-9][a-z0-9-]{0,63}") + + +def _is_int(value: Any) -> bool: + """True for real integers; booleans are not accepted as ints.""" + return isinstance(value, int) and not isinstance(value, bool) + + +def _is_positive_int(value: Any) -> bool: + return _is_int(value) and value > 0 + + +def _is_nonempty_str(value: Any) -> bool: + return isinstance(value, str) and value != "" + + +def _valid_node_id(value: Any) -> bool: + return _is_nonempty_str(value) and _NODE_ID_PATTERN.fullmatch(value) is not None + + +def _port_list(node: dict[str, Any], key: str) -> list[Any]: + """Return the node's inputs/outputs list, or [] when absent or malformed.""" + raw = node.get(key) + return raw if isinstance(raw, list) else [] + + +def _port_names(node: dict[str, Any], key: str) -> list[Any]: + return [entry.get("name") if isinstance(entry, dict) else None for entry in _port_list(node, key)] + + +def _declared_port_types(node: dict[str, Any], key: str) -> dict[str, str]: + """Map port name to type for well-formed entries of the node's port list.""" + ports: dict[str, str] = {} + for entry in _port_list(node, key): + if isinstance(entry, dict): + name, port_type = entry.get("name"), entry.get("type") + if _is_nonempty_str(name) and port_type in PORT_TYPES: + ports[name] = port_type + return ports + + +def _input_sources(node: dict[str, Any]) -> list[str]: + """Source node ids referenced by the node's inputs.""" + sources: list[str] = [] + for inp in _port_list(node, "inputs"): + if not isinstance(inp, dict): + continue + source = inp.get("from") + if isinstance(source, str) and "." in source: + sources.append(source.partition(".")[0]) + return sources + + +def _cycle_check_nodes(nodes_by_id: dict[str, dict[str, Any]]) -> list[dict[str, Any]]: + """Build a cleaned graph for the cycle check. + + Edges already reported as errors (unknown targets, self edges, and + resident targets) are dropped so the cycle check reports each problem + once instead of duplicating those errors. + """ + def edge_ok(node_id: str, dep: str) -> bool: + return ( + dep in nodes_by_id + and dep != node_id + and nodes_by_id[dep].get("lifecycle", NODE_LIFECYCLE_DEFAULT) != "resident" + ) + + cleaned: list[dict[str, Any]] = [] + for node_id, node in nodes_by_id.items(): + entry: dict[str, Any] = {"id": node_id} + entry["depends_on"] = [dep for dep in _port_list(node, "depends_on") if edge_ok(node_id, dep)] + kept_inputs = [ + {"from": inp["from"]} + for inp in _port_list(node, "inputs") + if isinstance(inp, dict) + and isinstance(inp.get("from"), str) + and "." in inp["from"] + and edge_ok(node_id, inp["from"].partition(".")[0]) + ] + if kept_inputs: + entry["inputs"] = kept_inputs + cleaned.append(entry) + return cleaned + + +def validate_swarm_spec(dag: Any) -> list[str]: + """Dry-run validation for a swarm DAG. + + Returns a list of human-readable error sentences; an empty list means + the specification is valid. Every rule checked here is enforced before + a swarm entry is stored, so an invalid DAG never reaches the store. + """ + errors: list[str] = [] + if not isinstance(dag, dict): + return ["swarm dag must be a JSON object"] + + run = dag.get("run") + if run is not None: + if not isinstance(run, dict): + errors.append("run must be an object") + run = None + run_budget = run.get("budget_ms") if isinstance(run, dict) else None + if run_budget is not None and not _is_positive_int(run_budget): + errors.append("run budget_ms must be a positive integer") + run_budget = None + run_policy = run.get("failure_policy") if isinstance(run, dict) else None + if run_policy is not None and run_policy not in FAILURE_POLICIES: + errors.append(f"run failure_policy must be one of {list(FAILURE_POLICIES)}, got {run_policy!r}") + max_parallel = run.get("max_parallel") if isinstance(run, dict) else None + if max_parallel is not None and not ( + _is_int(max_parallel) and MAX_PARALLEL_MIN <= max_parallel <= MAX_PARALLEL_MAX + ): + errors.append(f"run max_parallel must be an integer between {MAX_PARALLEL_MIN} and {MAX_PARALLEL_MAX}") + + nodes = dag.get("nodes") + if not isinstance(nodes, list): + errors.append("swarm dag requires a nodes list") + return errors + if not 1 <= len(nodes) <= MAX_NODES: + errors.append(f"swarm dag must declare between 1 and {MAX_NODES} nodes, got {len(nodes)}") + return errors + + seen_ids: set[str] = set() + nodes_by_id: dict[str, dict[str, Any]] = {} + for index, node in enumerate(nodes): + if not isinstance(node, dict): + errors.append(f"nodes[{index}] must be an object") + continue + node_id = node.get("id") + if not _is_nonempty_str(node_id): + errors.append(f"nodes[{index}] requires a non-empty id") + elif not _valid_node_id(node_id): + errors.append(f"nodes[{index}] id must match ^[a-z0-9][a-z0-9-]{{0,63}}$, got {node_id!r}") + elif node_id in seen_ids: + errors.append(f"nodes[{index}] duplicates node id {node_id!r}") + else: + seen_ids.add(node_id) + nodes_by_id[node_id] = node + + for node_id, node in nodes_by_id.items(): + ref = node_id + lifecycle = node.get("lifecycle", NODE_LIFECYCLE_DEFAULT) + if lifecycle not in LIFECYCLES: + errors.append(f"node {ref} lifecycle must be 'task' or 'resident', got {lifecycle!r}") + is_resident = lifecycle == "resident" + + subagent = node.get("subagent") + if _is_nonempty_str(subagent): + pass # Harness subagent entry id or title; resolved at run time. + elif isinstance(subagent, dict): + if not _is_nonempty_str(subagent.get("prompt")): + errors.append(f"node {ref} inline subagent requires a non-empty prompt") + for key in ("name", "model", "thinking"): + value = subagent.get(key) + if value is not None and not _is_nonempty_str(value): + errors.append(f"node {ref} inline subagent {key} must be a non-empty string when provided") + else: + errors.append( + f"node {ref} requires a subagent: a harness subagent id/title string " + "or an inline object with a prompt" + ) + + budget = node.get("budget_ms") + if budget is not None: + if not _is_positive_int(budget): + errors.append(f"node {ref} budget_ms must be a positive integer") + elif run_budget is not None and budget > run_budget: + errors.append(f"node {ref} budget_ms {budget} exceeds the run budget_ms {run_budget}") + + retries = node.get("retries") + if retries is not None and not (_is_int(retries) and 0 <= retries <= MAX_RETRIES): + errors.append(f"node {ref} retries must be an integer between 0 and {MAX_RETRIES}") + + policy = node.get("failure_policy") + if policy is not None and policy not in FAILURE_POLICIES: + errors.append(f"node {ref} failure_policy must be one of {list(FAILURE_POLICIES)}, got {policy!r}") + + outputs = node.get("outputs") + if outputs is not None and not isinstance(outputs, list): + errors.append(f"node {ref} outputs must be a list") + elif is_resident and isinstance(outputs, list) and outputs: + errors.append(f"resident node {ref} cannot declare outputs") + reported_duplicate_outputs: set[str] = set() + for index, out in enumerate(_port_list(node, "outputs")): + if not isinstance(out, dict): + errors.append(f"node {ref} outputs[{index}] must be an object") + continue + name, port_type = out.get("name"), out.get("type") + if not _is_nonempty_str(name): + errors.append(f"node {ref} outputs[{index}] requires a non-empty name") + elif _port_names(node, "outputs").count(name) > 1 and name not in reported_duplicate_outputs: + reported_duplicate_outputs.add(name) + errors.append(f"node {ref} declares duplicate output name {name!r}") + if port_type not in PORT_TYPES: + errors.append(f"node {ref} output {name!r} type must be 'text' or 'json'") + + inputs = node.get("inputs") + if inputs is not None and not isinstance(inputs, list): + errors.append(f"node {ref} inputs must be a list") + reported_duplicate_inputs: set[str] = set() + for index, inp in enumerate(_port_list(node, "inputs")): + if not isinstance(inp, dict): + errors.append(f"node {ref} inputs[{index}] must be an object") + continue + name, port_type, source = inp.get("name"), inp.get("type"), inp.get("from") + if not _is_nonempty_str(name): + errors.append(f"node {ref} inputs[{index}] requires a non-empty name") + elif _port_names(node, "inputs").count(name) > 1 and name not in reported_duplicate_inputs: + reported_duplicate_inputs.add(name) + errors.append(f"node {ref} declares duplicate input name {name!r}") + if port_type not in PORT_TYPES: + errors.append(f"node {ref} input {name!r} type must be 'text' or 'json'") + if not isinstance(source, str) or "." not in source: + errors.append( + f"node {ref} input {name!r} requires a 'from' reference of the form '.'" + ) + continue + src_id, _, src_output = source.partition(".") + if src_id not in nodes_by_id: + errors.append(f"node {ref} input {name!r} references unknown node {src_id!r}") + continue + src = nodes_by_id[src_id] + if src.get("lifecycle", NODE_LIFECYCLE_DEFAULT) == "resident": + errors.append(f"node {ref} input {name!r} cannot read from resident node {src_id!r}") + continue + src_output_types = _declared_port_types(src, "outputs") + if src_output not in src_output_types: + errors.append( + f"node {ref} input {name!r} references output {src_output!r} " + f"that node {src_id!r} does not declare" + ) + elif port_type in PORT_TYPES and src_output_types[src_output] != port_type: + errors.append( + f"node {ref} input {name!r} of type {port_type!r} cannot read from " + f"output {src_output!r} of type {src_output_types[src_output]!r}" + ) + + depends_on = node.get("depends_on") + if depends_on is not None: + if not isinstance(depends_on, list): + errors.append(f"node {ref} depends_on must be a list of node ids") + else: + for dep in depends_on: + if not _is_nonempty_str(dep): + errors.append(f"node {ref} depends_on entries must be non-empty node id strings") + elif dep == ref: + errors.append(f"node {ref} cannot depend on itself") + elif dep not in nodes_by_id: + errors.append(f"node {ref} depends on unknown node {dep!r}") + elif nodes_by_id[dep].get("lifecycle", NODE_LIFECYCLE_DEFAULT) == "resident": + errors.append(f"node {ref} cannot depend on resident node {dep!r}") + + foreach = node.get("foreach") + if foreach is not None: + if is_resident: + errors.append(f"resident node {ref} cannot use foreach") + if not isinstance(foreach, dict): + errors.append(f"node {ref} foreach must be an object") + else: + over = foreach.get("over") + if not _is_nonempty_str(over): + errors.append(f"node {ref} foreach.over must be a non-empty input name") + else: + declared_inputs = _declared_port_types(node, "inputs") + if over not in declared_inputs: + errors.append( + f"node {ref} foreach.over must name one of this node's inputs, got {over!r}" + ) + elif declared_inputs[over] != "json": + errors.append(f"node {ref} foreach.over input {over!r} must have type 'json'") + foreach_max = foreach.get("max") + if not (_is_int(foreach_max) and FOREACH_MAX_MIN <= foreach_max <= FOREACH_MAX_MAX): + errors.append( + f"node {ref} foreach.max must be an integer between {FOREACH_MAX_MIN} and {FOREACH_MAX_MAX}" + ) + + # The graph must be acyclic over effective dependencies (depends_on plus + # every inputs[].from source); edges already reported above are dropped. + try: + topological_order(_cycle_check_nodes(nodes_by_id)) + except ValueError as exc: + errors.append(str(exc)) + return errors + + +def canonicalize_swarm_spec(dag: Any) -> dict[str, Any]: + """Apply defaults and normalize a validated DAG into a clean dict. + + Raises ``ValueError`` with the joined error list when the DAG is + invalid. Defaults: run failure_policy 'escalate', run max_parallel 8, + node lifecycle 'task', node retries 0, and node failure_policy copied + from the run policy. + """ + errors = validate_swarm_spec(dag) + if errors: + raise ValueError("; ".join(errors)) + assert isinstance(dag, dict) # validated above + run_in = dag.get("run") if isinstance(dag.get("run"), dict) else {} + run_policy = run_in.get("failure_policy", RUN_FAILURE_POLICY_DEFAULT) + run: dict[str, Any] = { + "failure_policy": run_policy, + "max_parallel": run_in.get("max_parallel", RUN_MAX_PARALLEL_DEFAULT), + } + if "budget_ms" in run_in: + run["budget_ms"] = run_in["budget_ms"] + nodes_out: list[dict[str, Any]] = [] + for node in dag["nodes"]: + node_out: dict[str, Any] = { + "id": node["id"], + "subagent": copy.deepcopy(node["subagent"]), + "lifecycle": node.get("lifecycle", NODE_LIFECYCLE_DEFAULT), + "retries": node.get("retries", NODE_RETRIES_DEFAULT), + "failure_policy": node.get("failure_policy", run_policy), + } + if "budget_ms" in node: + node_out["budget_ms"] = node["budget_ms"] + if "depends_on" in node: + # Deduplicate while preserving order. + node_out["depends_on"] = list(dict.fromkeys(node.get("depends_on") or [])) + if "inputs" in node: + node_out["inputs"] = copy.deepcopy(node["inputs"]) + if "outputs" in node: + node_out["outputs"] = copy.deepcopy(node["outputs"]) + if "foreach" in node: + node_out["foreach"] = copy.deepcopy(node["foreach"]) + nodes_out.append(node_out) + return {"run": run, "nodes": nodes_out} + + +def topological_order(nodes: list[dict[str, Any]]) -> list[str]: + """Return node ids in a dependency-respecting order. + + Edges are the effective dependencies: ``depends_on`` plus every + ``inputs[].from`` source node. Raises ``ValueError`` on a duplicate id, + an unknown dependency, or a cycle. The order is stable: among ready + nodes, input order wins. + """ + if not isinstance(nodes, list): + raise ValueError("nodes must be a list") + index_of: dict[str, int] = {} + for index, node in enumerate(nodes): + if not isinstance(node, dict): + raise ValueError(f"nodes[{index}] must be an object") + node_id = node.get("id") + if not isinstance(node_id, str) or not node_id: + raise ValueError(f"nodes[{index}] requires a non-empty id") + if node_id in index_of: + raise ValueError(f"duplicate node id {node_id!r}") + index_of[node_id] = index + + deps: dict[str, set[str]] = {} + for node in nodes: + node_id = node["id"] + edges: set[str] = set() + depends_on = node.get("depends_on") + if depends_on is not None: + if not isinstance(depends_on, list): + raise ValueError(f"node {node_id!r} depends_on must be a list of node ids") + for dep in depends_on: + if not isinstance(dep, str) or not dep: + raise ValueError(f"node {node_id!r} depends_on entries must be non-empty node id strings") + edges.add(dep) + inputs = node.get("inputs") + if inputs is not None: + if not isinstance(inputs, list): + raise ValueError(f"node {node_id!r} inputs must be a list") + for inp in inputs: + if not isinstance(inp, dict): + raise ValueError(f"node {node_id!r} inputs entries must be objects") + source = inp.get("from") + if not isinstance(source, str) or "." not in source: + raise ValueError( + f"node {node_id!r} inputs require a 'from' reference of the form '.'" + ) + edges.add(source.partition(".")[0]) + deps[node_id] = edges + + for node_id, edges in deps.items(): + for dep in edges: + if dep not in index_of: + raise ValueError(f"node {node_id!r} depends on unknown node {dep!r}") + + remaining = {node_id: len(edges) for node_id, edges in deps.items()} + dependents: dict[str, list[str]] = {node_id: [] for node_id in index_of} + for node_id, edges in deps.items(): + for dep in edges: + dependents[dep].append(node_id) + ready = [(index_of[node_id], node_id) for node_id, count in remaining.items() if count == 0] + heapq.heapify(ready) + order: list[str] = [] + while ready: + _, current = heapq.heappop(ready) + order.append(current) + for dependent in dependents[current]: + remaining[dependent] -= 1 + if remaining[dependent] == 0: + heapq.heappush(ready, (index_of[dependent], dependent)) + if len(order) != len(index_of): + stuck = sorted(node_id for node_id, count in remaining.items() if count > 0) + raise ValueError(f"the swarm graph contains a cycle involving nodes: {', '.join(stuck)}") + return order + + +__all__ = [ + "SwarmExecutor", + "SwarmRun", + "canonicalize_swarm_spec", + "default_swarm_executor", + "resume_swarm", + "run_swarm", + "status_swarm", + "stop_swarm", + "topological_order", + "validate_swarm_spec", +] + + +# --------------------------------------------------------------------------- +# Executor: run a canonicalized DAG through the RLM supervisor. +# --------------------------------------------------------------------------- + +ANSWER_CAPTURE_CAP = 200 +"""Local safety cap for captured answers. + +``rlm.collect`` already returns previews: the host caps them at 160 +characters (``compactRlmText``). Input binding and every rendered prompt +therefore work on capped preview text; full child outputs stay in the +child's own session and are never seen by the executor. +""" + +EVENT_WINDOW = 50 +"""Number of trailing ledger events returned by ``status()``.""" + +POLL_TIMEOUT_MS = 2000 +"""How long each control-loop ``rlm.collect`` waits for unsettled children.""" + +BACKOFF_MAX_ATTEMPTS = 5 +"""Spawn admissions per node before a persistent rate limit fails the node.""" + +BACKOFF_BASE_SECONDS = 1.0 +BACKOFF_CAP_SECONDS = 60.0 +_RATE_LIMIT_MARKERS = ( + "rate limit", + "rate-limit", + "ratelimit", + "429", + "too many requests", + "throttled", + "quota", + "usage limit", +) + +_FENCED_JSON_RE = re.compile(r"```json\s*(.*?)\s*```", re.DOTALL) +TERMINAL_NODE_STATUSES = ("done", "error", "cancelled") + + +def _is_rate_limit_error(message: str) -> bool: + """Heuristic: the host reports admission failures as error strings.""" + lowered = message.lower() + return any(marker in lowered for marker in _RATE_LIMIT_MARKERS) + + +def _effective_deps(node_spec: dict[str, Any]) -> set[str]: + """Dependencies that gate a node: depends_on plus every inputs[].from source.""" + deps = set(node_spec.get("depends_on") or []) + for inp in node_spec.get("inputs") or []: + source = inp.get("from") + if isinstance(source, str) and "." in source: + deps.add(source.partition(".")[0]) + return deps + + +def _child_name(run_id: str, node_id: str, instance_index: int, attempt: int) -> str: + """Unique, readable sibling name for one spawned instance (host caps names at 64).""" + parts = ["sw", node_id[:20], run_id[:6]] + if instance_index >= 0: + parts.append(f"i{instance_index}") + if attempt > 1: + parts.append(f"a{attempt}") + return "-".join(parts) + + +def _parse_json_output(answer: str, output_name: str) -> tuple[Any, str | None]: + """Extract one named JSON output from an upstream answer. + + Prefers the trailing fenced `````json`` block whose object contains the + output name, then falls back to parsing the whole answer. Returns + ``(value, None)`` or ``(None, error_sentence)``. + """ + candidates: list[str] = [] + fenced = _FENCED_JSON_RE.findall(answer) + if fenced: + candidates.append(fenced[-1]) + candidates.append(answer.strip()) + for candidate in candidates: + try: + parsed = json.loads(candidate) + except (ValueError, TypeError): + continue + if isinstance(parsed, dict) and output_name in parsed: + return parsed[output_name], None + return None, f"no JSON object containing output {output_name!r} in the upstream answer" + + +def _render_prompt(template: str, values: dict[str, str]) -> str: + """Render bound input values into a prompt template. + + Each ``{input_name}`` placeholder is replaced in a single pass (a value + that itself looks like a placeholder is never re-substituted). Inputs + without a placeholder are appended in a trailing ``## Inputs`` section, + so no bound value is dropped. + """ + if not values: + return template + pattern = re.compile("|".join(re.escape("{" + name + "}") for name in values)) + used: set[str] = set() + + def _substitute(match: "re.Match[str]") -> str: + name = match.group(0)[1:-1] + used.add(name) + return values[name] + + rendered = pattern.sub(_substitute, template) + unplaced = [(name, value) for name, value in values.items() if name not in used] + if unplaced: + rendered += "\n\n## Inputs\n" + "".join(f"- {name}: {value}\n" for name, value in unplaced) + return rendered + + +@dataclass +class _NodeInstance: + """One spawned child of one node (a foreach node has one per item).""" + + index: int # -1 for plain nodes, 0..K-1 for foreach items + prompt: str # fully rendered; re-spawns reuse it verbatim + status: str = "pending" # pending | running | done | error | cancelled + attempt: int = 0 # spawn admissions tried for this instance + child_id: str | None = None + spawned_at: float | None = None + duration_ms: int | None = None + answer: str | None = None # capped collect preview (ANSWER_CAPTURE_CAP) + error: str | None = None + tool_uses: int = 0 + + +@dataclass +class _NodeRun: + """Executor-side state for one node of one run.""" + + node_id: str + spec: dict[str, Any] # canonical node spec + position: int # stable topological position for deterministic ordering + prompt_template: str + model: str | None = None + thinking: str | None = None + status: str = "pending" # pending | running | done | error | cancelled + instances: list[_NodeInstance] = field(default_factory=list) + error: str | None = None + + @property + def lifecycle(self) -> str: + return self.spec.get("lifecycle", NODE_LIFECYCLE_DEFAULT) + + +@dataclass +class SwarmRun: + """Executor-side state for one run. Kernel memory only: it does not + survive a kernel restart; the children (supervisor-owned) keep running.""" + + run_id: str + spec_id: str + name: str | None + state: str = "running" # running | stopping | paused | done | failed | stopped + started_at: float = 0.0 + max_parallel: int = RUN_MAX_PARALLEL_DEFAULT + run_budget_ms: int | None = None + budget_reported: bool = False + pause_reason: str | None = None + nodes: dict[str, _NodeRun] = field(default_factory=dict) + order: list[str] = field(default_factory=list) + events: list[dict[str, Any]] = field(default_factory=list) + milestones: set[str] = field(default_factory=set) + spawn_count: int = 0 + settle_count: int = 0 + tool_use_total: int = 0 + task: "asyncio.Task[None] | None" = None + + +def _validate_spawn_settings(model: Any, thinking: Any) -> str | None: + for key, value in (("model", model), ("thinking", thinking)): + if value is not None and (not isinstance(value, str) or not value.strip()): + return f"subagent {key} must be a non-empty string when provided" + return None + + +class SwarmExecutor: + """Runs canonicalized swarm DAGs through the existing RLM supervisor. + + Ownership split: the supervisor owns the children (admission via + ``rlm.spawn``, settlement via ``rlm.collect``, cancellation via + ``rlm.delete_subagent``); this executor owns the run state in kernel + memory. Every host call resolves through the module-level ``rlm`` + functions and ``host_request`` at call time, so tests can patch + ``rlm.host_request``. ``now`` (default ``time.monotonic``) and ``sleep`` + (default ``asyncio.sleep``) are injectable: budgets measure admission + to settlement, and rate-limit backoff is testable with fake sleeps. + + Runs do not survive a kernel restart (the registry lives in kernel + memory); children are supervisor-owned and keep running, so + ``rlm.list_subagents`` can still see and stop them after a restart. + """ + + def __init__( + self, + *, + now: "Callable[[], float] | None" = None, + sleep: "Callable[[float], Any] | None" = None, + harness: Any = None, + ) -> None: + import asyncio + + self._now_fn: Callable[[], float] = now or time.monotonic + self._sleep_fn: Callable[[float], Any] = sleep or asyncio.sleep + self._harness = harness + self._runs: dict[str, SwarmRun] = {} + + # -- public API --------------------------------------------------------- + + async def run(self, spec_id: str, *, name: str | None = None) -> dict[str, Any]: + """Validate a stored swarm spec and start a run of it. + + The dry run happens in two halves. Write time (``create_swarm``) + validated the graph; here ``run`` re-validates and canonicalizes it, + then resolves every node's subagent reference, reporting ALL + failures in one ``ValueError`` and starting nothing on any failure. + The resolved node count and ``max_parallel`` are reported in the + result; actual admission limits (concurrency, tree depth, provider + rate limits) are enforced at spawn time through the backoff path. + Admission spawns every ready node up to ``max_parallel``, records + handles, and returns; a background asyncio task continues the run, so + the calling model turn ends immediately (nonblocking). + """ + harness = self._resolve_harness() + entry = harness.get("swarm", spec_id) + if entry is None: + raise ValueError(f"unknown swarm spec {spec_id!r}") + dag = entry.arguments.get("dag") if isinstance(entry.arguments, dict) else None + canonical = canonicalize_swarm_spec(dag) + resolved, reference_errors = self._resolve_subagents(harness, canonical) + if reference_errors: + raise ValueError("; ".join(reference_errors)) + run = self._create_run(entry.id, canonical, resolved, name=name) + self._runs[run.run_id] = run + self._event(run, "run_started", detail=f"{len(run.nodes)} nodes, max_parallel {run.max_parallel}") + started = await self._spawn_ready(run, allow_backoff=False) + if self._run_complete(run): + await self._finalize(run) + else: + self._start_loop(run) + return { + "run_id": run.run_id, + "spec_id": entry.id, + "name": name, + "nodes": len(run.nodes), + "max_parallel": run.max_parallel, + "started": started, + "pending": self._pending_node_ids(run), + } + + async def status(self, run_id: str) -> dict[str, Any]: + """Node states, the trailing event window, elapsed time, and usage. + + Every call marks the whole ledger ``delivered`` (the parent read + it); the returned window is the last ``EVENT_WINDOW`` events. + Raises ``ValueError`` for an unknown run id. + """ + run = self._require_run(run_id) + nodes: list[dict[str, Any]] = [] + for node_id in run.order: + node = run.nodes[node_id] + entry: dict[str, Any] = { + "id": node.node_id, + "status": node.status, + "lifecycle": node.lifecycle, + "attempts": sum(instance.attempt for instance in node.instances), + "instances": [ + { + "index": instance.index, + "status": instance.status, + "attempt": instance.attempt, + "child": instance.child_id, + "duration_ms": instance.duration_ms, + "error": instance.error, + } + for instance in node.instances + ], + } + answer = self._node_answer(node) + if answer is not None: + entry["answer_preview"] = answer + if node.error is not None: + entry["error"] = node.error + nodes.append(entry) + for event in run.events: + event["stage"] = "delivered" + return { + "run_id": run.run_id, + "spec_id": run.spec_id, + "name": run.name, + "state": run.state, + "nodes": nodes, + "events": [dict(event) for event in run.events[-EVENT_WINDOW:]], + "elapsed_ms": int((self._now_fn() - run.started_at) * 1000), + "usage": { + "spawns": run.spawn_count, + "settled": run.settle_count, + "tool_uses": run.tool_use_total, + "max_parallel": run.max_parallel, + "running": self._running_instance_count(run), + }, + } + + async def stop(self, run_id: str) -> dict[str, Any]: + """Cancel every running child of the run and mark it stopped. + + Sets the transitional ``stopping`` state before the first await so + the control loop cannot admit new children or finalize the run + while the cancellations are in flight. Idempotent: a second stop + returns the same result without another ledger event. + """ + run = self._require_run(run_id) + if run.state == "stopped": + return {"run_id": run.run_id, "state": "stopped", "cancelled": []} + run.state = "stopping" + stopped = await self._halt_nonterminal(run, "run stopped") + run.state = "stopped" + self._event(run, "run_stopped", detail=f"stopped; {len(stopped)} node(s) cancelled") + return {"run_id": run.run_id, "state": "stopped", "cancelled": stopped} + + async def resume(self, run_id: str) -> dict[str, Any]: + """Resume a paused run (escalate or budget pause) and restart the loop. + + A budget pause is reported once per run: resuming after it is an + explicit operator decision and no further budget pauses fire. + Raises ``ValueError`` when the run is not paused. + """ + run = self._require_run(run_id) + if run.state != "paused": + raise ValueError(f"swarm run {run_id!r} is {run.state!r}, not paused") + run.state = "running" + run.pause_reason = None + self._event(run, "resumed", detail="resumed by caller") + # allow_backoff=False: like run(), resume() must never sleep inside + # the calling model turn; rate-limited admissions defer to the loop. + started = await self._spawn_ready(run, allow_backoff=False) + if self._run_complete(run): + await self._finalize(run) + elif run.state == "running": + self._start_loop(run) + return { + "run_id": run.run_id, + "state": run.state, + "started": started, + "pending": self._pending_node_ids(run), + } + + # -- setup -------------------------------------------------------------- + + def _resolve_harness(self) -> Any: + if self._harness is not None: + return self._harness + from . import rlm as rlm_namespace + + return rlm_namespace.harness + + def _require_run(self, run_id: str) -> SwarmRun: + run = self._runs.get(run_id) + if run is None: + raise ValueError(f"unknown swarm run {run_id!r}") + return run + + def _resolve_subagents( + self, harness: Any, canonical: dict[str, Any] + ) -> tuple[dict[str, tuple[str, str | None, str | None]], list[str]]: + """Resolve every node's subagent reference; collect ALL failures. + + A string reference is a harness subagent entry id or title: its + content is the prompt template and ``metadata.model``/``metadata.thinking`` + carry optional spawn settings. An inline object uses its own fields. + """ + resolved: dict[str, tuple[str, str | None, str | None]] = {} + errors: list[str] = [] + for node_spec in canonical["nodes"]: + node_id = node_spec["id"] + reference = node_spec["subagent"] + if isinstance(reference, dict): + prompt = reference.get("prompt") + model = reference.get("model") + thinking = reference.get("thinking") + else: + entry = harness.get("subagent", reference) + if entry is None: + entry = next((row for row in harness.list("subagent") if row.title == reference), None) + if entry is None: + errors.append(f"node {node_id!r} references unknown subagent {reference!r}") + continue + prompt = entry.content + metadata = entry.metadata if isinstance(entry.metadata, dict) else {} + model = metadata.get("model") + thinking = metadata.get("thinking") + if not isinstance(prompt, str) or not prompt.strip(): + errors.append(f"node {node_id!r} has an empty subagent prompt") + continue + settings_error = _validate_spawn_settings(model, thinking) + if settings_error is not None: + errors.append(f"node {node_id!r} {settings_error}") + continue + resolved[node_id] = (prompt, model, thinking) + return resolved, errors + + def _create_run( + self, + spec_id: str, + canonical: dict[str, Any], + resolved: dict[str, tuple[str, str | None, str | None]], + *, + name: str | None, + ) -> SwarmRun: + run_spec = canonical["run"] + run = SwarmRun( + run_id=uuid4().hex, + spec_id=spec_id, + name=name, + started_at=self._now_fn(), + max_parallel=run_spec["max_parallel"], + run_budget_ms=run_spec.get("budget_ms"), + ) + run.order = topological_order(canonical["nodes"]) + position_of = {node_id: index for index, node_id in enumerate(run.order)} + for node_spec in canonical["nodes"]: + node_id = node_spec["id"] + prompt, model, thinking = resolved[node_id] + run.nodes[node_id] = _NodeRun( + node_id=node_id, + spec=node_spec, + position=position_of[node_id], + prompt_template=prompt, + model=model, + thinking=thinking, + ) + return run + + # -- event ledger ------------------------------------------------------- + + def _event( + self, + run: SwarmRun, + kind: str, + *, + node: str | None = None, + instance: int | None = None, + detail: str | None = None, + stage: str = "recorded", + **extra: Any, + ) -> dict[str, Any]: + """Append one ledger entry. + + Stages follow the spec: ``arrived`` (a child answer settled and was + captured), ``recorded`` (everything else), ``shown`` (a milestone + notice was injected into the parent conversation), and ``delivered`` + (the parent read the ledger via ``status()``). + """ + event: dict[str, Any] = {"seq": len(run.events) + 1, "kind": kind, "stage": stage} + if node is not None: + event["node"] = node + if instance is not None: + event["instance"] = instance + if detail is not None: + event["detail"] = detail + event.update(extra) + run.events.append(event) + return event + + async def _milestone(self, run: SwarmRun, kind: str, detail: str, *, node: str | None = None) -> None: + """Record a run milestone and inject one quiet notice (one per kind).""" + if kind in run.milestones: + return + run.milestones.add(kind) + event = self._event(run, "milestone", milestone=kind, detail=detail, node=node) + try: + from . import host_request + + payload: dict[str, Any] = {"run_id": run.run_id, "kind": kind, "detail": detail} + if node is not None: + payload["node"] = node + await host_request("swarm.progress", payload) + event["stage"] = "shown" + except Exception: + # A dead bridge cannot be told; the ledger keeps the milestone and + # status() still surfaces it to the parent. + pass + + # -- readiness, binding, admission -------------------------------------- + + async def _spawn_ready(self, run: SwarmRun, *, allow_backoff: bool) -> list[str]: + """Initialize ready nodes and admit pending instances up to max_parallel. + + Returns the node ids that had at least one instance admitted here. + """ + started: list[str] = [] + while run.state == "running": + await self._initialize_ready_nodes(run) + if run.state != "running": + break + if self._running_instance_count(run) >= run.max_parallel: + break + pair = self._next_pending_instance(run) + if pair is None: + break + node, instance = pair + outcome = await self._admit(run, node, instance, allow_backoff=allow_backoff) + if outcome == "admitted" and node.node_id not in started: + started.append(node.node_id) + if outcome == "deferred": + # A rate limit is usually global, so stop admitting in this + # phase; the control loop retries with exponential backoff. + break + return started + + async def _initialize_ready_nodes(self, run: SwarmRun) -> None: + """Bind inputs and create instances for every node whose deps are terminal.""" + for node_id in run.order: + node = run.nodes[node_id] + if node.status != "pending": + continue + deps = _effective_deps(node.spec) + if not all(run.nodes[dep].status in TERMINAL_NODE_STATUSES for dep in deps): + continue + instances, reason = self._prepare_instances(run, node) + if reason is not None: + await self._apply_node_failure_policy(run, node, reason) + if run.state != "running": + return + continue + node.instances = instances + if instances: + node.status = "running" + self._event(run, "node_ready", node=node_id, detail=f"{len(instances)} instance(s) prepared") + else: + node.status = "done" + self._event(run, "node_ready", node=node_id, detail="foreach expanded to zero items; nothing to run") + + def _prepare_instances(self, run: SwarmRun, node: _NodeRun) -> "tuple[list[_NodeInstance] | None, str | None]": + """Bind inputs, expand foreach, and render one prompt per instance. + + Returns ``(instances, None)`` or ``(None, reason)`` on a binding + failure. Binding failures never retry: a deterministic binding + error would recur on every re-render, so the node fails and its + failure_policy applies directly. + """ + values: dict[str, str] = {} + foreach = node.spec.get("foreach") + items: list[Any] | None = None + for inp in node.spec.get("inputs") or []: + name, port_type, source = inp["name"], inp["type"], inp["from"] + src_id, _, src_output = source.partition(".") + source_node = run.nodes.get(src_id) + if source_node is None or source_node.status != "done": + status = source_node.status if source_node is not None else "missing" + return None, f"input {name!r} from node {src_id!r} is unavailable (status {status!r})" + answer = self._node_answer(source_node) + if answer is None: + return None, f"input {name!r} from node {src_id!r} has no captured answer" + if port_type == "text": + values[name] = answer + continue + parsed, error = _parse_json_output(answer, src_output) + if error is not None: + return None, f"input {name!r}: {error}" + if foreach is not None and foreach.get("over") == name: + if not isinstance(parsed, list): + return None, f"foreach.over input {name!r} is not a JSON list" + items = parsed + continue + values[name] = json.dumps(parsed) + if foreach is None: + return [_NodeInstance(index=-1, prompt=_render_prompt(node.prompt_template, values))], None + if items is None: + return None, "foreach node did not resolve its over input" + instances = [ + _NodeInstance( + index=index, + prompt=_render_prompt( + node.prompt_template, + {**values, foreach["over"]: item if isinstance(item, str) else json.dumps(item)}, + ), + ) + for index, item in enumerate(items[: foreach["max"]]) + ] + return instances, None + + def _next_pending_instance(self, run: SwarmRun) -> "tuple[_NodeRun, _NodeInstance] | None": + for node_id in run.order: + for instance in run.nodes[node_id].instances: + if instance.status == "pending": + return run.nodes[node_id], instance + return None + + async def _admit( + self, run: SwarmRun, node: _NodeRun, instance: _NodeInstance, *, allow_backoff: bool + ) -> str: + """Spawn one instance. Returns "admitted", "deferred", or "failed". + + Rate-limited admissions back off and retry: doubling delays capped + at 60s, at most ``BACKOFF_MAX_ATTEMPTS`` admissions per call, then + the node fails through its failure_policy. In the admission phase + (``allow_backoff=False``) a rate limit does not sleep inside + ``run()``: the instance stays pending ("deferred") and the control + loop retries it with backoff. Any other admission error fails the + node immediately. + """ + from . import spawn + + instance.attempt += 1 + child_name = _child_name(run.run_id, node.node_id, instance.index, instance.attempt) + tries = BACKOFF_MAX_ATTEMPTS if allow_backoff else 1 + delay = BACKOFF_BASE_SECONDS + last_error = "spawn admission failed" + for try_index in range(tries): + try: + handle = await spawn(instance.prompt, name=child_name, model=node.model, thinking=node.thinking) + except RuntimeError as exc: + last_error = str(exc) + if not _is_rate_limit_error(last_error): + break + if try_index < tries - 1: + self._event( + run, + "spawn_backoff", + node=node.node_id, + instance=instance.index, + detail=f"rate limited; retrying in {delay:g}s", + ) + await self._sleep_fn(delay) + delay = min(delay * 2, BACKOFF_CAP_SECONDS) + continue + instance.child_id = handle.rlm_child_id + instance.spawned_at = self._now_fn() + instance.status = "running" + run.spawn_count += 1 + self._event( + run, + "spawned", + node=node.node_id, + instance=instance.index, + attempt=instance.attempt, + child=handle.rlm_child_id, + name=child_name, + ) + return "admitted" + if not allow_backoff and _is_rate_limit_error(last_error): + self._event( + run, + "spawn_deferred", + node=node.node_id, + instance=instance.index, + detail=f"rate limited at admission: {last_error}", + ) + return "deferred" + await self._apply_instance_failure(run, node, instance, f"spawn admission failed: {last_error}", retry=False) + return "failed" + + # -- settlement, retries, policies --------------------------------------- + + async def _apply_settlement(self, run: SwarmRun, node: _NodeRun, instance: _NodeInstance, result: Any) -> None: + if instance.status != "running": + return # cancelled (stop/fail_fast) while the collect was in flight + instance.duration_ms = result.duration_ms + instance.tool_uses = result.tool_use_count or 0 + run.settle_count += 1 + run.tool_use_total += instance.tool_uses + child_reason: str | None = None + if result.status == "error": + child_reason = result.error or f"child settled with status {result.status!r}" + elif result.status == "cancelled": + child_reason = "child was cancelled" + elif result.status != "done": + child_reason = f"child settled with unexpected status {result.status!r}" + if child_reason is not None: + # Child failures retry (same rendered prompt, attempts+1) while + # attempts remain; then the node failure_policy applies. + await self._apply_instance_failure(run, node, instance, child_reason, retry=True) + return + budget_ms = node.spec.get("budget_ms") + if budget_ms is not None and instance.spawned_at is not None: + elapsed_ms = (self._now_fn() - instance.spawned_at) * 1000 + if elapsed_ms > budget_ms: + # Wall-clock budget (admission to settlement) exceeded: the + # budget is spent, so no retry; the failure_policy applies. + await self._apply_instance_failure( + run, + node, + instance, + f"node budget_ms {budget_ms} exceeded ({int(elapsed_ms)}ms from admission to settlement)", + retry=False, + ) + return + instance.status = "done" + instance.answer = (result.answer_preview or "")[:ANSWER_CAPTURE_CAP] or None + self._event( + run, + "settled", + node=node.node_id, + instance=instance.index, + status="done", + duration_ms=instance.duration_ms, + ) + if instance.answer: + self._event( + run, + "answer_captured", + node=node.node_id, + instance=instance.index, + answer=instance.answer, + stage="arrived", + ) + if node.status == "running" and node.instances and all(i.status == "done" for i in node.instances): + node.status = "done" + + async def _apply_instance_failure( + self, run: SwarmRun, node: _NodeRun, instance: _NodeInstance, reason: str, *, retry: bool + ) -> None: + instance.status = "error" + instance.error = reason + self._event( + run, + "settled", + node=node.node_id, + instance=instance.index, + status="error", + error=reason, + duration_ms=instance.duration_ms, + ) + retries = node.spec.get("retries", NODE_RETRIES_DEFAULT) + if retry and instance.attempt <= retries: + instance.status = "pending" + instance.error = None + self._event( + run, + "retry", + node=node.node_id, + instance=instance.index, + detail=f"attempt {instance.attempt} failed; re-spawning (retries {retries})", + ) + return + # The instance failed permanently, so the node fails NOW. A foreach + # node does not wait for its remaining instances: without this, a + # failure that settles before its siblings leaves the node stuck in + # running with every instance terminal, and fail_fast could never + # cancel in-flight siblings. The policy guard makes the second and + # later permanent failures no-ops. + await self._apply_node_failure_policy(run, node, reason) + + async def _apply_node_failure_policy(self, run: SwarmRun, node: _NodeRun, reason: str) -> None: + if node.status in ("error", "done", "cancelled"): + return # the policy already ran for this node + policy = node.spec.get("failure_policy", RUN_FAILURE_POLICY_DEFAULT) + node.status = "error" + node.error = reason + self._event(run, "node_error", node=node.node_id, error=reason, detail=f"failure_policy {policy}") + if run.state != "running": + # stop() (or another transition) owns the run state now; keep the + # node's error but do not overwrite the final state. + return + if policy == "fail_fast": + await self._halt_nonterminal(run, "run failed (fail_fast)") + if run.state != "running": + return # stop() landed during the cancellations; it wins + cancelled_children = sum( + 1 for other in run.nodes.values() for i in other.instances if i.status == "cancelled" + ) + run.state = "failed" + await self._milestone( + run, + "failed", + f"node {node.node_id} failed: {reason}; cancelled {cancelled_children} in-flight child(ren)", + node=node.node_id, + ) + elif policy == "continue": + pass # the node stays error; dependents see a terminal dep and fail at binding + else: # escalate (default) + run.state = "paused" + run.pause_reason = reason + await self._milestone( + run, + "paused", + f"node {node.node_id} failed: {reason}; resume with await rlm.swarm.resume('{run.run_id}')", + node=node.node_id, + ) + + async def _halt_nonterminal(self, run: SwarmRun, reason: str) -> list[str]: + """Delete every running child and cancel every non-terminal node.""" + stopped = [node_id for node_id in run.order if run.nodes[node_id].status in ("pending", "running")] + await self._cancel_running(run) + for node_id in stopped: + node = run.nodes[node_id] + if node.status in ("pending", "running"): + node.status = "cancelled" + self._event(run, "node_cancelled", node=node_id, detail=reason) + return stopped + + async def _cancel_running(self, run: SwarmRun) -> None: + from . import delete_subagent + + for node_id in run.order: + node = run.nodes[node_id] + for instance in node.instances: + if instance.status != "running" or instance.child_id is None: + continue + child_id = instance.child_id + try: + await delete_subagent(child_id) + except Exception as exc: + self._event(run, "cancel_failed", node=node_id, instance=instance.index, child=child_id, error=str(exc)) + else: + self._event(run, "cancelled", node=node_id, instance=instance.index, child=child_id) + # The child is supervisor-owned; a failed delete leaves it + # running there, but the executor treats its slot as released. + instance.status = "cancelled" + + # -- completion ---------------------------------------------------------- + + def _run_complete(self, run: SwarmRun) -> bool: + for node in run.nodes.values(): + if node.lifecycle == "resident": + # A resident node finishes the run's declarative work once it + # is admitted (or terminally failed); it then stays alive under + # the parent session until rlm.swarm.stop() or session teardown. + if node.status == "pending": + return False + if any(instance.status == "pending" for instance in node.instances): + return False + continue + if node.status not in TERMINAL_NODE_STATUSES: + return False + return True + + async def _finalize(self, run: SwarmRun) -> None: + if run.state != "running": + return # stop() or a failure policy owns the final state + errors = [node for node in run.nodes.values() if node.status == "error"] + if errors: + run.state = "failed" + await self._milestone( + run, + "failed", + "completed with node error(s): " + ", ".join(node.node_id for node in errors), + ) + return + run.state = "done" + residents = [node for node in run.nodes.values() if node.lifecycle == "resident" and node.status == "running"] + detail = f"run complete: {len(run.nodes)} node(s)" + if residents: + detail += f"; {len(residents)} resident node(s) still running (stop with await rlm.swarm.stop('{run.run_id}'))" + await self._milestone(run, "finished", detail) + + # -- control loop -------------------------------------------------------- + + def _start_loop(self, run: SwarmRun) -> None: + import asyncio + + try: + loop = asyncio.get_running_loop() + except RuntimeError: + run.state = "failed" + self._event(run, "executor_error", error="no running asyncio loop; the swarm control loop needs one") + return + run.task = loop.create_task(self._control_loop(run)) + + async def _control_loop(self, run: SwarmRun) -> None: + import asyncio + + try: + await self._loop_body(run) + except asyncio.CancelledError: + raise + except Exception as exc: + # A dead bridge or host failure must not wedge the run silently; + # children stay alive under the supervisor either way. A stop() + # that landed concurrently keeps ownership of the final state. + self._event(run, "executor_error", error=f"{type(exc).__name__}: {exc}") + if run.state == "running": + run.state = "failed" + try: + await self._milestone(run, "failed", f"executor error: {exc}") + except Exception: + pass + + async def _loop_body(self, run: SwarmRun) -> None: + import asyncio + + from . import collect + + while run.state == "running": + in_flight = [ + (run.nodes[node_id], instance) + for node_id in run.order + for instance in run.nodes[node_id].instances + if instance.status == "running" and instance.child_id is not None + ] + if in_flight: + results = await collect([instance.child_id for _, instance in in_flight], timeout_ms=POLL_TIMEOUT_MS) + settled = {entry.rlm_child_id: entry for entry in results if entry.settled} + for node, instance in in_flight: + entry = settled.get(instance.child_id or "") + if entry is not None: + await self._apply_settlement(run, node, instance, entry) + # Re-check state before completion: stop() (or a policy transition) + # can land while the collect above was in flight, and a run that + # was stopped must never finalize as done. + if run.state != "running": + return + if self._run_complete(run): + await self._finalize(run) + return + if run.run_budget_ms is not None and not run.budget_reported: + elapsed_ms = (self._now_fn() - run.started_at) * 1000 + if elapsed_ms > run.run_budget_ms: + # Run budget: pause new spawns only; children already in + # flight keep running and settle normally. + run.state = "paused" + run.pause_reason = "run budget exceeded" + run.budget_reported = True + await self._milestone( + run, + "budget_exceeded", + f"run budget_ms {run.run_budget_ms} exceeded after {int(elapsed_ms)}ms; no new spawns; " + f"resume with await rlm.swarm.resume('{run.run_id}')", + ) + return + started = await self._spawn_ready(run, allow_backoff=True) + if run.state != "running": + return + if not in_flight and not started and not self._has_pending_instance(run): + # Defensive: nothing in flight, nothing admitted, nothing + # pending. A validated DAG cannot reach this state; end the + # run instead of spinning. + self._event(run, "executor_error", error="control loop stalled: no in-flight or pending instances") + run.state = "failed" + try: + await self._milestone(run, "failed", "control loop stalled") + except Exception: + pass + return + # Yield once per iteration. A real collect already waits up to + # POLL_TIMEOUT_MS, but an instantly-settling host (tests, a fast + # supervisor) must not hot-spin the loop and starve other tasks. + await asyncio.sleep(0) + + # -- small helpers -------------------------------------------------------- + + def _node_answer(self, node: _NodeRun) -> str | None: + """Captured answer for binding: one preview, or all instances joined.""" + answers = [instance.answer for instance in node.instances if instance.status == "done" and instance.answer] + if not answers: + return None + return "\n\n".join(answers) + + def _running_instance_count(self, run: SwarmRun) -> int: + return sum(1 for node in run.nodes.values() for instance in node.instances if instance.status == "running") + + def _has_pending_instance(self, run: SwarmRun) -> bool: + return any(instance.status == "pending" for node in run.nodes.values() for instance in node.instances) + + def _pending_node_ids(self, run: SwarmRun) -> list[str]: + return [node_id for node_id in run.order if run.nodes[node_id].status == "pending"] + + +_DEFAULT_EXECUTOR: SwarmExecutor | None = None + + +def default_swarm_executor() -> SwarmExecutor: + """The process-wide executor behind the ``rlm.swarm`` namespace. + + Tests that need an injected clock or sleep assign their own + ``SwarmExecutor`` to ``swarm._DEFAULT_EXECUTOR``; the namespace then + routes through it. + """ + global _DEFAULT_EXECUTOR + if _DEFAULT_EXECUTOR is None: + _DEFAULT_EXECUTOR = SwarmExecutor() + return _DEFAULT_EXECUTOR + + +async def run_swarm(spec_id: str, *, name: str | None = None) -> dict[str, Any]: + """Validate a stored swarm spec and start a nonblocking run of it.""" + return await default_swarm_executor().run(spec_id, name=name) + + +async def status_swarm(run_id: str) -> dict[str, Any]: + """Return node states, the event window, elapsed time, and usage.""" + return await default_swarm_executor().status(run_id) + + +async def stop_swarm(run_id: str) -> dict[str, Any]: + """Cancel every running child of the run and mark it stopped.""" + return await default_swarm_executor().stop(run_id) + + +async def resume_swarm(run_id: str) -> dict[str, Any]: + """Resume a paused run (escalate or budget pause).""" + return await default_swarm_executor().resume(run_id) diff --git a/prime-agent-runtime/test/test_harness.py b/prime-agent-runtime/test/test_harness.py index fcd3be2f07..cf69f5b384 100644 --- a/prime-agent-runtime/test/test_harness.py +++ b/prime-agent-runtime/test/test_harness.py @@ -19,6 +19,23 @@ "call_pattern": "await run(...)", } +SWARM_DAG = { + "run": {"failure_policy": "continue", "max_parallel": 2}, + "nodes": [ + { + "id": "collect", + "subagent": "researcher", + "outputs": [{"name": "findings", "type": "text"}], + }, + { + "id": "review", + "subagent": {"prompt": "Review the findings."}, + "depends_on": ["collect"], + "inputs": [{"name": "draft", "type": "text", "from": "collect.findings"}], + }, + ], +} + class HarnessStateTest(unittest.TestCase): def test_crud_for_all_entry_kinds(self) -> None: @@ -56,6 +73,14 @@ def test_crud_for_all_entry_kinds(self) -> None: path="subagent/path", metadata={"kind": "subagent"}, ), + "swarm": state.create_swarm( + "Swarm", + "Swarm content", + id="swarm_entry", + path="swarm/path", + dag=SWARM_DAG, + metadata={"kind": "swarm"}, + ), } for kind, entry in created.items(): @@ -73,8 +98,9 @@ def test_crud_for_all_entry_kinds(self) -> None: arguments={"target": {"type": "string", "required": True}, "mode": {"type": "string"}}, ) state.update_subagent("subagent_entry", "Subagent", "Subagent content updated") + state.update_swarm("swarm_entry", "Swarm", "Swarm content updated", dag=SWARM_DAG) - for kind in ("prompt", "memory", "skill", "subagent"): + for kind in ("prompt", "memory", "skill", "subagent", "swarm"): entry_id = f"{kind}_entry" self.assertEqual(state.get(kind, entry_id).version, 2) self.assertIn("updated", state.get(kind, entry_id).content) @@ -106,6 +132,12 @@ def test_persists_entries_and_refinements(self) -> None: "Review the proposed patch for regressions and missing tests.", metadata={"max_turns": 3}, ) + swarm = state.create_swarm( + "PR review sweep", + "Sweep review across changed files.", + id="pr_sweep", + dag=SWARM_DAG, + ) state.create_prompt_note("Refinement cadence", "Refine only after repeated evidence.") event = state.record_refinement( "skill failed twice", @@ -120,6 +152,7 @@ def test_persists_entries_and_refinements(self) -> None: self.assertEqual(reloaded.get("skill", skill.id).version, 1) self.assertEqual(reloaded.get("skill", skill.id).arguments["failure_log"]["type"], "string") self.assertEqual(reloaded.get("subagent", subagent.id).metadata["max_turns"], 3) + self.assertEqual(reloaded.get("swarm", swarm.id).arguments["dag"], SWARM_DAG) self.assertEqual(reloaded.refinements[0].id, event.id) self.assertIn("Prefer focused patches", reloaded.overview()) self.assertIn( @@ -133,6 +166,94 @@ def test_persists_entries_and_refinements(self) -> None: self.assertIn("await rlm.list_subagents()", overview) self.assertIn("receiver_role='child'", overview) self.assertIn("refinements: 1", reloaded.overview()) + self.assertIn("swarm", reloaded.overview()) + self.assertIn("rlm.swarm.run", reloaded.overview()) + self.assertIn("create_swarm/update_swarm/delete_swarm", reloaded.overview()) + + def test_create_swarm_with_invalid_dag_raises_and_does_not_store(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + state = HarnessState(Path(temp_dir) / "harness_state.json") + + invalid_dags = [ + None, + "not a dag", + {"nodes": []}, + {"nodes": [{"id": "a", "subagent": "w"}, {"id": "a", "subagent": "w"}]}, + {"nodes": [{"id": "a", "subagent": "w", "depends_on": ["a"]}]}, + { + "nodes": [ + {"id": "a", "subagent": "w", "outputs": [{"name": "o", "type": "text"}]}, + {"id": "b", "subagent": "w", "inputs": [{"name": "i", "type": "json", "from": "a.o"}]}, + ] + }, + ] + for index, dag in enumerate(invalid_dags): + with self.assertRaises(ValueError, msg=f"dag #{index}") as ctx: + state.create_swarm("Broken sweep", "Should never store.", id=f"broken_{index}", dag=dag) + self.assertTrue(str(ctx.exception).strip()) + + self.assertEqual(state.list("swarm"), []) + for index in range(len(invalid_dags)): + self.assertIsNone(state.get("swarm", f"broken_{index}")) + + # An invalid dag leaves nothing on disk either. + reloaded = HarnessState(state.file_path) + self.assertEqual(reloaded.list("swarm"), []) + + def test_create_swarm_with_valid_dag_stores_arguments_dag(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + state = HarnessState(Path(temp_dir) / "harness_state.json") + + entry = state.create_swarm( + "PR review sweep", + "Sweep review across changed files.", + id="pr_sweep", + path="review", + dag=SWARM_DAG, + metadata={"owner": "kernel"}, + ) + + self.assertEqual(entry.kind, "swarm") + self.assertEqual(entry.arguments["dag"], SWARM_DAG) + self.assertEqual(state.get("swarm", "pr_sweep").arguments["dag"], SWARM_DAG) + self.assertIn(entry, state.list("swarm")) + + reloaded = HarnessState(state.file_path) + self.assertEqual(reloaded.get("swarm", "pr_sweep").arguments["dag"], SWARM_DAG) + + def test_update_swarm_without_dag_preserves_stored_dag(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + state = HarnessState(Path(temp_dir) / "harness_state.json") + entry = state.create_swarm("PR review sweep", "Sweep review.", id="pr_sweep", dag=SWARM_DAG) + stored_at_create = dict(entry.arguments["dag"]) + + updated = state.update_swarm("pr_sweep", "PR review sweep", "Sweep review across PRs.") + + self.assertEqual(updated.version, 2) + self.assertEqual(state.get("swarm", "pr_sweep").arguments["dag"], SWARM_DAG) + + replacement = { + "nodes": [ + {"id": "solo", "subagent": "worker", "outputs": [{"name": "report", "type": "text"}]} + ] + } + state.update_swarm("pr_sweep", "PR review sweep", "Sweep review across PRs.", dag=replacement) + self.assertEqual(state.get("swarm", "pr_sweep").arguments["dag"], replacement) + + with self.assertRaises(ValueError): + state.update_swarm("pr_sweep", "PR review sweep", "Bad dag.", dag={"nodes": []}) + self.assertEqual(state.get("swarm", "pr_sweep").arguments["dag"], replacement) + # The pre-update snapshot was taken before any replacement landed. + self.assertEqual(stored_at_create, SWARM_DAG) + + def test_delete_swarm_removes_the_entry(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + state = HarnessState(Path(temp_dir) / "harness_state.json") + state.create_swarm("PR review sweep", "Sweep review.", id="pr_sweep", dag=SWARM_DAG) + + self.assertTrue(state.delete_swarm("pr_sweep")) + self.assertIsNone(state.get("swarm", "pr_sweep")) + self.assertFalse(state.delete_swarm("pr_sweep")) def test_save_failure_preserves_previous_state_on_disk(self) -> None: with tempfile.TemporaryDirectory() as temp_dir: diff --git a/prime-agent-runtime/test/test_swarm_executor.py b/prime-agent-runtime/test/test_swarm_executor.py new file mode 100644 index 0000000000..ec20b540ee --- /dev/null +++ b/prime-agent-runtime/test/test_swarm_executor.py @@ -0,0 +1,1038 @@ +from __future__ import annotations + +import asyncio +import time +import unittest +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Any +from unittest.mock import patch + +import rlm as rlm_module +from rlm import swarm as swarm_module +from rlm.harness import HarnessState +from rlm.swarm import SwarmExecutor + + +def async_test(coroutine): + """Run one async test method on a fresh event loop.""" + + def wrapper(self): + return asyncio.run(coroutine(self)) + + wrapper.__name__ = coroutine.__name__ + return wrapper + + +class FakeClock: + """Injectable monotonic clock with optional per-collect advancement.""" + + def __init__(self, start: float = 1000.0, advance_per_collect: float = 0.0) -> None: + self.now = start + self.advance_per_collect = advance_per_collect + + def __call__(self) -> float: + return self.now + + def advance(self, seconds: float) -> None: + self.now += seconds + + +class SleepRecorder: + """Injectable sleep that records delays instead of waiting.""" + + def __init__(self) -> None: + self.sleeps: list[float] = [] + + async def __call__(self, seconds: float) -> None: + self.sleeps.append(seconds) + + +class FakeHost: + """Deterministic async host_request fake that routes by request type. + + Child names carry the node id as their second dash-separated part, so + node ids in these tests never contain "-". Collect outcomes are + scripted per child id first (``child_outcomes``), then per node id + (``outcomes``); a "running" outcome never settles. ``rate_limit_first`` + / ``rate_limit_forever`` make spawn admissions for a node fail with a + 429-style RuntimeError. ``gate("rlm.collect"|"rlm.delete_subagent", n)`` + suspends the n-th call of that type on an asyncio.Event so tests can + reproduce races between the control loop and stop()/resume(). + """ + + def __init__(self, clock: FakeClock | None = None) -> None: + self.calls: list[tuple[str, dict[str, Any]]] = [] + self.notices: list[dict[str, Any]] = [] + self.children: dict[str, dict[str, Any]] = {} + self.counter = 0 + self.collects = 0 + self.clock = clock + self.outcomes: dict[str, dict[str, Any]] = {} + self.child_outcomes: dict[str, dict[str, Any]] = {} + self.rate_limit_first: dict[str, int] = {} + self.rate_limit_forever: set[str] = set() + self.gates: dict[tuple[str, int], asyncio.Event] = {} + self._call_indices: dict[str, int] = {} + + def gate(self, request_type: str, call_number: int) -> asyncio.Event: + """Suspend the call_number-th collect/delete on a returned event.""" + event = asyncio.Event() + self.gates[(request_type, call_number)] = event + return event + + def calls_of(self, request_type: str) -> list[dict[str, Any]]: + return [payload for kind, payload in self.calls if kind == request_type] + + def spawn_calls(self, node_id: str) -> list[dict[str, Any]]: + return [p for p in self.calls_of("rlm.run") if p["kwargs"]["name"].split("-")[1] == node_id] + + def deleted_targets(self) -> list[str]: + return [p["target"] for p in self.calls_of("rlm.delete_subagent")] + + def notice_kinds(self) -> list[str]: + return [notice["kind"] for notice in self.notices] + + @staticmethod + def _entry( + *, + child_id: str, + name: str, + status: str, + settled: bool, + answer: str | None = None, + error: str | None = None, + ) -> dict[str, Any]: + entry: dict[str, Any] = { + "rlm_child_id": child_id, + "session_name": name, + "session_dir": f"/tmp/{child_id}", + "status": status, + "settled": settled, + "tool_use_count": 1, + "duration_ms": 5, + } + if answer is not None: + entry["answer_preview"] = answer + if error is not None: + entry["error"] = error + return entry + + async def __call__(self, request_type: str, payload: dict[str, Any] | None = None) -> dict[str, Any]: + payload = payload or {} + gate = None + if request_type in ("rlm.collect", "rlm.delete_subagent"): + index = self._call_indices.get(request_type, 0) + 1 + self._call_indices[request_type] = index + gate = self.gates.pop((request_type, index), None) + if gate is not None: + await gate.wait() + if request_type == "rlm.run": + # Count this node's prior admission calls before recording the + # current one, so rate_limit_first fails exactly the first n. + name = payload["kwargs"]["name"] + node_id = name.split("-")[1] + attempted = len( + [ + p + for kind, p in self.calls + if kind == "rlm.run" and p["kwargs"]["name"].split("-")[1] == node_id + ] + ) + self.calls.append((request_type, payload)) + limit = self.rate_limit_first.get(node_id, 0) + (999 if node_id in self.rate_limit_forever else 0) + if attempted < limit: + raise RuntimeError("spawn admission failed: 429 rate limit exceeded") + self.counter += 1 + child_id = f"child-{self.counter}" + self.children[child_id] = {"name": name, "node": node_id} + return { + "rlm_child_id": child_id, + "name": name, + "session_dir": f"/tmp/{child_id}", + "model": "fake-model", + } + self.calls.append((request_type, payload)) + if request_type == "rlm.collect": + self.collects += 1 + if self.clock is not None: + self.clock.advance(self.clock.advance_per_collect) + results = [] + for target in payload["targets"]: + child = self.children.get(target) + if child is None: + continue # deleted children vanish from collect results + outcome = ( + self.child_outcomes.get(target) + or self.outcomes.get(child["node"]) + or {"status": "done", "answer": f"answer-{child['node']}"} + ) + if outcome["status"] == "running": + results.append( + self._entry(child_id=target, name=child["name"], status="running", settled=False) + ) + continue + results.append( + self._entry( + child_id=target, + name=child["name"], + status=outcome["status"], + settled=True, + answer=outcome.get("answer"), + error=outcome.get("error"), + ) + ) + return {"results": results} + if request_type == "rlm.delete_subagent": + target = payload["target"] + child = self.children.pop(target, None) + return { + "subagent": { + "rlm_child_id": target, + "session_name": child["name"] if child else "unknown", + "session_dir": f"/tmp/{target}", + "status": "running", + }, + "outcome": "deleted", + } + if request_type == "swarm.progress": + self.notices.append(payload) + return {} + raise AssertionError(f"unexpected host request type {request_type!r}") + + +class SwarmExecutorTest(unittest.TestCase): + def setUp(self) -> None: + temp = TemporaryDirectory() + self.addCleanup(temp.cleanup) + self.harness = HarnessState(Path(temp.name) / "harness_state.json") + self.harness.create_subagent("Worker", "Do the work carefully.", id="worker") + self.clock = FakeClock() + self.host = FakeHost(clock=self.clock) + self.sleeps = SleepRecorder() + self.executor = SwarmExecutor(now=self.clock, sleep=self.sleeps, harness=self.harness) + previous_executor = swarm_module._DEFAULT_EXECUTOR + swarm_module._DEFAULT_EXECUTOR = self.executor + self.addCleanup(lambda: setattr(swarm_module, "_DEFAULT_EXECUTOR", previous_executor)) + # The fake is an async callable: patch it in directly (AsyncMock does + # not await async side effects), which preserves the patch seam. + patcher = patch.object(rlm_module, "host_request", self.host) + patcher.start() + self.addCleanup(patcher.stop) + + # -- helpers ------------------------------------------------------------- + + def store_swarm(self, dag: dict[str, Any], spec_id: str = "sw") -> None: + self.harness.create_swarm("Swarm", "Swarm content", id=spec_id, dag=dag) + + def node(self, node_id: str, **overrides: Any) -> dict[str, Any]: + base: dict[str, Any] = {"id": node_id, "subagent": "worker"} + base.update(overrides) + return base + + async def start(self, spec_id: str = "sw") -> dict[str, Any]: + return await rlm_module.rlm.swarm.run(spec_id) + + async def settle(self, run_result: dict[str, Any], *, max_polls: int = 50_000) -> dict[str, Any]: + """Yield to the control loop until the run leaves the running state.""" + run_id = run_result["run_id"] + for _ in range(max_polls): + run = self.executor._runs[run_id] + if run.state != "running": + return await rlm_module.rlm.swarm.status(run_id) + await asyncio.sleep(0) + self.fail(f"run {run_id} never left the running state") + + async def wait_until(self, predicate, *, max_polls: int = 50_000) -> None: + for _ in range(max_polls): + if predicate(): + return + await asyncio.sleep(0) + self.fail("condition never became true") + + def node_status(self, status: dict[str, Any], node_id: str) -> dict[str, Any]: + return next(entry for entry in status["nodes"] if entry["id"] == node_id) + + # -- dry run --------------------------------------------------------------- + + @async_test + async def test_run_rejects_invalid_dag_and_starts_nothing(self) -> None: + # Write-time validation blocks create_swarm, so bypass it with the + # generic create to prove run() re-validates on its own. + self.harness.create("swarm", "Empty", "content", id="empty", arguments={"dag": {"nodes": []}}) + self.harness.create( + "swarm", + "Cyclic", + "content", + id="cyclic", + arguments={ + "dag": { + "nodes": [ + {"id": "a", "subagent": "worker", "depends_on": ["b"]}, + {"id": "b", "subagent": "worker", "depends_on": ["a"]}, + ] + } + }, + ) + for spec_id in ("empty", "cyclic"): + with self.assertRaisesRegex(ValueError, "swarm"): + await self.start(spec_id) + self.assertEqual(self.host.calls, []) + + @async_test + async def test_run_lists_all_missing_subagent_references(self) -> None: + self.store_swarm( + { + "run": {"failure_policy": "continue"}, + "nodes": [ + {"id": "a", "subagent": "ghost-a"}, + {"id": "b", "subagent": "ghost-b"}, + ], + } + ) + with self.assertRaises(ValueError) as ctx: + await self.start() + self.assertIn("ghost-a", str(ctx.exception)) + self.assertIn("ghost-b", str(ctx.exception)) + self.assertEqual(self.host.calls, []) + + @async_test + async def test_run_rejects_unknown_spec(self) -> None: + with self.assertRaisesRegex(ValueError, "unknown swarm spec"): + await self.start("missing-spec") + + @async_test + async def test_resolves_subagent_by_id_and_title_with_model_settings(self) -> None: + self.harness.create_subagent( + "The Worker", + "Template by title.", + id="worker-md", + metadata={"model": "pi/test-model", "thinking": "low"}, + ) + self.store_swarm( + { + "run": {"max_parallel": 2}, + "nodes": [ + {"id": "x", "subagent": "The Worker"}, + {"id": "y", "subagent": "worker-md"}, + ], + } + ) + result = await self.start() + status = await self.settle(result) + self.assertEqual(status["state"], "done") + for node_id in ("x", "y"): + spawn = self.host.spawn_calls(node_id) + self.assertEqual(len(spawn), 1, node_id) + self.assertEqual(spawn[0]["prompt"], "Template by title.") + self.assertEqual(spawn[0]["kwargs"]["model"], "pi/test-model") + self.assertEqual(spawn[0]["kwargs"]["thinking"], "low") + + @async_test + async def test_run_starts_ready_nodes_and_reports_counts(self) -> None: + self.store_swarm( + { + "run": {"max_parallel": 2}, + "nodes": [ + {"id": "a", "subagent": "worker"}, + {"id": "b", "subagent": "worker", "depends_on": ["a"]}, + {"id": "c", "subagent": "worker", "depends_on": ["b"]}, + {"id": "d", "subagent": "worker"}, + ], + } + ) + result = await self.start() + self.assertIn("run_id", result) + self.assertEqual(result["spec_id"], "sw") + self.assertEqual(result["nodes"], 4) + self.assertEqual(result["max_parallel"], 2) + self.assertEqual(result["started"], ["a", "d"]) + self.assertEqual(result["pending"], ["b", "c"]) + self.assertEqual(len(self.host.calls_of("rlm.run")), 2) + status = await self.settle(result) + self.assertEqual(status["state"], "done") + self.assertTrue(all(entry["status"] == "done" for entry in status["nodes"])) + self.assertEqual(self.host.notice_kinds(), ["finished"]) + self.assertEqual(status["usage"]["spawns"], 4) + self.assertEqual(status["usage"]["settled"], 4) + + # -- propagation and binding ------------------------------------------------ + + @async_test + async def test_propagation_binds_answer_into_prompt(self) -> None: + self.host.outcomes["a"] = {"status": "done", "answer": "ANSWER-A"} + self.store_swarm( + { + "run": {"failure_policy": "continue"}, + "nodes": [ + {"id": "a", "subagent": "worker", "outputs": [{"name": "out", "type": "text"}]}, + { + "id": "b", + "subagent": {"prompt": "Summarize: {draft}"}, + "depends_on": ["a"], + "inputs": [{"name": "draft", "type": "text", "from": "a.out"}], + }, + ], + } + ) + result = await self.start() + self.assertEqual(result["started"], ["a"]) + status = await self.settle(result) + self.assertEqual(status["state"], "done") + prompts = [call["prompt"] for call in self.host.spawn_calls("b")] + self.assertEqual(prompts, ["Summarize: ANSWER-A"]) + self.assertEqual(self.node_status(status, "b")["answer_preview"], "answer-b") + + @async_test + async def test_json_input_prefers_fenced_block(self) -> None: + self.host.outcomes["a"] = { + "status": "done", + "answer": 'Verdict text.\n```json\n{"result": {"x": 1}}\n```', + } + self.store_swarm( + { + "run": {"failure_policy": "continue"}, + "nodes": [ + {"id": "a", "subagent": "worker", "outputs": [{"name": "result", "type": "json"}]}, + { + "id": "b", + "subagent": {"prompt": "Process {data}."}, + "depends_on": ["a"], + "inputs": [{"name": "data", "type": "json", "from": "a.result"}], + }, + ], + } + ) + result = await self.start() + status = await self.settle(result) + self.assertEqual(status["state"], "done") + prompts = [call["prompt"] for call in self.host.spawn_calls("b")] + self.assertEqual(prompts, ['Process {"x": 1}.']) + + @async_test + async def test_json_input_falls_back_to_whole_text(self) -> None: + self.host.outcomes["a"] = {"status": "done", "answer": '{"result": 7}'} + self.store_swarm( + { + "run": {"failure_policy": "continue"}, + "nodes": [ + {"id": "a", "subagent": "worker", "outputs": [{"name": "result", "type": "json"}]}, + { + "id": "b", + "subagent": {"prompt": "Process {data}."}, + "depends_on": ["a"], + "inputs": [{"name": "data", "type": "json", "from": "a.result"}], + }, + ], + } + ) + result = await self.start() + status = await self.settle(result) + self.assertEqual(status["state"], "done") + self.assertEqual([call["prompt"] for call in self.host.spawn_calls("b")], ["Process 7."]) + + @async_test + async def test_bad_json_input_fails_node_without_spawning(self) -> None: + self.host.outcomes["a"] = {"status": "done", "answer": "not json at all"} + self.store_swarm( + { + "run": {"failure_policy": "continue"}, + "nodes": [ + {"id": "a", "subagent": "worker", "outputs": [{"name": "result", "type": "json"}]}, + { + "id": "b", + "subagent": {"prompt": "Process {data}."}, + "depends_on": ["a"], + "inputs": [{"name": "data", "type": "json", "from": "a.result"}], + }, + {"id": "c", "subagent": "worker"}, + ], + } + ) + result = await self.start() + status = await self.settle(result) + # b failed at binding (continue policy): never spawned, marked error; + # c still ran and finished, so the run completes but reports failed. + self.assertEqual(self.host.spawn_calls("b"), []) + b = self.node_status(status, "b") + self.assertEqual(b["status"], "error") + self.assertIn("no JSON object containing output", b["error"]) + self.assertEqual(self.node_status(status, "c")["status"], "done") + self.assertEqual(status["state"], "failed") + + @async_test + async def test_unplaced_inputs_are_appended(self) -> None: + self.host.outcomes["a"] = {"status": "done", "answer": "ANSWER-A"} + self.store_swarm( + { + "run": {"failure_policy": "continue"}, + "nodes": [ + {"id": "a", "subagent": "worker", "outputs": [{"name": "out", "type": "text"}]}, + { + "id": "b", + "subagent": {"prompt": "No placeholders here."}, + "depends_on": ["a"], + "inputs": [{"name": "draft", "type": "text", "from": "a.out"}], + }, + ], + } + ) + result = await self.start() + status = await self.settle(result) + self.assertEqual(status["state"], "done") + self.assertEqual( + [call["prompt"] for call in self.host.spawn_calls("b")], + ["No placeholders here.\n\n## Inputs\n- draft: ANSWER-A\n"], + ) + + @async_test + async def test_two_parent_fan_in_binds_both_answers(self) -> None: + # Both parents settle in one collect batch; the fan-in node must + # bind both captured answers into a single prompt. + self.host.outcomes["a"] = {"status": "done", "answer": "ANSWER-A"} + self.host.outcomes["b"] = {"status": "done", "answer": "ANSWER-B"} + self.store_swarm( + { + "run": {"failure_policy": "continue"}, + "nodes": [ + {"id": "a", "subagent": "worker", "outputs": [{"name": "out", "type": "text"}]}, + {"id": "b", "subagent": "worker", "outputs": [{"name": "out", "type": "text"}]}, + { + "id": "c", + "subagent": {"prompt": "Combine {left} and {right}."}, + "depends_on": ["a", "b"], + "inputs": [ + {"name": "left", "type": "text", "from": "a.out"}, + {"name": "right", "type": "text", "from": "b.out"}, + ], + }, + ], + } + ) + result = await self.start() + self.assertEqual(result["started"], ["a", "b"]) + status = await self.settle(result) + self.assertEqual(status["state"], "done") + self.assertEqual( + [call["prompt"] for call in self.host.spawn_calls("c")], + ["Combine ANSWER-A and ANSWER-B."], + ) + + @async_test + async def test_answer_capture_cap_slices_previews(self) -> None: + long_answer = "x" * 300 + self.host.outcomes["a"] = {"status": "done", "answer": long_answer} + self.store_swarm( + { + "nodes": [{"id": "a", "subagent": "worker"}], + } + ) + result = await self.start() + status = await self.settle(result) + self.assertEqual(status["state"], "done") + captured = self.node_status(status, "a")["answer_preview"] + self.assertEqual(len(captured), 200) + self.assertEqual(captured, long_answer[:200]) + + # -- foreach ---------------------------------------------------------------- + + def store_fan_swarm(self, *, policy: str) -> None: + self.host.outcomes["src"] = {"status": "done", "answer": '{"items": ["a", "b", "c", "d", "e"]}'} + self.store_swarm( + { + "run": {"failure_policy": policy, "max_parallel": 8}, + "nodes": [ + {"id": "src", "subagent": "worker", "outputs": [{"name": "items", "type": "json"}]}, + { + "id": "fan", + "subagent": "worker", + "depends_on": ["src"], + "inputs": [{"name": "items", "type": "json", "from": "src.items"}], + "foreach": {"over": "items", "max": 5}, + }, + ], + } + ) + + @async_test + async def test_foreach_expands_clamped_instances(self) -> None: + self.host.outcomes["src"] = { + "status": "done", + "answer": 'Here.\n```json\n{"items": [1, 2, 3, 4, 5]}\n```', + } + self.store_swarm( + { + "run": {"failure_policy": "continue", "max_parallel": 8}, + "nodes": [ + {"id": "src", "subagent": "worker", "outputs": [{"name": "items", "type": "json"}]}, + { + "id": "fan", + "subagent": {"prompt": "Expand item {items}."}, + "depends_on": ["src"], + "inputs": [{"name": "items", "type": "json", "from": "src.items"}], + "foreach": {"over": "items", "max": 3}, + }, + ], + } + ) + result = await self.start() + status = await self.settle(result) + self.assertEqual(status["state"], "done") + fan = self.node_status(status, "fan") + self.assertEqual(fan["status"], "done") + # 5 items clamped to foreach.max 3; all instances settle -> node done. + self.assertEqual(len(fan["instances"]), 3) + self.assertEqual( + sorted(call["prompt"] for call in self.host.spawn_calls("fan")), + ["Expand item 1.", "Expand item 2.", "Expand item 3."], + ) + + @async_test + async def test_foreach_zero_items_marks_node_done(self) -> None: + self.host.outcomes["src"] = {"status": "done", "answer": '{"items": []}'} + self.store_swarm( + { + "run": {"failure_policy": "continue"}, + "nodes": [ + {"id": "src", "subagent": "worker", "outputs": [{"name": "items", "type": "json"}]}, + { + "id": "fan", + "subagent": "worker", + "depends_on": ["src"], + "inputs": [{"name": "items", "type": "json", "from": "src.items"}], + "foreach": {"over": "items", "max": 4}, + }, + ], + } + ) + result = await self.start() + status = await self.settle(result) + self.assertEqual(status["state"], "done") + self.assertEqual(self.node_status(status, "fan")["status"], "done") + self.assertEqual(self.host.spawn_calls("fan"), []) + + @async_test + async def test_foreach_mixed_instances_escalate_pauses(self) -> None: + self.store_fan_swarm(policy="escalate") + result = await self.start() + # src is child-1; the five fan instances are child-2..child-6. + # Instances 0-1 fail permanently; 2-4 succeed in the same collect + # batch. The node must reach error and the policy must apply even + # though the failures did not settle last. + self.host.child_outcomes["child-2"] = {"status": "error", "error": "boom-1"} + self.host.child_outcomes["child-3"] = {"status": "error", "error": "boom-2"} + status = await self.settle(result) + self.assertEqual(status["state"], "paused") + self.assertEqual(self.host.notice_kinds(), ["paused"]) + fan = self.node_status(status, "fan") + self.assertEqual(fan["status"], "error") + self.assertEqual([i["status"] for i in fan["instances"]], ["error", "error", "done", "done", "done"]) + + @async_test + async def test_foreach_mixed_instances_fail_fast_cancels_siblings(self) -> None: + self.store_fan_swarm(policy="fail_fast") + result = await self.start() + # instance 0 (child-2) fails first while its siblings are still in + # flight: fail_fast must cancel those siblings now, not after they + # settle. + self.host.child_outcomes["child-2"] = {"status": "error", "error": "boom"} + for child_id in ("child-3", "child-4", "child-5", "child-6"): + self.host.child_outcomes[child_id] = {"status": "running"} + status = await self.settle(result) + self.assertEqual(status["state"], "failed") + self.assertEqual(self.host.notice_kinds(), ["failed"]) + fan = self.node_status(status, "fan") + self.assertEqual(fan["status"], "error") + self.assertEqual( + [i["status"] for i in fan["instances"]], + ["error", "cancelled", "cancelled", "cancelled", "cancelled"], + ) + self.assertEqual(len(self.host.deleted_targets()), 4) + + @async_test + async def test_foreach_mixed_instances_continue_finishes_with_node_error(self) -> None: + self.store_fan_swarm(policy="continue") + result = await self.start() + self.host.child_outcomes["child-2"] = {"status": "error", "error": "boom-1"} + self.host.child_outcomes["child-3"] = {"status": "error", "error": "boom-2"} + status = await self.settle(result) + # the node fails on the first permanent instance failure, the rest + # still settle, and the run finishes with the node error recorded + self.assertEqual(status["state"], "failed") + self.assertEqual(self.host.notice_kinds(), ["failed"]) + fan = self.node_status(status, "fan") + self.assertEqual(fan["status"], "error") + self.assertEqual([i["status"] for i in fan["instances"]], ["error", "error", "done", "done", "done"]) + self.assertEqual(self.host.deleted_targets(), []) + + # -- failure policies --------------------------------------------------------- + + @async_test + async def test_fail_fast_cancels_running_children(self) -> None: + self.host.outcomes["a"] = {"status": "error", "error": "boom"} + self.host.outcomes["b"] = {"status": "running"} + self.host.outcomes["c"] = {"status": "running"} + self.store_swarm( + { + "run": {"failure_policy": "continue", "max_parallel": 8}, + "nodes": [ + {"id": "a", "subagent": "worker", "failure_policy": "fail_fast"}, + {"id": "b", "subagent": "worker"}, + {"id": "c", "subagent": "worker"}, + ], + } + ) + result = await self.start() + status = await self.settle(result) + self.assertEqual(status["state"], "failed") + self.assertEqual(self.node_status(status, "a")["status"], "error") + self.assertEqual(self.node_status(status, "b")["status"], "cancelled") + self.assertEqual(self.node_status(status, "c")["status"], "cancelled") + # delete_subagent cascaded to both running children + self.assertEqual(len(self.host.deleted_targets()), 2) + self.assertEqual(self.host.notice_kinds(), ["failed"]) + + @async_test + async def test_continue_policy_finishes_remaining_nodes(self) -> None: + self.host.outcomes["a"] = {"status": "error", "error": "boom"} + self.store_swarm( + { + "run": {"failure_policy": "continue"}, + "nodes": [ + {"id": "a", "subagent": "worker"}, + {"id": "b", "subagent": "worker", "depends_on": ["a"]}, + {"id": "c", "subagent": "worker"}, + ], + } + ) + result = await self.start() + status = await self.settle(result) + # a failed; b (depends_on a, no data edge) still ran and finished; c ran. + self.assertEqual(self.node_status(status, "a")["status"], "error") + self.assertEqual(self.node_status(status, "b")["status"], "done") + self.assertEqual(self.node_status(status, "c")["status"], "done") + self.assertEqual(self.host.deleted_targets(), []) + self.assertEqual(status["state"], "failed") + self.assertEqual(self.host.notice_kinds(), ["failed"]) + + @async_test + async def test_escalate_pauses_notifies_and_resume_continues(self) -> None: + self.host.outcomes["a"] = {"status": "error", "error": "boom"} + self.store_swarm( + { + "nodes": [ + {"id": "a", "subagent": "worker"}, + {"id": "b", "subagent": "worker", "depends_on": ["a"]}, + ], + } + ) + result = await self.start() + paused = await self.settle(result) + self.assertEqual(paused["state"], "paused") + self.assertEqual(self.node_status(paused, "a")["status"], "error") + self.assertEqual(self.node_status(paused, "b")["status"], "pending") + self.assertEqual(self.host.spawn_calls("b"), []) + self.assertIn("paused", self.host.notice_kinds()) + # resume() restarts the run: b is ready (a is terminal) and finishes. + resumed = await rlm_module.rlm.swarm.resume(result["run_id"]) + self.assertEqual(resumed["state"], "running") + final = await self.settle(result) + self.assertEqual(final["state"], "failed") # a errored, b done + self.assertEqual(self.node_status(final, "b")["status"], "done") + self.assertEqual(self.host.notice_kinds(), ["paused", "failed"]) + # a second resume on the finished run must refuse + with self.assertRaisesRegex(ValueError, "not paused"): + await rlm_module.rlm.swarm.resume(result["run_id"]) + + @async_test + async def test_resume_defers_rate_limited_admissions(self) -> None: + self.host.outcomes["a"] = {"status": "error", "error": "boom"} + self.host.rate_limit_first["b"] = 2 + self.store_swarm( + { + "nodes": [ + {"id": "a", "subagent": "worker"}, + {"id": "b", "subagent": "worker", "depends_on": ["a"]}, + ], + } + ) + result = await self.start() + paused = await self.settle(result) + self.assertEqual(paused["state"], "paused") + resumed = await rlm_module.rlm.swarm.resume(result["run_id"]) + # resume() must not sleep in the calling turn: the rate-limited + # admission defers to the control loop's backoff, exactly like run() + self.assertEqual(resumed["started"], []) + self.assertEqual(self.sleeps.sleeps, []) + final = await self.settle(result) + self.assertEqual(final["state"], "failed") # a errored, b done + self.assertEqual(self.node_status(final, "b")["status"], "done") + # b needed 3 admissions total: one deferred at resume, then one + # rate-limited retry and one success inside the loop's backoff + self.assertEqual(len(self.host.spawn_calls("b")), 3) + self.assertEqual(self.sleeps.sleeps, [1.0]) + + # -- retries -------------------------------------------------------------------- + + @async_test + async def test_retries_respawn_until_attempts_exhausted(self) -> None: + self.host.outcomes["a"] = {"status": "error", "error": "boom"} + self.store_swarm( + { + "run": {"failure_policy": "continue"}, + "nodes": [{"id": "a", "subagent": "worker", "retries": 2}], + } + ) + result = await self.start() + status = await self.settle(result) + # retries=2 -> 3 admissions total, then the policy applies + self.assertEqual(len(self.host.spawn_calls("a")), 3) + self.assertEqual(self.node_status(status, "a")["attempts"], 3) + self.assertEqual(self.node_status(status, "a")["status"], "error") + self.assertEqual(status["state"], "failed") + + # -- budgets ---------------------------------------------------------------------- + + @async_test + async def test_node_budget_marks_attempt_failed(self) -> None: + # The fake clock advances 2s per collect, so the node's 1000ms budget + # (admission to settlement) is exceeded when the child settles. + self.clock.advance_per_collect = 2.0 + self.store_swarm( + { + "run": {"failure_policy": "continue"}, + "nodes": [ + {"id": "a", "subagent": "worker", "budget_ms": 1000}, + {"id": "b", "subagent": "worker", "depends_on": ["a"]}, + ], + } + ) + result = await self.start() + status = await self.settle(result) + a = self.node_status(status, "a") + self.assertEqual(a["status"], "error") + self.assertIn("budget", a["error"]) + # budget failures do not retry: exactly one admission for a + self.assertEqual(len(self.host.spawn_calls("a")), 1) + # b (depends_on a, no data edge) still ran and finished + self.assertEqual(self.node_status(status, "b")["status"], "done") + self.assertEqual(status["state"], "failed") + + @async_test + async def test_run_budget_pauses_and_notifies_then_resume_completes(self) -> None: + self.clock.advance_per_collect = 2.0 + self.store_swarm( + { + "run": {"failure_policy": "continue", "budget_ms": 1500}, + "nodes": [ + {"id": "a", "subagent": "worker"}, + {"id": "b", "subagent": "worker", "depends_on": ["a"]}, + ], + } + ) + result = await self.start() + paused = await self.settle(result) + self.assertEqual(paused["state"], "paused") + self.assertIn("budget_exceeded", self.host.notice_kinds()) + self.assertEqual(self.host.spawn_calls("b"), []) + # resume continues despite the spent budget (reported once per run) + await rlm_module.rlm.swarm.resume(result["run_id"]) + final = await self.settle(result) + self.assertEqual(final["state"], "done") + self.assertEqual(self.node_status(final, "b")["status"], "done") + self.assertEqual(self.host.notice_kinds(), ["budget_exceeded", "finished"]) + + # -- stop and rate limits ------------------------------------------------------------- + + @async_test + async def test_stop_cancels_children_and_pending_nodes(self) -> None: + self.host.outcomes["a"] = {"status": "running"} + self.store_swarm( + { + "nodes": [ + {"id": "a", "subagent": "worker"}, + {"id": "b", "subagent": "worker", "depends_on": ["a"]}, + ], + } + ) + result = await self.start() + self.assertEqual(result["started"], ["a"]) + await self.wait_until(lambda: self.host.collects >= 1) + stopped = await rlm_module.rlm.swarm.stop(result["run_id"]) + self.assertEqual(stopped["run_id"], result["run_id"]) + self.assertEqual(stopped["state"], "stopped") + self.assertEqual(stopped["cancelled"], ["a", "b"]) + self.assertEqual(self.host.deleted_targets(), ["child-1"]) + status = await rlm_module.rlm.swarm.status(result["run_id"]) + self.assertEqual(status["state"], "stopped") + self.assertEqual(self.node_status(status, "a")["status"], "cancelled") + self.assertEqual(self.node_status(status, "b")["status"], "cancelled") + # repeated stop is idempotent: same result, no duplicate ledger event + again = await rlm_module.rlm.swarm.stop(result["run_id"]) + self.assertEqual(again, {"run_id": result["run_id"], "state": "stopped", "cancelled": []}) + run_stopped_events = [e for e in self.executor._runs[result["run_id"]].events if e["kind"] == "run_stopped"] + self.assertEqual(len(run_stopped_events), 1) + + @async_test + async def test_stop_window_admits_no_new_spawns_and_never_finalizes(self) -> None: + # max_parallel 2 with three ready nodes: c waits for a slot. The + # loop's first collect and stop()'s first delete are gated so the + # stop window opens mid-collect with free-able slots and a ready + # pending node: the loop must neither admit c nor finalize the run. + self.store_swarm( + { + "run": {"max_parallel": 2}, + "nodes": [ + {"id": "a", "subagent": "worker"}, + {"id": "b", "subagent": "worker"}, + {"id": "c", "subagent": "worker"}, + ], + } + ) + collect_gate = self.host.gate("rlm.collect", 1) + delete_gate = self.host.gate("rlm.delete_subagent", 1) + result = await self.start() + self.assertEqual(result["started"], ["a", "b"]) # c waits for a slot + run = self.executor._runs[result["run_id"]] + await asyncio.sleep(0) # the loop reaches collect#1 and suspends + stop_task = asyncio.ensure_future(rlm_module.rlm.swarm.stop(result["run_id"])) + for _ in range(1000): + if run.state == "stopping": + break + await asyncio.sleep(0) + self.assertEqual(run.state, "stopping") + # release the collect: the loop settles a and b inside the stop window + collect_gate.set() + await run.task # the loop must exit on "stopping" without admitting c + delete_gate.set() + stopped = await stop_task + self.assertEqual(stopped["state"], "stopped") + status = await rlm_module.rlm.swarm.status(result["run_id"]) + self.assertEqual(status["state"], "stopped") + # never finalized: no finished notice, and c was never admitted + self.assertEqual(self.host.notice_kinds(), []) + self.assertEqual(len(self.host.calls_of("rlm.run")), 2) + + @async_test + async def test_rate_limit_at_admission_defers_to_backoff(self) -> None: + # First two admissions for a fail with a 429: one at run() admission + # (deferred, no sleep), one inside the loop (backoff sleep), then success. + self.host.rate_limit_first["a"] = 2 + self.store_swarm({"nodes": [{"id": "a", "subagent": "worker"}]}) + result = await self.start() + self.assertEqual(result["started"], []) + status = await self.settle(result) + self.assertEqual(status["state"], "done") + # three host admissions in total: one deferred at run(), then one + # rate-limited retry inside the loop's backoff, then success + self.assertEqual(len(self.host.spawn_calls("a")), 3) + self.assertEqual(self.node_status(status, "a")["attempts"], 2) + self.assertEqual(self.sleeps.sleeps, [1.0]) + + @async_test + async def test_rate_limit_backoff_exhaustion_fails_node(self) -> None: + self.host.rate_limit_forever.add("b") + self.store_swarm( + { + "run": {"failure_policy": "continue"}, + "nodes": [ + {"id": "a", "subagent": "worker"}, + {"id": "b", "subagent": "worker", "depends_on": ["a"]}, + ], + } + ) + result = await self.start() + status = await self.settle(result) + # 5 admission attempts with doubling backoff capped at 60s, then error + self.assertEqual(len(self.host.spawn_calls("b")), 5) + self.assertEqual(self.sleeps.sleeps, [1.0, 2.0, 4.0, 8.0]) + b = self.node_status(status, "b") + self.assertEqual(b["status"], "error") + self.assertIn("spawn admission failed", b["error"]) + self.assertEqual(status["state"], "failed") + + # -- scale ----------------------------------------------------------------------------- + + @async_test + async def test_scale_chain_100_completes(self) -> None: + nodes = [{"id": "n0", "subagent": {"prompt": "step"}}] + for index in range(1, 100): + nodes.append({"id": f"n{index}", "subagent": {"prompt": "step"}, "depends_on": [f"n{index - 1}"]}) + self.store_swarm({"run": {"max_parallel": 8}, "nodes": nodes}) + started = time.monotonic() + result = await self.start() + status = await self.settle(result) + elapsed = time.monotonic() - started + self.assertEqual(status["state"], "done") + self.assertEqual(len(status["nodes"]), 100) + self.assertTrue(all(entry["status"] == "done" for entry in status["nodes"])) + self.assertLess(elapsed, 10.0) + + @async_test + async def test_scale_fan_1000_completes(self) -> None: + nodes = [{"id": "n0", "subagent": {"prompt": "step"}}] + for index in range(1, 1000): + nodes.append({"id": f"n{index}", "subagent": {"prompt": "step"}, "depends_on": ["n0"]}) + self.store_swarm({"run": {"max_parallel": 64}, "nodes": nodes}) + started = time.monotonic() + result = await self.start() + status = await self.settle(result) + elapsed = time.monotonic() - started + self.assertEqual(status["state"], "done") + self.assertEqual(len(status["nodes"]), 1000) + self.assertTrue(all(entry["status"] == "done" for entry in status["nodes"])) + self.assertLess(elapsed, 10.0) + + # -- status, ledger, residents ---------------------------------------------------------- + + @async_test + async def test_status_marks_events_delivered_and_unknown_run_raises(self) -> None: + self.store_swarm({"nodes": [{"id": "a", "subagent": "worker"}]}) + result = await self.start() + run = self.executor._runs[result["run_id"]] + await self.wait_until(lambda: run.state != "running") + # Before any status() read: answers are arrived, the milestone is shown. + stages = {event["kind"]: event["stage"] for event in run.events} + self.assertEqual(stages["answer_captured"], "arrived") + self.assertEqual(stages["milestone"], "shown") + status = await rlm_module.rlm.swarm.status(result["run_id"]) + self.assertTrue(all(event["stage"] == "delivered" for event in status["events"])) + self.assertTrue(all(event["stage"] == "delivered" for event in run.events)) + self.assertLessEqual(len(status["events"]), 50) + # exactly one notice for the one milestone + self.assertEqual(self.host.notice_kinds(), ["finished"]) + for call in ("status", "stop", "resume"): + with self.assertRaisesRegex(ValueError, "unknown swarm run"): + await getattr(rlm_module.rlm.swarm, call)("no-such-run") + + @async_test + async def test_resident_node_spawns_stays_alive_and_stops(self) -> None: + self.host.outcomes["watcher"] = {"status": "running"} + self.store_swarm( + { + "nodes": [ + {"id": "t", "subagent": "worker"}, + {"id": "watcher", "subagent": "worker", "lifecycle": "resident"}, + ], + } + ) + result = await self.start() + self.assertEqual(result["started"], ["t", "watcher"]) + status = await self.settle(result) + # The task node settled; the run reports done while the resident stays + # alive under the supervisor (V1 wake source: its own prompt/tooling). + self.assertEqual(status["state"], "done") + self.assertEqual(self.node_status(status, "t")["status"], "done") + resident = self.node_status(status, "watcher") + self.assertEqual(resident["status"], "running") + self.assertEqual(resident["lifecycle"], "resident") + self.assertIn("finished", self.host.notice_kinds()) + # stop() tears the resident child down. + stopped = await rlm_module.rlm.swarm.stop(result["run_id"]) + self.assertEqual(stopped["cancelled"], ["watcher"]) + self.assertEqual(self.host.deleted_targets(), ["child-2"]) + self.assertEqual((await rlm_module.rlm.swarm.status(result["run_id"]))["state"], "stopped") + + +if __name__ == "__main__": + unittest.main() diff --git a/prime-agent-runtime/test/test_swarm_spec.py b/prime-agent-runtime/test/test_swarm_spec.py new file mode 100644 index 0000000000..cbfc00516b --- /dev/null +++ b/prime-agent-runtime/test/test_swarm_spec.py @@ -0,0 +1,642 @@ +from __future__ import annotations + +import unittest +from typing import Any + +from rlm.swarm import canonicalize_swarm_spec, topological_order, validate_swarm_spec + + +def node(node_id: str, **overrides: Any) -> dict[str, Any]: + base: dict[str, Any] = {"id": node_id, "subagent": "worker"} + base.update(overrides) + return base + + +def valid_dag() -> dict[str, Any]: + return { + "run": {"budget_ms": 600_000, "failure_policy": "continue", "max_parallel": 4}, + "nodes": [ + { + "id": "collect", + "subagent": "researcher", + "outputs": [{"name": "findings", "type": "text"}], + }, + { + "id": "fan-out", + "subagent": {"prompt": "Expand each item.", "name": "expander", "model": "m1", "thinking": "low"}, + "depends_on": ["collect"], + "inputs": [{"name": "items", "type": "text", "from": "collect.findings"}], + }, + { + "id": "review", + "subagent": {"prompt": "Review the fan-out."}, + "depends_on": ["collect", "fan-out"], + "inputs": [{"name": "draft", "type": "text", "from": "collect.findings"}], + "retries": 2, + "budget_ms": 100_000, + "failure_policy": "fail_fast", + }, + ], + } + + +class ValidateSwarmSpecTest(unittest.TestCase): + def test_valid_spec_has_no_errors(self) -> None: + self.assertEqual(validate_swarm_spec(valid_dag()), []) + + def test_dag_must_be_an_object(self) -> None: + for bad in (None, [], "nodes", 42): + errors = validate_swarm_spec(bad) + self.assertEqual(len(errors), 1) + self.assertIn("swarm dag must be a JSON object", errors[0]) + + def test_nodes_required_and_must_be_a_list(self) -> None: + self.assertEqual( + validate_swarm_spec({"nodes": "nope"}), + ["swarm dag requires a nodes list"], + ) + self.assertEqual( + validate_swarm_spec({"run": "bad", "nodes": "nope"}), + ["run must be an object", "swarm dag requires a nodes list"], + ) + + def test_node_cap(self) -> None: + at_cap = {"nodes": [node(f"n{i}") for i in range(1024)]} + self.assertEqual(validate_swarm_spec(at_cap), []) + over_cap = {"nodes": [node(f"n{i}") for i in range(1025)]} + errors = validate_swarm_spec(over_cap) + self.assertEqual(len(errors), 1) + self.assertIn("between 1 and 1024 nodes", errors[0]) + + def test_node_ids(self) -> None: + for good in ("a", "node-1", "1st-node", "a" * 64): + self.assertEqual(validate_swarm_spec({"nodes": [node(good)]}), [], good) + for bad in ("-abc", "ABC", "a_b", "a.b", "a" * 65): + errors = validate_swarm_spec({"nodes": [{"id": bad, "subagent": "w"}]}) + self.assertEqual(len(errors), 1, bad) + self.assertIn("id must match", errors[0]) + for bad in ("", None, 5): + errors = validate_swarm_spec({"nodes": [{"id": bad, "subagent": "w"}]}) + self.assertEqual(errors, ["nodes[0] requires a non-empty id"], repr(bad)) + + def test_duplicate_node_ids(self) -> None: + errors = validate_swarm_spec({"nodes": [node("dup"), node("dup")]}) + self.assertEqual(len(errors), 1) + self.assertIn("duplicates node id 'dup'", errors[0]) + + def test_subagent_forms(self) -> None: + by_ref = {"nodes": [node("a", subagent="reviewer")]} + self.assertEqual(validate_swarm_spec(by_ref), []) + inline = {"nodes": [node("a", subagent={"prompt": "Do work."})]} + self.assertEqual(validate_swarm_spec(inline), []) + inline_full = { + "nodes": [ + node("a", subagent={"prompt": "Do work.", "name": "w", "model": "m", "thinking": "high"}) + ] + } + self.assertEqual(validate_swarm_spec(inline_full), []) + + missing = {"nodes": [{"id": "a"}]} + errors = validate_swarm_spec(missing) + self.assertEqual(len(errors), 1) + self.assertIn("requires a subagent", errors[0]) + + empty_ref = {"nodes": [node("a", subagent="")]} + errors = validate_swarm_spec(empty_ref) + self.assertEqual(len(errors), 1) + self.assertIn("requires a subagent", errors[0]) + + empty_prompt = {"nodes": [node("a", subagent={"prompt": ""})]} + errors = validate_swarm_spec(empty_prompt) + self.assertEqual(len(errors), 1) + self.assertIn("requires a non-empty prompt", errors[0]) + + bad_name = {"nodes": [node("a", subagent={"prompt": "p", "model": 5})]} + errors = validate_swarm_spec(bad_name) + self.assertEqual(len(errors), 1) + self.assertIn("model must be a non-empty string", errors[0]) + + bad_thinking = {"nodes": [node("a", subagent={"prompt": "p", "thinking": ""})]} + errors = validate_swarm_spec(bad_thinking) + self.assertEqual(len(errors), 1) + self.assertIn("thinking must be a non-empty string", errors[0]) + + def test_lifecycle(self) -> None: + for good in ("task", "resident"): + self.assertEqual(validate_swarm_spec({"nodes": [node("a", lifecycle=good)]}), []) + errors = validate_swarm_spec({"nodes": [node("a", lifecycle="daemon")]}) + self.assertEqual(len(errors), 1) + self.assertIn("lifecycle must be 'task' or 'resident'", errors[0]) + + def test_run_budget_and_node_budget(self) -> None: + ok = { + "run": {"budget_ms": 1000}, + "nodes": [node("a", budget_ms=1000)], + } + self.assertEqual(validate_swarm_spec(ok), []) + + over = { + "run": {"budget_ms": 1000}, + "nodes": [node("a", budget_ms=1001)], + } + errors = validate_swarm_spec(over) + self.assertEqual(len(errors), 1) + self.assertIn("exceeds the run budget_ms", errors[0]) + + for bad in (0, -5, 1.5, "10", True): + errors = validate_swarm_spec({"run": {"budget_ms": bad}, "nodes": [node("a")]}) + self.assertEqual(errors, ["run budget_ms must be a positive integer"], bad) + errors = validate_swarm_spec({"nodes": [node("a", budget_ms=bad)]}) + self.assertEqual(errors, ["node a budget_ms must be a positive integer"], bad) + + # No run budget set: any positive node budget is fine. + self.assertEqual(validate_swarm_spec({"nodes": [node("a", budget_ms=999_999)]}), []) + + def test_run_budget_invalid(self) -> None: + errors = validate_swarm_spec({"run": "bad", "nodes": [node("a")]}) + self.assertEqual(errors, ["run must be an object"]) + + def test_run_failure_policy(self) -> None: + for good in ("fail_fast", "continue", "escalate"): + self.assertEqual(validate_swarm_spec({"run": {"failure_policy": good}, "nodes": [node("a")]}), []) + errors = validate_swarm_spec({"run": {"failure_policy": "stop"}, "nodes": [node("a")]}) + self.assertEqual(len(errors), 1) + self.assertIn("run failure_policy must be one of", errors[0]) + + def test_run_max_parallel(self) -> None: + for good in (1, 8, 64): + self.assertEqual(validate_swarm_spec({"run": {"max_parallel": good}, "nodes": [node("a")]}), []) + for bad in (0, 65, -1, 1.5, "8", True): + errors = validate_swarm_spec({"run": {"max_parallel": bad}, "nodes": [node("a")]}) + self.assertEqual(errors, ["run max_parallel must be an integer between 1 and 64"], bad) + + def test_node_failure_policy(self) -> None: + for good in ("fail_fast", "continue", "escalate"): + self.assertEqual(validate_swarm_spec({"nodes": [node("a", failure_policy=good)]}), []) + errors = validate_swarm_spec({"nodes": [node("a", failure_policy="retry")]}) + self.assertEqual(len(errors), 1) + self.assertIn("node a failure_policy must be one of", errors[0]) + + def test_retries(self) -> None: + for good in (0, 5, 10): + self.assertEqual(validate_swarm_spec({"nodes": [node("a", retries=good)]}), []) + for bad in (-1, 11, 1.5, "2", True): + errors = validate_swarm_spec({"nodes": [node("a", retries=bad)]}) + self.assertEqual(errors, ["node a retries must be an integer between 0 and 10"], bad) + + def test_depends_on(self) -> None: + ok = {"nodes": [node("a"), node("b", depends_on=["a"])]} + self.assertEqual(validate_swarm_spec(ok), []) + + self_dep = {"nodes": [node("a", depends_on=["a"])]} + errors = validate_swarm_spec(self_dep) + self.assertEqual(errors, ["node a cannot depend on itself"]) + + unknown = {"nodes": [node("a", depends_on=["ghost"])]} + errors = validate_swarm_spec(unknown) + self.assertEqual(errors, ["node a depends on unknown node 'ghost'"]) + + not_list = {"nodes": [node("a", depends_on="b")]} + errors = validate_swarm_spec(not_list) + self.assertEqual(errors, ["node a depends_on must be a list of node ids"]) + + bad_entry = {"nodes": [node("a", depends_on=[5])]} + errors = validate_swarm_spec(bad_entry) + self.assertEqual(errors, ["node a depends_on entries must be non-empty node id strings"]) + + def test_data_edges(self) -> None: + ok = { + "nodes": [ + node("a", outputs=[{"name": "out", "type": "text"}]), + node("b", inputs=[{"name": "in", "type": "text", "from": "a.out"}]), + ] + } + self.assertEqual(validate_swarm_spec(ok), []) + + # A data edge implies ordering even without depends_on. + no_declared_dep = { + "nodes": [ + node("a", outputs=[{"name": "out", "type": "json"}]), + node("b", inputs=[{"name": "in", "type": "json", "from": "a.out"}]), + ] + } + self.assertEqual(validate_swarm_spec(no_declared_dep), []) + + unknown_source = { + "nodes": [node("b", inputs=[{"name": "in", "type": "text", "from": "ghost.out"}])] + } + errors = validate_swarm_spec(unknown_source) + self.assertEqual(len(errors), 1) + self.assertIn("references unknown node 'ghost'", errors[0]) + + undeclared_output = { + "nodes": [ + node("a"), + node("b", inputs=[{"name": "in", "type": "text", "from": "a.missing"}]), + ] + } + errors = validate_swarm_spec(undeclared_output) + self.assertEqual(len(errors), 1) + self.assertIn("does not declare", errors[0]) + + type_mismatch = { + "nodes": [ + node("a", outputs=[{"name": "out", "type": "text"}]), + node("b", inputs=[{"name": "in", "type": "json", "from": "a.out"}]), + ] + } + errors = validate_swarm_spec(type_mismatch) + self.assertEqual(len(errors), 1) + self.assertIn("cannot read from output", errors[0]) + self.assertIn("of type 'text'", errors[0]) + + malformed_from = { + "nodes": [ + node("a", outputs=[{"name": "out", "type": "text"}]), + node("b", inputs=[{"name": "in", "type": "text", "from": "no-dot"}]), + ] + } + errors = validate_swarm_spec(malformed_from) + self.assertEqual(len(errors), 1) + self.assertIn("requires a 'from' reference", errors[0]) + + def test_input_output_ports(self) -> None: + ok = { + "nodes": [ + node( + "a", + outputs=[{"name": "o1", "type": "text"}, {"name": "o2", "type": "json"}], + ) + ] + } + self.assertEqual(validate_swarm_spec(ok), []) + + bad_output_type = {"nodes": [node("a", outputs=[{"name": "o", "type": "yaml"}])]} + errors = validate_swarm_spec(bad_output_type) + self.assertEqual(len(errors), 1) + self.assertIn("output 'o' type must be 'text' or 'json'", errors[0]) + + dup_output = { + "nodes": [node("a", outputs=[{"name": "o", "type": "text"}, {"name": "o", "type": "json"}])] + } + errors = validate_swarm_spec(dup_output) + self.assertEqual(errors, ["node a declares duplicate output name 'o'"]) + + bad_input_type = { + "nodes": [ + node("b", outputs=[{"name": "o", "type": "text"}]), + node("a", inputs=[{"name": "i", "type": "yaml", "from": "b.o"}]), + ] + } + errors = validate_swarm_spec(bad_input_type) + self.assertEqual(errors, ["node a input 'i' type must be 'text' or 'json'"]) + + dup_input = { + "nodes": [ + node( + "a", + inputs=[ + {"name": "i", "type": "text", "from": "b.o1"}, + {"name": "i", "type": "text", "from": "b.o2"}, + ], + ), + node("b", outputs=[{"name": "o1", "type": "text"}, {"name": "o2", "type": "text"}]), + ] + } + errors = validate_swarm_spec(dup_input) + self.assertEqual(errors, ["node a declares duplicate input name 'i'"]) + + missing_name = {"nodes": [node("a", outputs=[{"type": "text"}])]} + errors = validate_swarm_spec(missing_name) + self.assertEqual(errors, ["node a outputs[0] requires a non-empty name"]) + + not_a_list = {"nodes": [node("a", outputs="nope")]} + errors = validate_swarm_spec(not_a_list) + self.assertEqual(errors, ["node a outputs must be a list"]) + not_a_list = {"nodes": [node("a", inputs="nope")]} + errors = validate_swarm_spec(not_a_list) + self.assertEqual(errors, ["node a inputs must be a list"]) + + def test_resident_rules(self) -> None: + resident_ok = {"nodes": [node("watcher", lifecycle="resident")]} + self.assertEqual(validate_swarm_spec(resident_ok), []) + + depended_on = { + "nodes": [node("watcher", lifecycle="resident"), node("task", depends_on=["watcher"])] + } + errors = validate_swarm_spec(depended_on) + self.assertEqual(errors, ["node task cannot depend on resident node 'watcher'"]) + + read_from = { + "nodes": [ + node("watcher", lifecycle="resident"), + node("task", inputs=[{"name": "i", "type": "text", "from": "watcher.o"}]), + ] + } + errors = validate_swarm_spec(read_from) + self.assertEqual(errors, ["node task input 'i' cannot read from resident node 'watcher'"]) + + declares_outputs = {"nodes": [node("watcher", lifecycle="resident", outputs=[{"name": "o", "type": "text"}])]} + errors = validate_swarm_spec(declares_outputs) + self.assertEqual(errors, ["resident node watcher cannot declare outputs"]) + + uses_foreach = { + "nodes": [ + node("src", outputs=[{"name": "items", "type": "json"}]), + node( + "watcher", + lifecycle="resident", + inputs=[{"name": "items", "type": "json", "from": "src.items"}], + foreach={"over": "items", "max": 4}, + ), + ] + } + errors = validate_swarm_spec(uses_foreach) + self.assertEqual(errors, ["resident node watcher cannot use foreach"]) + + # Empty outputs list on a resident node is fine: nothing is declared. + empty_outputs = {"nodes": [node("watcher", lifecycle="resident", outputs=[])]} + self.assertEqual(validate_swarm_spec(empty_outputs), []) + + def test_foreach(self) -> None: + ok = { + "nodes": [ + node("a", outputs=[{"name": "items", "type": "json"}]), + node( + "b", + inputs=[{"name": "items", "type": "json", "from": "a.items"}], + foreach={"over": "items", "max": 16}, + ), + ] + } + self.assertEqual(validate_swarm_spec(ok), []) + + for good_max in (1, 256): + ok_max = { + "nodes": [ + node("a", outputs=[{"name": "items", "type": "json"}]), + node( + "b", + inputs=[{"name": "items", "type": "json", "from": "a.items"}], + foreach={"over": "items", "max": good_max}, + ), + ] + } + self.assertEqual(validate_swarm_spec(ok_max), []) + + for bad_max in (0, 257, -1, 1.5, "8", True): + bad = { + "nodes": [ + node("a", outputs=[{"name": "items", "type": "json"}]), + node( + "b", + inputs=[{"name": "items", "type": "json", "from": "a.items"}], + foreach={"over": "items", "max": bad_max}, + ), + ] + } + errors = validate_swarm_spec(bad) + self.assertEqual(errors, ["node b foreach.max must be an integer between 1 and 256"], bad_max) + + wrong_port = { + "nodes": [ + node("a", outputs=[{"name": "items", "type": "json"}]), + node( + "b", + inputs=[{"name": "items", "type": "json", "from": "a.items"}], + foreach={"over": "not-an-input", "max": 4}, + ), + ] + } + errors = validate_swarm_spec(wrong_port) + self.assertEqual( + errors, ["node b foreach.over must name one of this node's inputs, got 'not-an-input'"] + ) + + text_port = { + "nodes": [ + node("a", outputs=[{"name": "draft", "type": "text"}]), + node( + "b", + inputs=[{"name": "draft", "type": "text", "from": "a.draft"}], + foreach={"over": "draft", "max": 4}, + ), + ] + } + errors = validate_swarm_spec(text_port) + self.assertEqual(errors, ["node b foreach.over input 'draft' must have type 'json'"]) + + not_object = {"nodes": [node("a", foreach=["bad"])]} + errors = validate_swarm_spec(not_object) + self.assertEqual(errors, ["node a foreach must be an object"]) + + def test_cycle_detection(self) -> None: + depends_cycle = { + "nodes": [ + node("a", depends_on=["b"]), + node("b", depends_on=["a"]), + ] + } + errors = validate_swarm_spec(depends_cycle) + self.assertEqual(errors, ["the swarm graph contains a cycle involving nodes: a, b"]) + + data_cycle = { + "nodes": [ + node("a", outputs=[{"name": "o", "type": "json"}], inputs=[ + {"name": "i", "type": "json", "from": "b.o"} + ]), + node("b", outputs=[{"name": "o", "type": "json"}], inputs=[ + {"name": "i", "type": "json", "from": "a.o"} + ]), + ] + } + errors = validate_swarm_spec(data_cycle) + self.assertEqual(len(errors), 1) + self.assertIn("contains a cycle", errors[0]) + + three_cycle = { + "nodes": [ + node("a", depends_on=["c"]), + node("b", depends_on=["a"]), + node("c", depends_on=["b"]), + ] + } + errors = validate_swarm_spec(three_cycle) + self.assertEqual(len(errors), 1) + self.assertIn("contains a cycle", errors[0]) + + def test_collects_multiple_errors(self) -> None: + dag = { + "run": {"max_parallel": 99, "failure_policy": "nope"}, + "nodes": [ + node("a", depends_on=["ghost"]), + node("b", retries=99), + node("c", failure_policy="retry"), + ], + } + errors = validate_swarm_spec(dag) + self.assertEqual( + errors, + [ + "run failure_policy must be one of ['fail_fast', 'continue', 'escalate'], got 'nope'", + "run max_parallel must be an integer between 1 and 64", + "node a depends on unknown node 'ghost'", + "node b retries must be an integer between 0 and 10", + "node c failure_policy must be one of ['fail_fast', 'continue', 'escalate'], got 'retry'", + ], + ) + + +class CanonicalizeSwarmSpecTest(unittest.TestCase): + def test_applies_defaults(self) -> None: + dag = {"nodes": [{"id": "a", "subagent": "worker"}]} + self.assertEqual( + canonicalize_swarm_spec(dag), + { + "run": {"failure_policy": "escalate", "max_parallel": 8}, + "nodes": [ + { + "id": "a", + "subagent": "worker", + "lifecycle": "task", + "retries": 0, + "failure_policy": "escalate", + } + ], + }, + ) + + def test_preserves_explicit_values(self) -> None: + dag = { + "run": {"budget_ms": 5000, "failure_policy": "continue", "max_parallel": 2}, + "nodes": [ + { + "id": "a", + "subagent": {"prompt": "Work."}, + "lifecycle": "task", + "retries": 3, + "failure_policy": "fail_fast", + "budget_ms": 4000, + "depends_on": [], + "outputs": [{"name": "o", "type": "text"}], + } + ], + } + self.assertEqual( + canonicalize_swarm_spec(dag), + { + "run": {"failure_policy": "continue", "max_parallel": 2, "budget_ms": 5000}, + "nodes": [ + { + "id": "a", + "subagent": {"prompt": "Work."}, + "lifecycle": "task", + "retries": 3, + "failure_policy": "fail_fast", + "budget_ms": 4000, + "depends_on": [], + "outputs": [{"name": "o", "type": "text"}], + } + ], + }, + ) + + def test_node_failure_policy_defaults_to_run_policy(self) -> None: + dag = { + "run": {"failure_policy": "continue"}, + "nodes": [{"id": "a", "subagent": "w"}, {"id": "b", "subagent": "w", "failure_policy": "escalate"}], + } + result = canonicalize_swarm_spec(dag) + self.assertEqual(result["nodes"][0]["failure_policy"], "continue") + self.assertEqual(result["nodes"][1]["failure_policy"], "escalate") + + def test_deduplicates_depends_on(self) -> None: + dag = { + "nodes": [ + {"id": "a", "subagent": "w"}, + {"id": "b", "subagent": "w", "depends_on": ["a", "a", "a"]}, + ] + } + result = canonicalize_swarm_spec(dag) + self.assertEqual(result["nodes"][1]["depends_on"], ["a"]) + + def test_raises_with_joined_errors_on_invalid_input(self) -> None: + with self.assertRaises(ValueError) as ctx: + canonicalize_swarm_spec({"nodes": [node("a", depends_on=["ghost"])]}) + message = str(ctx.exception) + self.assertIn("depends on unknown node", message) + + with self.assertRaises(ValueError) as ctx: + canonicalize_swarm_spec("not a dag") + self.assertIn("swarm dag must be a JSON object", str(ctx.exception)) + + def test_does_not_mutate_input(self) -> None: + dag = {"nodes": [{"id": "a", "subagent": {"prompt": "p"}, "outputs": [{"name": "o", "type": "json"}]}]} + snapshot = {"nodes": [dict(dag["nodes"][0])]} + result = canonicalize_swarm_spec(dag) + result["nodes"][0]["subagent"]["prompt"] = "mutated" + result["nodes"][0]["outputs"][0]["type"] = "text" + self.assertEqual(dag["nodes"][0]["subagent"]["prompt"], "p") + self.assertEqual(dag["nodes"][0]["outputs"][0]["type"], "json") + self.assertEqual(snapshot["nodes"][0]["id"], "a") + + +class TopologicalOrderTest(unittest.TestCase): + def test_happy_path_respects_effective_dependencies(self) -> None: + nodes = [ + node("z", inputs=[{"name": "i", "type": "text", "from": "m.o"}]), + node("a"), + node("m", outputs=[{"name": "o", "type": "text"}], depends_on=["a"]), + ] + self.assertEqual(topological_order(nodes), ["a", "m", "z"]) + + def test_chain(self) -> None: + nodes = [ + node("c", depends_on=["b"]), + node("b", depends_on=["a"]), + node("a"), + ] + self.assertEqual(topological_order(nodes), ["a", "b", "c"]) + + def test_cycle_raises(self) -> None: + nodes = [node("a", depends_on=["b"]), node("b", depends_on=["a"])] + with self.assertRaises(ValueError) as ctx: + topological_order(nodes) + self.assertIn("contains a cycle", str(ctx.exception)) + + self_cycle = [node("a", depends_on=["a"])] + with self.assertRaises(ValueError) as ctx: + topological_order(self_cycle) + self.assertIn("contains a cycle", str(ctx.exception)) + + def test_missing_dependency_raises(self) -> None: + with self.assertRaises(ValueError) as ctx: + topological_order([node("a", depends_on=["ghost"])]) + self.assertIn("depends on unknown node 'ghost'", str(ctx.exception)) + + def test_duplicate_id_raises(self) -> None: + with self.assertRaises(ValueError) as ctx: + topological_order([node("a"), node("a")]) + self.assertIn("duplicate node id 'a'", str(ctx.exception)) + + def test_malformed_nodes_raise(self) -> None: + with self.assertRaises(ValueError): + topological_order(["not an object"]) # type: ignore[list-item] + with self.assertRaises(ValueError): + topological_order([{"subagent": "w"}]) + with self.assertRaises(ValueError): + topological_order([node("a", depends_on="b")]) # type: ignore[arg-type] + with self.assertRaises(ValueError): + topological_order([node("a", inputs="b")]) # type: ignore[arg-type] + with self.assertRaises(ValueError): + topological_order([node("a", inputs=["not an object"])]) # type: ignore[list-item] + with self.assertRaises(ValueError): + topological_order([node("a", inputs=[{"name": "i", "type": "text", "from": "nodot"}])]) + + def test_stable_order_uses_input_position(self) -> None: + nodes = [node("b"), node("c"), node("a"), node("d")] + self.assertEqual(topological_order(nodes), ["b", "c", "a", "d"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tsconfig.json b/tsconfig.json index 4dfbf3d254..9ed4f2fdda 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -20,6 +20,11 @@ "@earendil-works/pi-agent-old/*": ["./packages/agent-old/src/*"] } }, - "include": ["packages/*/src/**/*", "packages/*/test/**/*", "packages/coding-agent/examples/**/*"], + "include": [ + "packages/*/src/**/*", + "packages/*/test/**/*", + "packages/coding-agent/examples/**/*", + "packages/coding-agent/scripts/**/*" + ], "exclude": ["**/dist/**"] }