diff --git a/README.md b/README.md index ff9f06c..d4c4013 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,16 @@ board to a PR — or fork it as a starting point. requires a spec, EARS acceptance criteria, and explicit `files_to_modify`. - **Escalation (opt-in)** — with a `coders` map of >1 distinct delegate, a capability failure climbs a model tier (`fast→smart→reasoning`) and blocks at the top. +- **coder.solve() board seam (ADR 0064 P2)** — on a fresh build, when the + [`coder`](https://github.com/protoLabsAI/protoAgent/tree/main/plugins/coder) plugin + is enabled AND the feature has acceptance criteria AND `coder_solve_test_cmd` (or + `local_gate_cmd`) is set, the loop dispatches through `coder.solve()`'s + execution-grounded ladder (greedy → best-of-k → tree-search) instead of a single + `delegate_to(acp)` shot — gated on the feature's acceptance tests actually PASSING + in a real candidate worktree, never an LLM judge. Composes WITH the tier ladder + above (solve() searches *within* a tier; a search that never passes escalates a + tier, or blocks, exactly like a no-diff dispatch). Missing coder/acceptance/test + command ⇒ honest degrade to the single shot — see `coder_seam.py`. - **Planning layer** — two reasoning subagents (`decompose` + `antagonist`) driven by the `decompose-project` skill: idea → outline → MADR ADRs → epics › milestones › features, hardened by an adversary, with a per-epic human gate. @@ -99,6 +109,11 @@ project_board: # With local_gate_cmd set, Max-Mode is EXECUTION-GROUNDED (ADR 0064): the winner is # picked from candidates whose gate actually PASSES; the LLM judge only breaks ties # among the passing set (or decides when no gate is set / none pass). + coder_solve: true # OPT-OUT valve for the ADR 0064 P2 seam (default on; the + # real gate below still requires the `coder` plugin + + # acceptance criteria + a test command — see "What it does"). + coder_solve_test_cmd: "pytest tests/ -q" # solve()'s verify() oracle; falls back to + # local_gate_cmd if blank, else the seam honest-degrades. # webhook_secret: "..." # set before exposing /webhook/pr publicly ``` @@ -124,6 +139,7 @@ project_board: | `store.py` | the `br`/beads wrapper — board projection + the Ready/Done invariants | | `loop.py` | the puller: `ready → worktree → coder → PR → in_review` (+ opt-in escalation) | | `worktree.py` | per-feature worktree lifecycle, scoped coder dispatch, `open_pr` | +| `coder_seam.py` | the ADR 0064 P2 seam — dispatches a build through `coder.solve()` when available, else honest-degrades | | `api.py` | the HTTP API + the `/webhook/pr` Done edge (HMAC-verified) | | `board_view.py` | the Kanban/list console view | | `retro.py` | loop-retro mining: bead attempt/outcome history → recurring failure classes (the self-improving flywheel) | diff --git a/coder_seam.py b/coder_seam.py new file mode 100644 index 0000000..f9edc0a --- /dev/null +++ b/coder_seam.py @@ -0,0 +1,331 @@ +"""The P2 board seam (ADR 0064): dispatch a feature's build through the `coder` +plugin's execution-grounded ``solve()`` ladder instead of a single +``delegate_to(acp)`` shot — greedy → best-of-k → tree-search, gated on the +feature's acceptance tests actually PASSING in a real worktree, never an LLM judge. + +**Composes** `plugins.coder.solve` (a separate, git-URL-installed plugin — imported +lazily/best-effort so this repo carries no hard dependency on it and no import-time +coupling) with THIS repo's own worktree primitives. The coder plugin never sees a +board worktree; it only supplies the deterministic ladder (`solve()`, `Budget`, +`Verdict`). Each candidate the ladder tries gets its OWN throwaway worktree — the +"independent-parallel acp attempts" `coder`'s own generator module already flags as +"the P2 path" (`plugins/coder/generate.py`) — and the winning (test-passing) +candidate is PROMOTED to the feature's canonical worktree/branch so the rest of the +drive (fixups, the pre-PR local gate, `open_pr`, the CI bounce, tier escalation) is +UNCHANGED; every other candidate is reaped. + +**Honest degrade** (ADR 0064's no-LLM-judge rule, applied at the board layer): the +dispatch decision (``should_use_solve``) requires ALL THREE — the `coder` plugin +importable (the host has it enabled), the feature's acceptance criteria present +(the Ready gate's oracle), and a configured, runnable acceptance-test command (the +actual executable verifier — `solve()` cannot run prose). Missing any of the three +⇒ the caller falls back to today's single ``delegate_to(acp)`` shot; never a silent +best-of-k/judge substitute. + +**Deferred** (see the ADR + the PR that lands this): compiling EARS acceptance +criteria into a generated test file. The simplest-correct path used here instead: +the coder is already prompted (``loop._build_prompt``) to write tests satisfying +the acceptance criteria as part of its definition of done; this module's ``verify`` +just RUNS whatever tests exist in a candidate's worktree via the configured command +and gates on its exit code — real execution, no fabricated grounding. +""" + +from __future__ import annotations + +import asyncio +import importlib +import logging +from typing import Callable + +from . import worktree + +log = logging.getLogger("protoagent.plugins.project_board") + + +class SolveExhausted(worktree.WorktreeError): + """``coder.solve()`` spent its whole generation budget against this feature's + acceptance tests and no candidate passed. A CAPABILITY failure for the CURRENT + model tier — real diffs existed, they just failed the tests, so this is NOT + "no diff" — but the loop treats it exactly like ``NoChangesError``/ + ``CoderTimeout``: escalate a configured tier ladder, or block. Never opens a PR + on an unverified best-partial (ADR 0064's honest-degrade contract).""" + + +def _import_solve(): + """Best-effort import of the `coder` plugin's solve library. Returns the module, + or ``None`` — `coder` is a separate, git-URL-installed plugin (ADR 0064), not a + dependency of this one, and it ships DISABLED by default, so absent/disabled is + the expected common case (not an error worth logging).""" + try: + return importlib.import_module("plugins.coder.solve") + except Exception: # noqa: BLE001 — coder plugin absent/disabled → honest degrade + return None + + +def should_use_solve(feature: dict, *, test_cmd: str, _solve_mod=None) -> bool: + """The P2 dispatch decision (ADR 0064): use `coder.solve()` only when ALL of — + the `coder` plugin is importable, the feature carries acceptance criteria (the + Ready gate's oracle), and a runnable acceptance-test command is configured (the + actual verifier `solve()` gates on — prose acceptance criteria alone isn't + executable). Missing any ⇒ False, the honest degrade to a single delegate_to(acp) + shot. ``_solve_mod`` is a test-injection seam; production callers never pass it — + the real best-effort import happens here.""" + mod = _solve_mod if _solve_mod is not None else _import_solve() + if mod is None: + return False + if not str(feature.get("acceptance_criteria") or "").strip(): + return False + if not str(test_cmd or "").strip(): + return False + return True + + +def _augment_prompt(task: str, feedback: str | None) -> str: + """Fold the ladder's failing-test feedback into the next candidate's prompt. + Every candidate gets a FRESH worktree off base (see ``_WorktreeSolveAdapter`` — + the same "fresh-both" discipline ``worktree.dispatch_coder`` already documents + for re-dispatches), so the coder is told explicitly there is no prior diff to + build on in THIS worktree — only the failure to fix.""" + if not feedback: + return task + return ( + f"{task}\n\n" + "## Your previous attempt FAILED the acceptance tests — fix exactly this\n" + "This is a fresh worktree (no prior diff here); re-implement with the failure " + f"below in mind:\n{feedback.strip()}\n" + ) + + +class _WorktreeSolveAdapter: + """Adapts `coder.solve()`'s ``generate``/``verify`` contract onto board + worktrees. `solve()` treats a candidate as an opaque string; here that string is + a candidate's WORKTREE PATH, not code text — the coder edits files, it doesn't + return a source string. Each ``generate()`` call creates a fresh throwaway + worktree, dispatches the ACP coder into it, and hands the path back; ``verify()`` + then runs the acceptance-test command in that same worktree and reports real + pass/fail. Tracks every candidate worktree it creates so the caller can promote + the winner and reap the losers.""" + + def __init__( + self, + *, + repo: str, + base: str, + root: str, + fid: str, + coder, + dispatch_timeout: float | None, + test_cmd: str, + test_timeout: float, + verdict_cls, + ): + self.repo = repo + self.base = base + self.root = root + self.fid = fid + self.coder = coder + self.dispatch_timeout = dispatch_timeout + self.test_cmd = test_cmd + self.test_timeout = test_timeout + self.verdict_cls = verdict_cls # `plugins.coder.solve.Verdict` — passed in, never imported here + self.candidates: list[tuple[str, str]] = [] # (worktree_path, branch) + # `git worktree add` against the SAME repo must not run concurrently (best- + # of-k dispatches `generate()` via asyncio.gather) — serialize just that + # step; the slow part (the coder dispatch) still runs in parallel. + self._wt_lock = asyncio.Lock() + self._n = 0 + + async def generate(self, task: str, *, feedback: str | None = None) -> str: + self._n += 1 + cid = f"{self.fid}.g{self._n}" + async with self._wt_lock: + wt, branch = await worktree.create_worktree(self.repo, self.base, cid, self.root) + self.candidates.append((wt, branch)) + await worktree.dispatch_coder(self.coder, wt, _augment_prompt(task, feedback), timeout=self.dispatch_timeout) + return wt + + async def verify(self, candidate_wt: str): + Verdict = self.verdict_cls + try: + proc = await asyncio.create_subprocess_shell( + self.test_cmd, + cwd=candidate_wt, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.STDOUT, + ) + except OSError as exc: + return Verdict(passed=False, total=1, failed=1, output=f"could not launch acceptance tests: {exc}") + try: + out, _ = await asyncio.wait_for(proc.communicate(), timeout=self.test_timeout) + except asyncio.TimeoutError: + try: + proc.kill() + except ProcessLookupError: + pass + await proc.wait() + # Unlike the pre-PR local gate (which fails OPEN on a timeout — a broken + # gate must never block otherwise-good work), THIS is the ladder's own + # search oracle: a candidate we couldn't confirm passed must never be + # silently treated as passing, or we'd be faking grounding. + return Verdict( + passed=False, total=1, failed=1, output=f"acceptance tests timed out after {self.test_timeout:.0f}s" + ) + text = (out or b"").decode("utf-8", "replace").strip() + ok = proc.returncode == 0 + return Verdict( + passed=ok, + total=1, + failed=0 if ok else 1, + failing=[] if ok else [f"{self.test_cmd!r} (exit {proc.returncode})"], + output=text[-4000:], + ) + + +RecordGens = Callable[[int], None] + + +def _record_gens_best_effort(record_gens: RecordGens, fid: str, n: int) -> None: + """Call ``record_gens(n)`` and swallow any exception it raises. + + ``store.record_gens_spent`` documents itself as fire-and-forget ("a br hiccup + here must never fail the build the way a missing PR would") — a transient `br` + failure (lock contention, a flaky CLI invocation, a race with a concurrent + label write) must never propagate out of ``dispatch()``. Left unguarded, it + would surface as an unrelated ``BoardError`` past every capability-failure + handler in the caller's loop, discarding an already-verified (or already- + reaped) candidate purely because of a bookkeeping label write.""" + try: + record_gens(n) + except Exception: # noqa: BLE001 — fire-and-forget cost accounting, never fails the build + log.warning("[project_board] %s record_gens(%d) failed (ignored — fire-and-forget)", fid, n, exc_info=True) + + +async def dispatch( + *, + task: str, + coder, + repo: str, + base: str, + root: str, + fid: str, + dispatch_timeout: float | None, + test_cmd: str, + test_timeout: float, + budget: int, + k: int, + tree_depth: int, + record_gens: RecordGens | None = None, + _solve=None, + _budget_cls=None, + _verdict_cls=None, +) -> tuple[str, str, str]: + """Run the execution-grounded ladder for one feature build. + + Returns ``(worktree, branch, result_text)`` on a passing candidate — the SAME + 3-tuple shape ``_dispatch_max_mode`` returns, so the caller's downstream drive + (fixups, local gate, ``open_pr``) is unchanged. Raises :class:`SolveExhausted` + (a capability failure) when the budget is spent with no passing candidate, after + reaping every candidate worktree it created. + + ``record_gens`` (if given) is called with ``result.gens_spent`` exactly once, + win or lose — the cost accounting (ADR 0064) must survive a failed search too. + ``_solve``/``_budget_cls``/``_verdict_cls`` are test-injection seams for + ``solve()``/``Budget``/``Verdict``; production callers never pass them (the real + import happens here, deferred so this module carries no hard dependency on the + `coder` plugin). + + **``solve()`` itself can raise.** The ladder (`coder`'s own ``solve.py``) has no + try/except around ``generate``/``verify`` — it assumes a candidate attempt never + errors, only that it might fail its tests. A REAL dispatch can still raise + (``CoderTimeout`` on one best-of-k candidate, a worktree op erroring) and that + propagates straight out of ``solve()`` uncaught. Every worktree ``generate()`` + already created for THIS run is tracked in ``adapter.candidates`` (appended right + after ``create_worktree`` returns, before the dispatch that might fail) but would + otherwise leak forever: it's untracked in the loop's ``_inflight`` map until this + function returns, and invisible to the health sweep (a `.gN` candidate id isn't a + real board feature, so the sweep's own ``get_feature`` lookup raises and the sweep + skips it rather than reaping). So any exception here reaps every candidate seen so + far, surfaces the attempted cost, and re-raises the ORIGINAL exception unchanged — + the loop's existing capability-failure handling (retry/escalate/block) still + applies to whatever it actually was.""" + if _solve is not None: + solve, Budget, Verdict = _solve, _budget_cls, _verdict_cls + else: + from plugins.coder.solve import Budget, Verdict, solve + + adapter = _WorktreeSolveAdapter( + repo=repo, + base=base, + root=root, + fid=fid, + coder=coder, + dispatch_timeout=dispatch_timeout, + test_cmd=test_cmd, + test_timeout=test_timeout, + verdict_cls=Verdict, + ) + try: + result = await solve( + task, + generate=adapter.generate, + verify=adapter.verify, + budget=Budget(budget), + k=k, + tree_depth=tree_depth, + ) + except Exception as exc: + for wt, branch in adapter.candidates: + await worktree.remove_worktree(repo, wt, branch) + if record_gens is not None and adapter._n: + # `solve()` never got to return a `gens_spent` count — the attempted + # generation count is the honest stand-in (a failed dispatch still spent + # the gen; ADR 0064's cost accounting doesn't get to look the other way). + # Best-effort per store.record_gens_spent's own contract ("a br hiccup + # here must never fail the build"): the worktrees above are ALREADY + # reaped and the original exception below is what the loop must see — + # a transient `br` failure recording the spend must never mask it. + _record_gens_best_effort(record_gens, fid, adapter._n) + log.warning( + "[project_board] %s coder.solve raised mid-ladder (%d candidate(s) reaped): %s", + fid, + len(adapter.candidates), + exc, + ) + raise + if record_gens is not None: + # Same fire-and-forget contract as above: a bookkeeping failure here must + # never discard a candidate that ALREADY exists on disk (test-verified or + # not) — the promote/reap logic below still has to run either way. + _record_gens_best_effort(record_gens, fid, result.gens_spent) + + if not result.passed or not result.solution: + for wt, branch in adapter.candidates: + await worktree.remove_worktree(repo, wt, branch) + detail = result.verdict.feedback() if result.verdict else "" + log.info( + "[project_board] %s coder.solve exhausted (rung=%s, gens=%d/%d) — no candidate passed", + fid, + result.rung, + result.gens_spent, + budget, + ) + raise SolveExhausted( + f"coder.solve exhausted after {result.gens_spent} generation(s) (rung={result.rung}): " + f"{detail or result.note}" + ) + + win_wt = result.solution + win_branch = next(b for wt, b in adapter.candidates if wt == win_wt) + canon_wt, canon_branch = await worktree.promote_worktree(repo, win_wt, win_branch, fid, root) + for wt, branch in adapter.candidates: + if wt != win_wt: + await worktree.remove_worktree(repo, wt, branch) + log.info( + "[project_board] %s coder.solve verified by acceptance tests (rung=%s, gens=%d/%d)", + fid, + result.rung, + result.gens_spent, + budget, + ) + result_text = f"[coder.solve rung={result.rung} gens={result.gens_spent}] {result.note}" + return canon_wt, canon_branch, result_text diff --git a/loop.py b/loop.py index 5505974..8fde737 100644 --- a/loop.py +++ b/loop.py @@ -20,6 +20,17 @@ each ``in_review`` PR's state and drives the terminal edges: merged → done (the same idempotent edge), closed-unmerged → blocked. Up to ``max_concurrent`` features build concurrently, each in its own worktree. + +**coder.solve() board seam (ADR 0064 P2, opt-in, see ``coder_seam.py``).** On a +fresh build (not a keep-worktree/CI-bounce re-dispatch), when the `coder` plugin is +importable AND the feature has acceptance criteria AND a runnable acceptance-test +command is configured, ``delegate_to(coder)`` is replaced by +``coder_seam.dispatch()`` — an execution-grounded ladder (greedy → best-of-k → +tree-search) that runs the feature's acceptance tests in real candidate worktrees +and gates on them actually PASSING, never an LLM judge. It composes WITH the +`coders`-map tier ladder below (search happens WITHIN a tier; a search that never +passes is a capability failure that escalates/blocks exactly like a no-diff +dispatch). Missing any of the three gates ⇒ honest degrade to the single shot above. """ from __future__ import annotations @@ -29,7 +40,7 @@ import re import time -from . import worktree +from . import coder_seam, worktree from .failures import classify from .store import BoardError, escalation_enabled, get_store @@ -188,6 +199,41 @@ def __init__(self, cfg: dict): self.local_gate_max = max(0, int(self.cfg.get("local_gate_max", 2))) self.local_gate_timeout = float(self.cfg.get("local_gate_timeout_s", 600)) self.local_gate_output_chars = max(500, int(self.cfg.get("local_gate_output_chars", 4000))) + # ── coder.solve() board seam (ADR 0064 P2, opt-in) ───────────────────────── + # Route a FRESH build (not a keep-worktree/CI-bounce re-dispatch) through the + # `coder` plugin's execution-grounded solve() ladder (greedy → best-of-k → + # tree-search) instead of a single delegate_to(acp) shot — gated on the + # feature's acceptance tests actually PASSING in a real worktree, never an + # LLM judge. HONEST DEGRADE (coder_seam.should_use_solve): only fires when + # the `coder` plugin is importable (host has it enabled) AND this feature + # carries acceptance_criteria AND a runnable test command is configured + # below — missing any of the three falls back to today's single shot, so an + # existing deployment can't regress just by upgrading. Composes WITH (does + # NOT replace) the coders-map tier ladder: solve() searches within the + # CURRENT tier; a search that never passes raises SolveExhausted, which + # `_drive` treats as the same capability failure as a no-diff dispatch + # (escalate a tier, or block) — the tier ladder still climbs when search + # itself stalls. + # + # Precedence vs. Max-Mode (`max_mode_n>1`, below): coder_solve ONLY preempts + # Max-Mode when Max-Mode itself is off (`max_mode_n<=1`) — see + # `_use_coder_solve`. Without this, a board already running the README's own + # execution-grounded Max-Mode recipe (`max_mode_n>1` + `local_gate_cmd`) would + # silently stop using Max-Mode the moment the separate `coder` plugin became + # importable for any unrelated reason, with zero change to THIS board's own + # config — and unlike Max-Mode's LLM-judge fallback (which always ships a + # best-effort PR), an exhausted solve() ladder blocks the feature outright. + # That's a behavior change an operator must opt into, not inherit for free. + self.coder_solve = bool(self.cfg.get("coder_solve", True)) + # The ladder's verifier: the command that runs THIS feature's (coder- + # authored) acceptance tests in a candidate worktree, e.g. "pytest tests/ -q". + # Blank ⇒ falls back to local_gate_cmd (many repos already configure that as + # the real test command); still blank ⇒ no runnable oracle ⇒ honest degrade. + self.coder_solve_test_cmd = str(self.cfg.get("coder_solve_test_cmd", "")).strip() or self.local_gate_cmd + self.coder_solve_test_timeout = float(self.cfg.get("coder_solve_test_timeout_s", 300)) + self.coder_solve_budget = max(1, int(self.cfg.get("coder_solve_budget", 6))) + self.coder_solve_k = max(1, int(self.cfg.get("coder_solve_k", 3))) + self.coder_solve_tree_depth = max(0, int(self.cfg.get("coder_solve_tree_depth", 2))) # KG lessons (the flywheel READ half): before dispatching a coder, query the # knowledge graph (via graph.sdk) for distilled lessons relevant to THIS feature # and inject them into the prompt — so the coder heeds this area's known failure @@ -611,6 +657,12 @@ async def _drive(self, feature: dict): # the coder only ADDS what the reviewer flagged (usually tests); a # fresh rebuild makes it re-implement and never reach the tests (the # bd-2fd/bd-3cj block). + # • coder.solve (ADR 0064 P2, opt-in) → the execution-grounded + # ladder over the feature's acceptance tests (coder_seam.py). + # Same "from-scratch build only" rule as max-mode: a carried- + # forward re-dispatch FIXES the existing diff with one coder. + # Only preempts max-mode when max_mode_n<=1 (_use_coder_solve) — + # a board already running Max-Mode keeps that behavior. # • max-mode → N parallel candidates, judge, promote the winner (#21). # ONLY for a from-scratch build: a carried-forward re-dispatch (a CI # bounce / goal-fix / gate-fix — all signalled by _ci_feedback) FIXES @@ -620,6 +672,23 @@ async def _drive(self, feature: dict): keep_wt = False # consume the reuse self._inflight[fid] = (repo, wt, branch) result = await worktree.dispatch_coder(coder, wt, prompt, timeout=self.coder_timeout or None) + elif self._use_coder_solve(feature) and not self._ci_feedback.get(fid): + wt, branch, result = await coder_seam.dispatch( + task=prompt, + coder=coder, + repo=repo, + base=base, + root=self.root, + fid=fid, + dispatch_timeout=self.coder_timeout or None, + test_cmd=self.coder_solve_test_cmd, + test_timeout=self.coder_solve_test_timeout, + budget=self.coder_solve_budget, + k=self.coder_solve_k, + tree_depth=self.coder_solve_tree_depth, + record_gens=lambda n: store.record_gens_spent(fid, n), + ) + self._inflight[fid] = (repo, wt, branch) elif self.max_mode_n > 1 and not self._ci_feedback.get(fid): wt, branch, result = await self._dispatch_max_mode(feature, coder, prompt, repo, base, fid) self._inflight[fid] = (repo, wt, branch) @@ -704,7 +773,7 @@ async def _drive(self, feature: dict): # same coder won't help) — they escalate a tier or block. Only true # infra failures (push/fetch/gh network/rate-limit) get the backoff. capability = ( - isinstance(exc, (worktree.NoChangesError, worktree.CoderTimeout)) + isinstance(exc, (worktree.NoChangesError, worktree.CoderTimeout, coder_seam.SolveExhausted)) or str(exc).startswith("coder dispatch failed") or str(exc).startswith("goal verification failed") ) @@ -785,6 +854,25 @@ async def _request_review(self, fid: str, pr_url: str): log.warning("[project_board] review dispatch for %s failed: %s", fid, exc) # ── helpers ─────────────────────────────────────────────────────────────── + def _use_coder_solve(self, feature: dict) -> bool: + """The P2 board-seam dispatch decision (ADR 0064) — see coder_seam.py. + `coder_solve` is this repo's own opt-out valve (default on); the actual + grounding gate (coder plugin importable + acceptance criteria + a runnable + test command) lives in ``coder_seam.should_use_solve``. + + Max-Mode takes precedence when both are configured (`max_mode_n>1`): a + board already relying on Max-Mode's judge-fallback (always ships a + best-effort PR) must not have that silently swapped for solve()'s harder + "block if nothing passes" behavior just because the separate `coder` + plugin became importable. Enabling coder.solve on such a board is a + deliberate config change (set `max_mode_n<=1`), never a side effect of + installing `coder` for something else.""" + if not self.coder_solve: + return False + if self.max_mode_n > 1: + return False + return coder_seam.should_use_solve(feature, test_cmd=self.coder_solve_test_cmd) + def _resolve_delegate(self, name: str, expect_type: str): """Look up a live delegate by name from the delegates registry. Returns the Delegate or None (not configured / wrong type / plugin disabled).""" diff --git a/protoagent.plugin.yaml b/protoagent.plugin.yaml index 5968aa3..e680125 100644 --- a/protoagent.plugin.yaml +++ b/protoagent.plugin.yaml @@ -1,6 +1,6 @@ id: project_board name: Project Board (coding orchestration) -version: 0.25.0 +version: 0.26.0 description: >- A board-driven coding-orchestration plugin: a lean 6-state board (backlog → ready → in_progress → in_review → done, + a blocked flag) backed by **beads** (`br`), an @@ -53,6 +53,38 @@ config: # opens ANYWAY (CI + ci_fix_max stay the backstop — a flaky or # misconfigured gate never blocks good work). local_gate_timeout_s: 600 # hard cap on one gate run; a timeout degrades to "pass" (CI gates). + coder_solve: true # OPT-OUT valve for the ADR 0064 P2 board seam (default on; + # the real gate is coder_seam.should_use_solve — see below). + # On a FRESH build (not a keep-worktree/CI-bounce re-dispatch), + # when the `coder` plugin is importable AND the feature has + # acceptance_criteria AND coder_solve_test_cmd (or + # local_gate_cmd) is set, the loop dispatches through + # coder.solve()'s execution-grounded ladder (greedy → + # best-of-k → tree-search) instead of a single + # delegate_to(acp) shot — gated on the feature's acceptance + # tests actually PASSING in a real worktree, never an LLM + # judge. Missing any of those three ⇒ honest degrade to + # today's single shot (non-regressing by default: the + # `coder` plugin ships disabled, and this needs a test + # command configured too). Composes WITH the `coders`-map + # tier ladder: solve() searches WITHIN a tier; a search + # that never passes escalates a tier (or blocks) exactly + # like a no-diff dispatch. + coder_solve_test_cmd: "" # the command that runs THIS feature's (coder-authored) + # acceptance tests in a candidate worktree, e.g. + # "pytest tests/ -q" — solve()'s verify() oracle. Blank + # falls back to local_gate_cmd (many repos already + # configure that as the real test command); still blank + # ⇒ no runnable oracle ⇒ honest degrade to the single shot. + coder_solve_test_timeout_s: 300 # hard cap on one candidate's acceptance-test run; + # a timeout is a FAILED verdict (never a silent pass — + # this is the ladder's own search oracle, unlike the + # fail-open pre-PR local_gate_cmd above). + coder_solve_budget: 6 # hard cap on total candidate generations across the + # whole ladder for one feature build. + coder_solve_k: 3 # best-of-k width (rung 2): candidates generated (each in + # its own throwaway worktree) before execution-select. + coder_solve_tree_depth: 2 # refine-on-failing-tests depth (rung 3). max_mode_n: 1 # best-of-N "Max-Mode" (borrowed from MiMo-Code). 1 = off. >1 builds a # feature N ways in parallel (N candidate worktrees, all dispatched at # once), a low-temp judge picks the diff that best satisfies diff --git a/pyproject.toml b/pyproject.toml index 4ec22d7..9175a50 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "project-board" -version = "0.25.0" +version = "0.26.0" description = "Board-driven coding-orchestration plugin for protoAgent (beads board + ACP spawn loop + planning layer + console view)." requires-python = ">=3.11" diff --git a/store.py b/store.py index a20d73d..3a9dc61 100644 --- a/store.py +++ b/store.py @@ -56,6 +56,10 @@ # non-foundation blocker, which can release dependents at in_review under dep_gate: # review). Inert under the default dep_gate: merge (then every blocker gates on merge). LABEL_FOUNDATION = "foundation" +# Cumulative generations `coder.solve()` has spent on this feature (ADR 0064 P2 board +# seam) — `gens:`, replaced (not accumulated as separate labels) each time so a +# single label always carries the running total for `portfolio_rollup` to read. +LABEL_GENS_PREFIX = "gens:" # difficulty → initial model tier (the escalation ladder's first rung, D10). DIFFICULTY_TIER = {"small": "smart", "medium": "reasoning", "large": "reasoning", "architectural": "opus"} @@ -426,6 +430,24 @@ def next_tier(self, current: str) -> str | None: return TIER_LADDER[0] return TIER_LADDER[i + 1] if i + 1 < len(TIER_LADDER) else None + # ── coder.solve() cost accounting (ADR 0064 P2 board seam) ──────────────── + def record_gens_spent(self, fid: str, n: int) -> dict: + """Accumulate `n` more generations `coder.solve()` spent on this feature onto + its `gens:` label — a single, replaced label so `portfolio_rollup` (the + PM tier) can read the running cost without raw reads, per the ADR's cost-v1 + ethos. Called once per `solve()` run, win or lose (a failed search still spent + gens). Best-effort in the sense that a `br` hiccup here must never fail the + build the way a missing PR would — callers should treat it as fire-and-forget.""" + f = self._require(fid) + total = int(f.get("gens_spent", 0)) + max(0, int(n)) + stale = [l for l in f.get("labels") or [] if l.startswith(LABEL_GENS_PREFIX)] + args = ["update", fid] + for label in stale: + args += ["--remove-label", label] + args += ["--add-label", f"{LABEL_GENS_PREFIX}{total}"] + self._run(*args) + return self.get_feature(fid) + # ── reads (the projection) ──────────────────────────────────────────────── def get_feature(self, fid: str) -> dict | None: rows = self._run("show", fid, want_json=True) @@ -566,6 +588,16 @@ def _project(self, bead: dict) -> dict: attempts = sorted( int(l.split(":", 1)[1]) for l in labels if l.startswith("attempt:") and l.split(":", 1)[1].isdigit() ) + # coder.solve()'s cumulative generation cost (ADR 0064 P2), read from the + # single replaced `gens:` label — 0 for a feature the seam never touched. + gens_spent = next( + ( + int(l[len(LABEL_GENS_PREFIX) :]) + for l in labels + if l.startswith(LABEL_GENS_PREFIX) and l[len(LABEL_GENS_PREFIX) :].isdigit() + ), + 0, + ) # `dag_blocked`: marked `ready` but a `blocks` dependency is still open, so # the puller won't claim it. Only `br show` carries dependencies (`br list` # doesn't); list_features patches this by cross-referencing the puller. @@ -594,6 +626,7 @@ def _project(self, bead: dict) -> dict: "foundation": LABEL_FOUNDATION in labels, "difficulty": diff, "attempts": attempts, + "gens_spent": gens_spent, "labels": labels, "repo": self.repo, "base_branch": self.base_branch, diff --git a/tests/test_coder_seam.py b/tests/test_coder_seam.py new file mode 100644 index 0000000..b8c7492 --- /dev/null +++ b/tests/test_coder_seam.py @@ -0,0 +1,618 @@ +"""Tests for the ADR 0064 P2 board seam (coder_seam.py). + +Covers the dispatch DECISION (``should_use_solve``) — the honest-degrade gate that +must fire false the instant any one of coder/acceptance/test-cmd is missing — and +``dispatch()``'s own orchestration (worktree-per-candidate, promote the winner, reap +the losers, surface gens-spent, raise ``SolveExhausted`` on a spent budget) with the +`coder` plugin's ``solve``/``Budget``/``Verdict`` injected as fakes, so none of this +needs the (separate, git-URL-installed) `coder` plugin to be present.""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +from project_board import coder_seam, worktree +from project_board.coder_seam import SolveExhausted, _WorktreeSolveAdapter, dispatch, should_use_solve + +FEATURE_WITH_AC = {"id": "bd-1", "acceptance_criteria": "WHEN x THE SYSTEM SHALL y"} +FEATURE_NO_AC = {"id": "bd-2", "acceptance_criteria": ""} + + +# ── the dispatch decision (honest degrade) ─────────────────────────────────────── + + +def test_should_use_solve_true_when_all_three_gates_hold(): + assert should_use_solve(FEATURE_WITH_AC, test_cmd="pytest -q", _solve_mod=object()) is True + + +def test_should_use_solve_false_when_coder_plugin_unavailable(): + assert should_use_solve(FEATURE_WITH_AC, test_cmd="pytest -q", _solve_mod=None) is False + + +def test_should_use_solve_false_without_acceptance_criteria(): + assert should_use_solve(FEATURE_NO_AC, test_cmd="pytest -q", _solve_mod=object()) is False + + +def test_should_use_solve_false_without_a_test_command(): + assert should_use_solve(FEATURE_WITH_AC, test_cmd="", _solve_mod=object()) is False + assert should_use_solve(FEATURE_WITH_AC, test_cmd=" ", _solve_mod=object()) is False + + +def test_import_solve_returns_none_when_the_coder_plugin_is_absent(): + """`coder` is a separate plugin repo — genuinely absent in this standalone test + env, which IS the honest-degrade case in production too (not a mock).""" + assert coder_seam._import_solve() is None + + +def test_solve_exhausted_is_a_worktree_error(): + """So `_drive`'s existing ``except (worktree.NoChangesError, worktree.WorktreeError)`` + catches it with no changes to the except clause itself.""" + assert issubclass(SolveExhausted, worktree.WorktreeError) + + +# ── fakes standing in for plugins.coder.solve (never imported here) ───────────── + + +@dataclass +class _FakeVerdict: + passed: bool + total: int = 0 + failed: int = 0 + failing: list = field(default_factory=list) + output: str = "" + + def feedback(self) -> str: + return "" if self.passed else f"{self.failed}/{self.total} failing: {self.output}" + + +@dataclass +class _FakeResult: + solution: str | None + passed: bool | None + rung: str + gens_spent: int + candidates_tried: int + verdict: _FakeVerdict | None = None + note: str = "" + + +class _FakeBudget: + def __init__(self, total): + self.total = total + + +def _stub_worktree(monkeypatch, *, created=None, removed=None, promoted=None): + created = created if created is not None else [] + removed = removed if removed is not None else [] + promoted = promoted if promoted is not None else [] + + async def _create(repo, base, cid, root): + created.append(cid) + return (f"/wt/feat-{cid}", f"feat/{cid}") + + async def _dispatch(coder, wt, prompt, *, timeout=None): + return f"reply from {wt}" + + async def _remove(repo, wt, branch=""): + removed.append(wt) + + async def _promote(repo, src_wt, src_branch, fid, root=".worktrees"): + promoted.append((src_wt, src_branch, fid)) + return (f"/wt/feat-{fid}", f"feat/{fid}") + + monkeypatch.setattr(worktree, "create_worktree", _create) + monkeypatch.setattr(worktree, "dispatch_coder", _dispatch) + monkeypatch.setattr(worktree, "remove_worktree", _remove) + monkeypatch.setattr(worktree, "promote_worktree", _promote) + return created, removed, promoted + + +# ── dispatch(): the winning-candidate path ─────────────────────────────────────── + + +async def test_dispatch_promotes_the_winner_and_reaps_the_losers(monkeypatch): + created, removed, promoted = _stub_worktree(monkeypatch) + + async def _fake_solve(task, *, generate, verify, budget, k, tree_depth): + # exercise the adapter for real: two candidates, the second "wins". + await generate(task, feedback=None) + c1 = await generate(task, feedback=None) + return _FakeResult(solution=c1, passed=True, rung="best-of-k", gens_spent=2, candidates_tried=2) + + gens = [] + wt, branch, result = await dispatch( + task="do the thing", + coder=object(), + repo="/repo", + base="main", + root=".worktrees", + fid="bd-1", + dispatch_timeout=None, + test_cmd="pytest -q", + test_timeout=30, + budget=6, + k=3, + tree_depth=2, + record_gens=gens.append, + _solve=_fake_solve, + _budget_cls=_FakeBudget, + _verdict_cls=_FakeVerdict, + ) + assert created == ["bd-1.g1", "bd-1.g2"] + assert promoted == [("/wt/feat-bd-1.g2", "feat/bd-1.g2", "bd-1")] + assert removed == ["/wt/feat-bd-1.g1"] # only the loser reaped + assert (wt, branch) == ("/wt/feat-bd-1", "feat/bd-1") # canonical name + assert "best-of-k" in result and "gens=2" in result + assert gens == [2] # cost surfaced exactly once + + +async def test_dispatch_records_gens_even_on_a_single_greedy_win(monkeypatch): + _stub_worktree(monkeypatch) + + async def _fake_solve(task, *, generate, verify, budget, k, tree_depth): + c0 = await generate(task, feedback=None) + return _FakeResult(solution=c0, passed=True, rung="greedy", gens_spent=1, candidates_tried=1) + + gens = [] + await dispatch( + task="t", + coder=object(), + repo="/repo", + base="main", + root=".worktrees", + fid="bd-9", + dispatch_timeout=None, + test_cmd="pytest -q", + test_timeout=30, + budget=6, + k=3, + tree_depth=2, + record_gens=gens.append, + _solve=_fake_solve, + _budget_cls=_FakeBudget, + _verdict_cls=_FakeVerdict, + ) + assert gens == [1] + + +async def test_dispatch_promotes_the_winner_even_if_record_gens_raises(monkeypatch): + """`store.record_gens_spent` documents itself as fire-and-forget ("a br hiccup + here must never fail the build the way a missing PR would") — a `BoardError` + (lock contention, a flaky `br` call) out of `record_gens` must never discard an + already-verified winning candidate or leak it un-promoted.""" + created, removed, promoted = _stub_worktree(monkeypatch) + + async def _fake_solve(task, *, generate, verify, budget, k, tree_depth): + c0 = await generate(task, feedback=None) + return _FakeResult(solution=c0, passed=True, rung="greedy", gens_spent=1, candidates_tried=1) + + def _boom_record(n): + raise RuntimeError("br hiccup: lock contention") + + wt, branch, result = await dispatch( + task="t", + coder=object(), + repo="/repo", + base="main", + root=".worktrees", + fid="bd-10", + dispatch_timeout=None, + test_cmd="pytest -q", + test_timeout=30, + budget=6, + k=3, + tree_depth=2, + record_gens=_boom_record, + _solve=_fake_solve, + _budget_cls=_FakeBudget, + _verdict_cls=_FakeVerdict, + ) + assert created == ["bd-10.g1"] + assert promoted == [("/wt/feat-bd-10.g1", "feat/bd-10.g1", "bd-10")] # still promoted + assert (wt, branch) == ("/wt/feat-bd-10", "feat/bd-10") # dispatch() itself never raised + + +# ── dispatch(): the exhausted (no passing candidate) path ─────────────────────── + + +async def test_dispatch_raises_solve_exhausted_and_reaps_every_candidate(monkeypatch): + created, removed, promoted = _stub_worktree(monkeypatch) + + async def _fake_solve(task, *, generate, verify, budget, k, tree_depth): + await generate(task, feedback=None) + c1 = await generate(task, feedback="prior failure") + v = _FakeVerdict(passed=False, total=1, failed=1, output="AssertionError: nope") + return _FakeResult(solution=c1, passed=False, rung="best-partial", gens_spent=2, candidates_tried=2, verdict=v) + + gens = [] + try: + await dispatch( + task="t", + coder=object(), + repo="/repo", + base="main", + root=".worktrees", + fid="bd-2", + dispatch_timeout=None, + test_cmd="pytest -q", + test_timeout=30, + budget=6, + k=3, + tree_depth=2, + record_gens=gens.append, + _solve=_fake_solve, + _budget_cls=_FakeBudget, + _verdict_cls=_FakeVerdict, + ) + raised = False + except SolveExhausted as exc: + raised = True + assert "2 generation(s)" in str(exc) + assert "best-partial" in str(exc) + assert raised + assert promoted == [] # nothing promoted — never opens a PR on an unverified partial + assert set(removed) == {"/wt/feat-bd-2.g1", "/wt/feat-bd-2.g2"} # every candidate reaped + assert gens == [2] # cost surfaced even though the search failed + + +async def test_dispatch_exhausted_with_no_candidates_at_all(monkeypatch): + """Budget exhausted before even one generation (an edge solve() itself covers) — + dispatch() must still raise cleanly with nothing to reap.""" + _created, removed, promoted = _stub_worktree(monkeypatch) + + async def _fake_solve(task, *, generate, verify, budget, k, tree_depth): + return _FakeResult( + solution=None, passed=None, rung="none", gens_spent=0, candidates_tried=0, note="budget exhausted" + ) + + try: + await dispatch( + task="t", + coder=object(), + repo="/repo", + base="main", + root=".worktrees", + fid="bd-3", + dispatch_timeout=None, + test_cmd="pytest -q", + test_timeout=30, + budget=0, + k=3, + tree_depth=2, + _solve=_fake_solve, + _budget_cls=_FakeBudget, + _verdict_cls=_FakeVerdict, + ) + raised = False + except SolveExhausted: + raised = True + assert raised + assert removed == [] and promoted == [] + + +async def test_dispatch_still_reaps_and_raises_solve_exhausted_when_record_gens_raises(monkeypatch): + """Same fire-and-forget contract on the exhausted path: a `record_gens` failure + must not prevent every reaped candidate from actually being reaped, nor swallow + the (honest) `SolveExhausted` the caller needs to see.""" + created, removed, promoted = _stub_worktree(monkeypatch) + + async def _fake_solve(task, *, generate, verify, budget, k, tree_depth): + await generate(task, feedback=None) + c1 = await generate(task, feedback="prior failure") + v = _FakeVerdict(passed=False, total=1, failed=1, output="AssertionError: nope") + return _FakeResult(solution=c1, passed=False, rung="best-partial", gens_spent=2, candidates_tried=2, verdict=v) + + def _boom_record(n): + raise RuntimeError("br hiccup: flaky CLI invocation") + + try: + await dispatch( + task="t", + coder=object(), + repo="/repo", + base="main", + root=".worktrees", + fid="bd-11", + dispatch_timeout=None, + test_cmd="pytest -q", + test_timeout=30, + budget=6, + k=3, + tree_depth=2, + record_gens=_boom_record, + _solve=_fake_solve, + _budget_cls=_FakeBudget, + _verdict_cls=_FakeVerdict, + ) + raised = False + except SolveExhausted: + raised = True + assert raised # the honest SolveExhausted still surfaces, not the record_gens RuntimeError + assert created == ["bd-11.g1", "bd-11.g2"] + assert set(removed) == {"/wt/feat-bd-11.g1", "/wt/feat-bd-11.g2"} # both still reaped + assert promoted == [] + + +# ── dispatch(): solve() itself raises mid-ladder (not just returns unpassed) ──── + + +async def test_dispatch_reaps_candidates_when_solve_raises_and_reraises_original(monkeypatch): + """`solve()` (the `coder` plugin's ladder) has no try/except of its own around + `generate`/`verify` — a real candidate failure (e.g. `CoderTimeout` on one + best-of-k candidate, or a worktree op erroring) propagates straight out instead + of being scored as a loss. Every worktree already created before the raise must + still be reaped — untracked in `_inflight` until `dispatch()` returns, and + invisible to the health sweep (a `.gN` id isn't a real board feature) — or a + single flaky candidate leaks a worktree forever. The original exception must + surface unchanged so the loop's existing capability-failure handling classifies + it correctly (e.g. a `CoderTimeout` still escalates/blocks like it always has).""" + created, removed, promoted = _stub_worktree(monkeypatch) + + class _Boom(RuntimeError): + pass + + calls = {"n": 0} + + async def _dispatch(coder, wt, prompt, *, timeout=None): + calls["n"] += 1 + if calls["n"] == 2: + raise _Boom("candidate coder timed out") + return "ok" + + monkeypatch.setattr(worktree, "dispatch_coder", _dispatch) + + async def _fake_solve(task, *, generate, verify, budget, k, tree_depth): + await generate(task, feedback=None) # candidate 1: dispatch succeeds + await generate(task, feedback=None) # candidate 2: dispatch raises — uncaught by solve() + + gens = [] + try: + await dispatch( + task="t", + coder=object(), + repo="/repo", + base="main", + root=".worktrees", + fid="bd-5", + dispatch_timeout=None, + test_cmd="pytest -q", + test_timeout=30, + budget=6, + k=3, + tree_depth=2, + record_gens=gens.append, + _solve=_fake_solve, + _budget_cls=_FakeBudget, + _verdict_cls=_FakeVerdict, + ) + raised = False + except _Boom: + raised = True + assert raised # the ORIGINAL exception surfaces, not something dispatch() invented + assert created == ["bd-5.g1", "bd-5.g2"] + assert set(removed) == {"/wt/feat-bd-5.g1", "/wt/feat-bd-5.g2"} # both reaped, none leaked + assert promoted == [] + assert gens == [2] # the attempted-candidate count still surfaces as spent cost + + +async def test_dispatch_reraises_the_original_mid_ladder_exception_even_if_record_gens_also_raises(monkeypatch): + """If `record_gens` itself blows up (e.g. `BoardError` from a `br` hiccup) while + handling a REAL mid-ladder failure, the original exception must still be what + the caller sees — not the bookkeeping failure, and not silently swallowed.""" + created, removed, promoted = _stub_worktree(monkeypatch) + + class _Boom(RuntimeError): + pass + + calls = {"n": 0} + + async def _dispatch(coder, wt, prompt, *, timeout=None): + calls["n"] += 1 + if calls["n"] == 2: + raise _Boom("candidate coder timed out") + return "ok" + + monkeypatch.setattr(worktree, "dispatch_coder", _dispatch) + + async def _fake_solve(task, *, generate, verify, budget, k, tree_depth): + await generate(task, feedback=None) + await generate(task, feedback=None) + + def _boom_record(n): + raise RuntimeError("br hiccup: concurrent label write") + + try: + await dispatch( + task="t", + coder=object(), + repo="/repo", + base="main", + root=".worktrees", + fid="bd-12", + dispatch_timeout=None, + test_cmd="pytest -q", + test_timeout=30, + budget=6, + k=3, + tree_depth=2, + record_gens=_boom_record, + _solve=_fake_solve, + _budget_cls=_FakeBudget, + _verdict_cls=_FakeVerdict, + ) + raised_boom = False + except _Boom: + raised_boom = True + assert raised_boom # the ORIGINAL _Boom surfaces, not record_gens's RuntimeError + assert created == ["bd-12.g1", "bd-12.g2"] + assert set(removed) == {"/wt/feat-bd-12.g1", "/wt/feat-bd-12.g2"} # still reaped + assert promoted == [] + + +async def test_dispatch_raise_with_no_candidates_created_yet_skips_record_gens(monkeypatch): + """A raise before any `generate()` call completed (e.g. `Budget()` itself blew + up) has nothing to reap and nothing real to cost-account — `record_gens` must + not be called with a bogus zero.""" + _created, removed, promoted = _stub_worktree(monkeypatch) + + class _Boom(RuntimeError): + pass + + async def _fake_solve(task, *, generate, verify, budget, k, tree_depth): + raise _Boom("blew up before any generation") + + gens = [] + try: + await dispatch( + task="t", + coder=object(), + repo="/repo", + base="main", + root=".worktrees", + fid="bd-6", + dispatch_timeout=None, + test_cmd="pytest -q", + test_timeout=30, + budget=6, + k=3, + tree_depth=2, + record_gens=gens.append, + _solve=_fake_solve, + _budget_cls=_FakeBudget, + _verdict_cls=_FakeVerdict, + ) + raised = False + except _Boom: + raised = True + assert raised + assert removed == [] and promoted == [] + assert gens == [] # nothing attempted — never fabricate a cost + + +# ── the adapter itself: generate() creates a worktree per candidate, verify() runs +# the acceptance-test command and reports real pass/fail ───────────────────── + + +async def test_adapter_generate_creates_a_fresh_worktree_per_call(monkeypatch): + created, _removed, _promoted = _stub_worktree(monkeypatch) + prompts = [] + + async def _dispatch(coder, wt, prompt, *, timeout=None): + prompts.append(prompt) + return "ok" + + monkeypatch.setattr(worktree, "dispatch_coder", _dispatch) + adapter = _WorktreeSolveAdapter( + repo="/repo", + base="main", + root=".worktrees", + fid="bd-7", + coder=object(), + dispatch_timeout=None, + test_cmd="pytest -q", + test_timeout=30, + verdict_cls=_FakeVerdict, + ) + wt1 = await adapter.generate("do the thing", feedback=None) + wt2 = await adapter.generate("do the thing", feedback="tests X failed") + assert created == ["bd-7.g1", "bd-7.g2"] + assert wt1 != wt2 + assert adapter.candidates == [("/wt/feat-bd-7.g1", "feat/bd-7.g1"), ("/wt/feat-bd-7.g2", "feat/bd-7.g2")] + assert "tests X failed" not in prompts[0] + assert "tests X failed" in prompts[1] # feedback folded into the retry's prompt only + assert "fresh worktree" in prompts[1].lower() + + +async def test_adapter_verify_passes_on_exit_zero(monkeypatch): + async def _ok(*a, **k): + class _Proc: + returncode = 0 + + async def communicate(self): + return (b"3 passed in 0.01s\n", None) + + return _Proc() + + monkeypatch.setattr("asyncio.create_subprocess_shell", _ok) + adapter = _WorktreeSolveAdapter( + repo="/repo", + base="main", + root=".worktrees", + fid="bd-1", + coder=object(), + dispatch_timeout=None, + test_cmd="pytest -q", + test_timeout=30, + verdict_cls=_FakeVerdict, + ) + v = await adapter.verify("/wt/feat-bd-1.g1") + assert v.passed is True and v.failed == 0 + + +async def test_adapter_verify_fails_on_nonzero_exit(monkeypatch): + async def _bad(*a, **k): + class _Proc: + returncode = 1 + + async def communicate(self): + return (b"1 failed, 2 passed in 0.01s\nAssertionError: boom", None) + + return _Proc() + + monkeypatch.setattr("asyncio.create_subprocess_shell", _bad) + adapter = _WorktreeSolveAdapter( + repo="/repo", + base="main", + root=".worktrees", + fid="bd-1", + coder=object(), + dispatch_timeout=None, + test_cmd="pytest -q", + test_timeout=30, + verdict_cls=_FakeVerdict, + ) + v = await adapter.verify("/wt/feat-bd-1.g1") + assert v.passed is False and v.failed == 1 + assert "boom" in v.output + + +async def test_adapter_verify_times_out_as_failed_not_silently_passed(monkeypatch): + """Unlike the pre-PR local gate (fail-open on timeout), the ladder's OWN oracle + must never silently treat an unconfirmed candidate as passing.""" + import asyncio as real_asyncio + + class _Proc: + returncode = None + + async def communicate(self): + raise real_asyncio.TimeoutError() + + def kill(self): + pass + + async def wait(self): + return None + + async def _hang(*a, **k): + return _Proc() + + async def _boom_wait_for(coro, timeout): + coro.close() + raise real_asyncio.TimeoutError() + + monkeypatch.setattr("asyncio.create_subprocess_shell", _hang) + monkeypatch.setattr("project_board.coder_seam.asyncio.wait_for", _boom_wait_for) + adapter = _WorktreeSolveAdapter( + repo="/repo", + base="main", + root=".worktrees", + fid="bd-1", + coder=object(), + dispatch_timeout=None, + test_cmd="pytest -q", + test_timeout=0.01, + verdict_cls=_FakeVerdict, + ) + v = await adapter.verify("/wt/feat-bd-1.g1") + assert v.passed is False + assert "timed out" in v.output diff --git a/tests/test_loop.py b/tests/test_loop.py index abb78c4..23eb589 100644 --- a/tests/test_loop.py +++ b/tests/test_loop.py @@ -19,6 +19,7 @@ class FakeLoopStore: def __init__(self): self.calls = [] + self.gens_spent = {} # fid -> cumulative gens (record_gens_spent) def current_tier(self, fid): return "fast" @@ -31,6 +32,10 @@ def flag_blocked(self, fid, reason): self.calls.append(("flag_blocked", fid, reason)) return {"id": fid} + def record_gens_spent(self, fid, n): + self.gens_spent[fid] = self.gens_spent.get(fid, 0) + n + return {"id": fid} + def names(self): return [c[0] for c in self.calls] @@ -170,7 +175,7 @@ async def _boom(*a, **k): async def _drive_with( - monkeypatch, *, open_pr, coder=object(), dispatch=None, cfg=None, gate=None, judge=None, seed=None + monkeypatch, *, open_pr, coder=object(), dispatch=None, cfg=None, gate=None, judge=None, seed=None, feature=None ): """Run _drive over FEATURE with the worktree helpers + delegate stubbed. Returns the FakeLoopStore so the test can assert the recorded transitions. @@ -217,7 +222,7 @@ async def _promote(repo, src_wt, src_branch, fid, root=".worktrees"): monkeypatch.setattr(loop, "_judge_candidates", judge) if seed is not None: seed(loop) - await loop._drive(FEATURE) + await loop._drive(feature if feature is not None else FEATURE) return loop, store @@ -384,6 +389,263 @@ async def _open_pr(wt, branch, *, base, title, body): assert loop._inflight == {} +# ── coder.solve() board seam (ADR 0064 P2) ─────────────────────────────────────── + + +def test_coder_solve_config_defaults(): + loop = BoardLoop({}) + assert loop.coder_solve is True # opt-OUT valve; the real gate is coder_seam + assert loop.coder_solve_test_cmd == "" # no local_gate_cmd to fall back to either + assert loop.coder_solve_budget == 6 + assert loop.coder_solve_k == 3 + assert loop.coder_solve_tree_depth == 2 + assert loop.coder_solve_test_timeout == 300 + + +def test_coder_solve_test_cmd_falls_back_to_local_gate_cmd(): + assert BoardLoop({"local_gate_cmd": "pytest -q"}).coder_solve_test_cmd == "pytest -q" + loop = BoardLoop({"local_gate_cmd": "pytest -q", "coder_solve_test_cmd": "pytest tests/unit -q"}) + assert loop.coder_solve_test_cmd == "pytest tests/unit -q" # explicit wins over the fallback + + +def test_use_coder_solve_requires_the_opt_out_flag_plus_the_seam_gate(monkeypatch): + from project_board import coder_seam + + monkeypatch.setattr(coder_seam, "_import_solve", lambda: object()) # pretend `coder` is installed + on = BoardLoop({"local_gate_cmd": "pytest -q"}) + assert on._use_coder_solve({"acceptance_criteria": "WHEN x THE SYSTEM SHALL y"}) is True + assert on._use_coder_solve({"acceptance_criteria": ""}) is False # no oracle → degrade + + off = BoardLoop({"local_gate_cmd": "pytest -q", "coder_solve": False}) + assert off._use_coder_solve({"acceptance_criteria": "WHEN x THE SYSTEM SHALL y"}) is False # opted out + + +def test_use_coder_solve_false_when_coder_plugin_unavailable(monkeypatch): + from project_board import coder_seam + + monkeypatch.setattr(coder_seam, "_import_solve", lambda: None) # coder plugin absent/disabled + loop = BoardLoop({"local_gate_cmd": "pytest -q"}) + assert loop._use_coder_solve({"acceptance_criteria": "WHEN x THE SYSTEM SHALL y"}) is False + + +def test_use_coder_solve_false_without_a_test_command(monkeypatch): + from project_board import coder_seam + + monkeypatch.setattr(coder_seam, "_import_solve", lambda: object()) + loop = BoardLoop({}) # no local_gate_cmd, no coder_solve_test_cmd + assert loop._use_coder_solve({"acceptance_criteria": "WHEN x THE SYSTEM SHALL y"}) is False + + +async def _pass_gate(wt): + """A stand-in for `_run_local_gate` — pass immediately. These tests set + `local_gate_cmd` (needed as the coder_solve_test_cmd fallback) but the drive's + fake worktree paths don't exist on disk, so the REAL gate would just shell out + against a bogus cwd; stub it rather than rely on that degrading to a pass.""" + return None + + +async def test_drive_uses_coder_solve_when_available_and_records_gens(monkeypatch): + """coder available + acceptance present + a test command → the solve path runs + INSTEAD of the single delegate_to(acp) shot, and gens-spent lands on the feature + via store.record_gens_spent (so portfolio_rollup can read it).""" + from project_board import coder_seam + + seen = {} + + async def _fake_dispatch( + *, + task, + coder, + repo, + base, + root, + fid, + dispatch_timeout, + test_cmd, + test_timeout, + budget, + k, + tree_depth, + record_gens=None, + ): + seen["fid"] = fid + seen["test_cmd"] = test_cmd + seen["task"] = task + record_gens(4) + return (f"/wt/feat-{fid}", f"feat/{fid}", "[coder.solve rung=best-of-k gens=4] solved") + + monkeypatch.setattr(coder_seam, "_import_solve", lambda: object()) + monkeypatch.setattr(coder_seam, "dispatch", _fake_dispatch) + + async def _open_pr(wt, branch, *, base, title, body): + return "https://example/pr/42" + + loop, store = await _drive_with( + monkeypatch, open_pr=_open_pr, cfg={"coder": "proto", "local_gate_cmd": "pytest -q"}, gate=_pass_gate + ) + assert seen["fid"] == "bd-1" and seen["test_cmd"] == "pytest -q" + assert "Add a thing" in seen["task"] # the same built prompt, not a different one + assert store.gens_spent.get("bd-1") == 4 + assert ("open_review", "bd-1", "https://example/pr/42") in store.calls + assert store.creates == [] # solve()'s own per-candidate worktrees replaced the single create + + +async def test_drive_falls_back_to_single_shot_without_acceptance_criteria(monkeypatch): + """Honest degrade: even with the coder plugin available and a test command + configured, a feature with NO acceptance criteria takes today's single + delegate_to(acp) shot — never a silent best-of-k.""" + from project_board import coder_seam + + monkeypatch.setattr(coder_seam, "_import_solve", lambda: object()) + + async def _boom(**kw): + raise AssertionError("coder.solve must not run without acceptance criteria") + + monkeypatch.setattr(coder_seam, "dispatch", _boom) + + async def _open_pr(wt, branch, *, base, title, body): + return "https://example/pr/1" + + feature = dict(FEATURE, acceptance_criteria="") + loop, store = await _drive_with( + monkeypatch, + open_pr=_open_pr, + cfg={"coder": "proto", "local_gate_cmd": "pytest -q"}, + feature=feature, + gate=_pass_gate, + ) + assert store.creates == ["bd-1"] # the plain single-worktree path ran + assert ("open_review", "bd-1", "https://example/pr/1") in store.calls + + +async def test_drive_falls_back_to_single_shot_when_coder_plugin_unavailable(monkeypatch): + """Honest degrade: acceptance criteria + a test command present, but `coder` + itself isn't installed/enabled — still the single shot, never a fake ladder.""" + from project_board import coder_seam + + monkeypatch.setattr(coder_seam, "_import_solve", lambda: None) + + async def _boom(**kw): + raise AssertionError("coder.solve must not run when the coder plugin is unavailable") + + monkeypatch.setattr(coder_seam, "dispatch", _boom) + + async def _open_pr(wt, branch, *, base, title, body): + return "https://example/pr/2" + + loop, store = await _drive_with( + monkeypatch, open_pr=_open_pr, cfg={"coder": "proto", "local_gate_cmd": "pytest -q"}, gate=_pass_gate + ) + assert store.creates == ["bd-1"] + assert ("open_review", "bd-1", "https://example/pr/2") in store.calls + + +async def test_drive_falls_back_to_single_shot_without_a_test_command(monkeypatch): + """Honest degrade: `coder` available + acceptance present, but NO test command + configured (no coder_solve_test_cmd, no local_gate_cmd) — no runnable oracle, so + the single shot runs rather than fake grounding.""" + from project_board import coder_seam + + monkeypatch.setattr(coder_seam, "_import_solve", lambda: object()) + + async def _boom(**kw): + raise AssertionError("coder.solve must not run with no runnable test command") + + monkeypatch.setattr(coder_seam, "dispatch", _boom) + + async def _open_pr(wt, branch, *, base, title, body): + return "https://example/pr/3" + + loop, store = await _drive_with(monkeypatch, open_pr=_open_pr, cfg={"coder": "proto"}) # no test cmd anywhere + assert store.creates == ["bd-1"] + assert ("open_review", "bd-1", "https://example/pr/3") in store.calls + + +async def test_drive_coder_solve_exhausted_blocks_like_a_capability_failure(monkeypatch): + """A SolveExhausted (no candidate passed the acceptance tests) is treated exactly + like NoChangesError/CoderTimeout — blocked immediately with no ladder configured.""" + from project_board import coder_seam + + async def _exhausted(**kw): + raise coder_seam.SolveExhausted("coder.solve exhausted after 6 generation(s) (rung=best-partial): 1/3 failing") + + monkeypatch.setattr(coder_seam, "_import_solve", lambda: object()) + monkeypatch.setattr(coder_seam, "dispatch", _exhausted) + + async def _open_pr(wt, branch, *, base, title, body): + raise AssertionError("open_pr should not run — no candidate passed") + + loop, store = await _drive_with( + monkeypatch, open_pr=_open_pr, cfg={"coder": "proto", "local_gate_cmd": "pytest -q"} + ) + assert "flag_blocked" in store.names() + assert "open_review" not in store.names() + assert loop._inflight == {} + + +async def test_drive_coder_solve_skipped_on_a_carried_forward_ci_bounce(monkeypatch): + """A CI-bounce re-dispatch (signalled by _ci_feedback) fixes the EXISTING diff + with the single coder — coder.solve must not re-fan-out on that retry, same rule + as Max-Mode.""" + from project_board import coder_seam + + monkeypatch.setattr(coder_seam, "_import_solve", lambda: object()) + + async def _boom(**kw): + raise AssertionError("coder.solve must not run on a carried-forward re-dispatch") + + monkeypatch.setattr(coder_seam, "dispatch", _boom) + + async def _open_pr(wt, branch, *, base, title, body): + return "https://example/pr/9" + + loop, store = await _drive_with( + monkeypatch, + open_pr=_open_pr, + cfg={"coder": "proto", "local_gate_cmd": "pytest -q"}, + seed=lambda lp: lp._ci_feedback.__setitem__("bd-1", "CI failed: lint"), + gate=_pass_gate, + ) + assert store.creates == ["bd-1"] # the plain single-worktree path ran, not solve() + assert ("open_review", "bd-1", "https://example/pr/9") in store.calls + + +async def test_drive_max_mode_wins_precedence_over_coder_solve_when_both_configured(monkeypatch): + """A board already running Max-Mode (`max_mode_n>1`) must keep fanning out N + candidates and judging, NOT silently switch to coder.solve's ladder, even once + the `coder` plugin becomes importable and every one of + coder_seam.should_use_solve's gates (acceptance criteria + a runnable test + command) is satisfied. Pins the fix for the precedence bug: coder.solve only + preempts Max-Mode when max_mode_n<=1. (Uses `coder_solve_test_cmd`, not + `local_gate_cmd`, to satisfy the test-command gate without also flipping + Max-Mode's OWN candidate-selection strategy from judge to execution-grounded — + that's an orthogonal knob this test isn't about.)""" + from project_board import coder_seam + + monkeypatch.setattr(coder_seam, "_import_solve", lambda: object()) # coder plugin available + + async def _boom(**kw): + raise AssertionError("coder.solve must not run — Max-Mode has precedence when max_mode_n>1") + + monkeypatch.setattr(coder_seam, "dispatch", _boom) + + async def _open_pr(wt, branch, *, base, title, body): + return "https://example/pr/11" + + async def _judge(feature, base, worktrees): + assert len(worktrees) == 3 # Max-Mode's fan-out ran, not solve() + return 0 + + loop, store = await _drive_with( + monkeypatch, + open_pr=_open_pr, + judge=_judge, + cfg={"coder": "proto", "max_mode_n": 3, "coder_solve_test_cmd": "pytest -q"}, + ) + assert store.creates == ["bd-1.c0", "bd-1.c1", "bd-1.c2"] # Max-Mode's candidates, not solve()'s + assert ("open_review", "bd-1", "https://example/pr/11") in store.calls + + # ── goal-verification gate (MiMo-borrowed; opt-in `goal_verify`) ───────────────── diff --git a/tests/test_store.py b/tests/test_store.py index ab245e3..52d7148 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -148,6 +148,28 @@ def test_next_tier_walks_then_stops_at_the_top(make_board): assert b.next_tier("fast") == store.TIER_LADDER[0] # a now-removed tier falls back to the floor +# ── coder.solve() cost accounting (ADR 0064 P2 board seam) ────────────────────── + + +def test_record_gens_spent_adds_a_fresh_label(make_board, monkeypatch): + br = Br() + b = make_board(br) + monkeypatch.setattr(b, "get_feature", lambda fid: {"id": fid, "labels": []}) + monkeypatch.setattr(b, "_require", lambda fid: {"id": fid, "gens_spent": 0, "labels": []}) + b.record_gens_spent("bd-1", 3) + assert ("update", "bd-1", "--add-label", "gens:3") in br.calls + + +def test_record_gens_spent_accumulates_and_replaces_the_old_label(make_board, monkeypatch): + br = Br() + b = make_board(br) + monkeypatch.setattr(b, "get_feature", lambda fid: {"id": fid, "labels": []}) + monkeypatch.setattr(b, "_require", lambda fid: {"id": fid, "gens_spent": 5, "labels": ["gens:5", "ready"]}) + b.record_gens_spent("bd-1", 4) + # the stale gens:5 label is removed and replaced by the new cumulative total + assert ("update", "bd-1", "--remove-label", "gens:5", "--add-label", "gens:9") in br.calls + + # ── _project: the bead → feature view mapping ─────────────────────────────────── @@ -171,6 +193,12 @@ def test_project_maps_labels_notes_and_external_ref(make_board): assert f["attempts"] == [1, 2] # sorted ints assert f["pr_url"] == "https://example/pr/1" assert f["repo"] == "/repo" and f["base_branch"] == "main" + assert f["gens_spent"] == 0 # no gens: label → coder.solve() never touched this feature + + +def test_project_exposes_gens_spent_from_the_label(make_board): + b = make_board(Br()) + assert b._project({"id": "bd-2", "status": "open", "labels": ["gens:11"]})["gens_spent"] == 11 def test_project_marks_dag_blocked_when_a_blocks_dep_is_open(make_board):