diff --git a/docs/cli.md b/docs/cli.md index 9f3bb254..ab636dfd 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -79,6 +79,15 @@ Execution: `--harness`, `--judge`, `--no-judge`, `--no-upload`, and the `--browser-*` flags behave as in `clawbench-run`. A `batch-summary.json` is written alongside the per-run directories. +### Exit codes + +| Code | Meaning | +| --- | --- | +| 0 | Intercepted, and judged a match (or `--no-judge`) | +| 1 | The run failed: not intercepted, or the judge returned a mismatch | +| 2 | The run itself errored before producing a result | +| 3 | Intercepted, but the judge returned no verdict — a judge outage, not a model failure. `clawbench-batch` counts these as `unjudged`; re-judge them with `clawbench-rescore --only-unjudged` | + ## `clawbench-rescore` Re-judge existing trajectories — no browser, no agent compute. @@ -95,6 +104,7 @@ clawbench-rescore --judge-model deepseek-v4-pro --rubric both | `--force` | off | Re-judge tasks that already have a verdict for this rubric | | `--limit ` | 0 (all) | Judge at most *n* tasks | | `--only-batch ` | — | Restrict to one batch inside a sweep | +| `--only-unjudged` | off | Only re-score runs that intercepted but have no judge verdict — what a judge outage leaves behind | | `--eval-results-dir ` | `./eval_results` | Where per-task CSV + `summary.json` are written | | `--no-eval-results` | off | Skip writing the `eval_results/` artifact | | `--models-yaml `, `--sweep-root ` | — | Override config / sweep locations | diff --git a/src/clawbench/eval/rescore.py b/src/clawbench/eval/rescore.py index 3df30a75..2a73cde6 100644 --- a/src/clawbench/eval/rescore.py +++ b/src/clawbench/eval/rescore.py @@ -38,11 +38,33 @@ import yaml +# run_support.results holds no container-engine probe, so importing it here +# keeps rescore usable on a host with neither Docker nor Podman. +from clawbench.runner.run_support.results import is_judge_inconclusive + JUDGE_FILE = {"strict": "judge.json", "lenient": "judge_llm.json"} -def find_run_dirs(root: Path) -> list[Path]: - return [p.parent for p in root.rglob("run-meta.json")] +def find_run_dirs(root: Path, only_unjudged: bool = False) -> list[Path]: + """Run directories under `root`, optionally only those still unjudged. + + A judge outage leaves an otherwise complete run with no verdict. Those are + the runs worth re-judging after the provider recovers, and re-judging only + them costs a fraction of a full sweep. + """ + run_dirs = [p.parent for p in root.rglob("run-meta.json")] + if not only_unjudged: + return run_dirs + + selected = [] + for run_dir in run_dirs: + try: + meta = json.loads((run_dir / "run-meta.json").read_text(encoding="utf-8")) + except (OSError, ValueError): + continue + if isinstance(meta, dict) and is_judge_inconclusive(meta): + selected.append(run_dir) + return selected def rescore_one( @@ -242,6 +264,15 @@ def main() -> int: ) p.add_argument("--limit", type=int, default=0) p.add_argument("--only-batch", type=Path, default=None) + p.add_argument( + "--only-unjudged", + action="store_true", + help=( + "Only re-score runs that intercepted but have no judge verdict " + "(judge_match null) -- what a judge outage leaves behind. Use " + "after the provider recovers to fill in exactly those runs." + ), + ) args = p.parse_args() cfg_all = yaml.safe_load(args.models_yaml.read_text()) @@ -267,11 +298,11 @@ def main() -> int: judge_funcs["lenient"] = judge_lenient - run_dirs = ( - find_run_dirs(args.only_batch) - if args.only_batch - else find_run_dirs(args.sweep_root) - ) + root = args.only_batch or args.sweep_root + run_dirs = find_run_dirs(root, only_unjudged=args.only_unjudged) + if args.only_unjudged and not run_dirs: + print(f"No runs with a missing judge verdict under {root}") + return 0 pending = [] for rd in run_dirs: diff --git a/src/clawbench/runner/batch.py b/src/clawbench/runner/batch.py index 8d3637e5..d8f4318f 100644 --- a/src/clawbench/runner/batch.py +++ b/src/clawbench/runner/batch.py @@ -17,6 +17,7 @@ import yaml +from clawbench.runner.run_support.results import JUDGE_INCONCLUSIVE_EXIT from clawbench.utils.paths import ASSET_ROOT, WORKSPACE_ROOT, ensure_workspace_templates @@ -194,6 +195,12 @@ class Job: proc: asyncio.subprocess.Process | None = field(default=None, repr=False) +# Terminal job statuses, in reporting order. "unjudged" is a run that +# intercepted but whose stage-2 verdict never arrived; it is neither a pass nor +# a model failure, and it is the set rescore should re-judge. +JOB_STATUSES = ("passed", "failed", "unjudged", "error", "skipped") + + def fmt_duration(s: float) -> str: m, sec = divmod(int(s), 60) return f"{m}m{sec:02d}s" @@ -326,6 +333,10 @@ async def run_job( job.status = "passed" elif proc.returncode == 1: job.status = "failed" + elif proc.returncode == JUDGE_INCONCLUSIVE_EXIT: + # The run is fine; only the verdict is missing. Counting it + # as "failed" silently deflates the batch's reward. + job.status = "unjudged" else: job.status = "error" except asyncio.CancelledError: @@ -409,11 +420,7 @@ def print_summary( totals = {} for j in jobs: totals[j.status] = totals.get(j.status, 0) + 1 - parts = [ - f"{totals.get(s, 0)} {s}" - for s in ("passed", "failed", "error", "skipped") - if totals.get(s) - ] + parts = [f"{totals.get(s, 0)} {s}" for s in JOB_STATUSES if totals.get(s)] print(f"\nTotal: {len(jobs)} jobs | {' | '.join(parts)}") print(f"Total elapsed: {fmt_duration(elapsed)} (max_concurrent={max_concurrent})") @@ -574,10 +581,7 @@ def write_summary_json( } for j in jobs ], - "totals": { - s: sum(1 for j in jobs if j.status == s) - for s in ("passed", "failed", "error", "skipped") - }, + "totals": {s: sum(1 for j in jobs if j.status == s) for s in JOB_STATUSES}, } (base_output / "batch-summary.json").write_text(json.dumps(data, indent=2)) diff --git a/src/clawbench/runner/run.py b/src/clawbench/runner/run.py index 4c9bf3d3..f2863487 100644 --- a/src/clawbench/runner/run.py +++ b/src/clawbench/runner/run.py @@ -53,6 +53,7 @@ from clawbench.runner.run_support.email import create_email, delete_email from clawbench.runner.run_support.metadata import make_run_meta, write_run_meta from clawbench.runner.run_support.results import ( + JUDGE_INCONCLUSIVE_EXIT, classify_run, ensure_interception, print_results, @@ -794,7 +795,12 @@ def handle_sigint(sig, frame): f"\nINTERCEPTED but JUDGE {'MISMATCH' if verdict is False else 'INCONCLUSIVE'} " f"— results in {output_dir}\n reason: {reason[:200]}" ) - sys.exit(1) + # An inconclusive judge is a missing verdict, not a failed task: the + # agent did intercept. Exiting 1 here made a judge outage -- an HTTP + # 402 mid-sweep, a timeout -- indistinguishable from the model + # failing, and whole batches were recorded as failures. A distinct + # code lets batch.py count these apart and rescore target them. + sys.exit(1 if verdict is False else JUDGE_INCONCLUSIVE_EXIT) if final_pass: status = "INTERCEPTED" if args.no_judge else "INTERCEPTED + JUDGE MATCH" print(f"\n{status} — results in {output_dir}") diff --git a/src/clawbench/runner/run_support/results.py b/src/clawbench/runner/run_support/results.py index 3d795d1d..3bf788b5 100644 --- a/src/clawbench/runner/run_support/results.py +++ b/src/clawbench/runner/run_support/results.py @@ -57,6 +57,30 @@ " 429 ", ) +# clawbench-run exit code for "intercepted, but the judge returned no verdict". +# Distinct from 1 (the agent genuinely failed stage 1 or the judge said no) so a +# judge outage is not recorded as the model having failed the task. +JUDGE_INCONCLUSIVE_EXIT = 3 + + +def is_judge_inconclusive(meta: dict[str, Any]) -> bool: + """Did stage 2 fail to produce a verdict on an otherwise complete run? + + judge.py returns match=None when the call fails after retries -- HTTP 402, + a timeout, an unsupported api_type. The run itself is fine; what is missing + is the judgement, so it needs re-judging rather than counting against the + model. + + A run that was never judged (--no-judge) has no judge_match key and is not + inconclusive: nothing was attempted, so nothing is outstanding. + """ + if not meta.get("intercepted"): + return False + if "judge_match" not in meta: + return False + return meta["judge_match"] is None + + NON_MODEL_FAILURE_CATEGORIES = { "infra_failure", "api_or_credit", diff --git a/tests/test_judge_inconclusive.py b/tests/test_judge_inconclusive.py new file mode 100644 index 00000000..bbee7ca0 --- /dev/null +++ b/tests/test_judge_inconclusive.py @@ -0,0 +1,171 @@ +"""A judge outage is a missing verdict, not a failed task. + +judge.py returns match=None when the call fails after retries -- HTTP 402, a +timeout, an unsupported api_type. run.py exited 1 for that, batch.py counts +exit 1 as "failed", and so a judge account hitting 402 mid-sweep recorded whole +batches as agent failures, silently deflating reward. The run itself was fine; +only stage 2 was missing. +""" + +from __future__ import annotations + +import inspect +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +from clawbench.eval import rescore +from clawbench.runner.batch import JOB_STATUSES, Job, write_summary_json +from clawbench.runner.run_support.results import ( + JUDGE_INCONCLUSIVE_EXIT, + is_judge_inconclusive, +) + +RUN_PY = Path(__file__).resolve().parents[1] / "src" / "clawbench" / "runner" / "run.py" + + +# --- what counts as inconclusive --------------------------------------------- + + +def test_a_judge_outage_on_an_intercepted_run_is_inconclusive() -> None: + assert is_judge_inconclusive({"intercepted": True, "judge_match": None}) + + +@pytest.mark.parametrize( + "meta", + [ + {"intercepted": True, "judge_match": True}, + {"intercepted": True, "judge_match": False}, + ], +) +def test_a_real_verdict_is_not_inconclusive(meta: dict) -> None: + """A judge that said "no" answered the question. Only a judge that could + not answer leaves the run outstanding.""" + assert not is_judge_inconclusive(meta) + + +def test_no_judge_is_not_inconclusive() -> None: + """--no-judge attempted nothing, so nothing is outstanding; without this the + whole --no-judge corpus would look like it needed re-judging.""" + assert not is_judge_inconclusive({"intercepted": True}) + + +def test_a_run_that_never_intercepted_is_not_inconclusive() -> None: + """Stage 1 already decided it. Re-judging cannot change that.""" + assert not is_judge_inconclusive({"intercepted": False, "judge_match": None}) + + +# --- run.py's exit code ------------------------------------------------------- + + +def test_the_exit_code_is_distinct_from_agent_failure() -> None: + assert JUDGE_INCONCLUSIVE_EXIT not in (0, 1, 2) + + +def test_run_exits_the_distinct_code_only_when_the_verdict_is_missing() -> None: + """The branch already printed MISMATCH vs INCONCLUSIVE but exited 1 for + both. Read from source rather than imported: run.py pulls in + run_support.config, which probes for a container engine at import time and + exits when none is installed (#315).""" + src = RUN_PY.read_text(encoding="utf-8") + + assert "sys.exit(1 if verdict is False else JUDGE_INCONCLUSIVE_EXIT)" in src + assert "JUDGE_INCONCLUSIVE_EXIT," in src # imported, not redefined + + +# --- batch.py's accounting ---------------------------------------------------- + + +def test_unjudged_is_a_status_of_its_own() -> None: + assert "unjudged" in JOB_STATUSES + for expected in ("passed", "failed", "error", "skipped"): + assert expected in JOB_STATUSES, expected + + +def test_the_summary_counts_unjudged_apart_from_failed(tmp_path: Path) -> None: + """The reported bug: "N runs unjudged" was indistinguishable from "N runs + the model failed" in batch-summary.json.""" + jobs = [ + Job(model="glm-5.1", case_dir=Path("c"), case_name="a", status="passed"), + Job(model="glm-5.1", case_dir=Path("c"), case_name="b", status="failed"), + Job(model="glm-5.1", case_dir=Path("c"), case_name="c", status="unjudged"), + Job(model="glm-5.1", case_dir=Path("c"), case_name="d", status="unjudged"), + ] + + write_summary_json(jobs, tmp_path, 1.0, 2, "2026-01-01T00:00:00+00:00") + totals = json.loads((tmp_path / "batch-summary.json").read_text())["totals"] + + assert totals["unjudged"] == 2 + assert totals["failed"] == 1 + assert totals["passed"] == 1 + + +def test_batch_maps_the_exit_code_to_unjudged() -> None: + """Guard the wiring between the two modules: run.py's code and batch.py's + status have to stay in agreement or the count silently reverts to failed.""" + from clawbench.runner import batch as batch_mod + + src = inspect.getsource(batch_mod.run_job) + + assert "elif proc.returncode == JUDGE_INCONCLUSIVE_EXIT:" in src + assert 'job.status = "unjudged"' in src + + +# --- rescore --only-unjudged -------------------------------------------------- + + +def _write_run(base: Path, name: str, meta: dict) -> Path: + run_dir = base / name + run_dir.mkdir(parents=True, exist_ok=True) + (run_dir / "run-meta.json").write_text(json.dumps(meta)) + return run_dir + + +def test_only_unjudged_selects_exactly_the_runs_missing_a_verdict( + tmp_path: Path, +) -> None: + outage = _write_run(tmp_path, "outage", {"intercepted": True, "judge_match": None}) + _write_run(tmp_path, "matched", {"intercepted": True, "judge_match": True}) + _write_run(tmp_path, "mismatch", {"intercepted": True, "judge_match": False}) + _write_run(tmp_path, "not-intercepted", {"intercepted": False}) + + assert rescore.find_run_dirs(tmp_path, only_unjudged=True) == [outage] + + +def test_without_the_flag_every_run_is_still_returned(tmp_path: Path) -> None: + _write_run(tmp_path, "outage", {"intercepted": True, "judge_match": None}) + _write_run(tmp_path, "matched", {"intercepted": True, "judge_match": True}) + + assert len(rescore.find_run_dirs(tmp_path)) == 2 + + +def test_an_unreadable_run_meta_is_skipped_not_crashed_on(tmp_path: Path) -> None: + bad = _write_run(tmp_path, "bad", {"intercepted": True, "judge_match": None}) + (bad / "run-meta.json").write_text('{"intercepted": tr') + + assert rescore.find_run_dirs(tmp_path, only_unjudged=True) == [] + + +def test_rescore_still_does_not_need_a_container_engine() -> None: + """rescore imports run_support.results now. That module must stay free of + the import-time engine probe, or a post-hoc scoring tool starts requiring + Docker. Imported in a subprocess with an empty PATH so the probe would + actually fire if it were reachable.""" + result = subprocess.run( + [sys.executable, "-c", "import clawbench.eval.rescore"], + capture_output=True, + text=True, + env={ + **os.environ, + "PATH": "", + "CONTAINER_ENGINE": "", + "PYTHONPATH": str(RUN_PY.parents[2]), + }, + ) + + assert result.returncode == 0, result.stdout + result.stderr + assert "not found on PATH" not in result.stdout diff --git a/tests/test_run_judge_stage.py b/tests/test_run_judge_stage.py index d911492f..20c7e791 100644 --- a/tests/test_run_judge_stage.py +++ b/tests/test_run_judge_stage.py @@ -30,6 +30,8 @@ import pytest +from clawbench.runner.run_support.results import JUDGE_INCONCLUSIVE_EXIT + def _import_run_module(monkeypatch: pytest.MonkeyPatch) -> ModuleType: for module_name in ( @@ -237,8 +239,10 @@ def explode(*args: Any, **kwargs: Any) -> dict: with pytest.raises(SystemExit) as excinfo: run_mod.main() - # Inconclusive judge -> normal exit 1, not an uncaught crash. - assert excinfo.value.code == 1 + # A clean exit, not an uncaught crash. #299 gave the inconclusive verdict + # its own code so a judge outage is not counted as the model failing; this + # run intercepted, and only stage 2 is missing. + assert excinfo.value.code == JUDGE_INCONCLUSIVE_EXIT assert docker_calls == ["run"] # the agent did run # The core regression: its results must not be silently discarded.