fix(judge): a judge outage is unjudged, not an agent failure

docs/cli.md

@@ -79,6 +79,15 @@ Execution:
 
 `--harness`, `--judge`, `--no-judge`, `--no-upload`, and the `--browser-*` flags behave as in `clawbench-run`. A `batch-summary.json` is written alongside the per-run directories.
 
+### Exit codes
+
+| Code | Meaning |
+| --- | --- |
+| 0 | Intercepted, and judged a match (or `--no-judge`) |
+| 1 | The run failed: not intercepted, or the judge returned a mismatch |
+| 2 | The run itself errored before producing a result |
+| 3 | Intercepted, but the judge returned no verdict — a judge outage, not a model failure. `clawbench-batch` counts these as `unjudged`; re-judge them with `clawbench-rescore --only-unjudged` |
+
 ## `clawbench-rescore`
 
 Re-judge existing trajectories — no browser, no agent compute.
@@ -95,6 +104,7 @@ clawbench-rescore <run-or-batch-dir> --judge-model deepseek-v4-pro --rubric both
 | `--force` | off | Re-judge tasks that already have a verdict for this rubric |
 | `--limit <n>` | 0 (all) | Judge at most *n* tasks |
 | `--only-batch <name>` | — | Restrict to one batch inside a sweep |
+| `--only-unjudged` | off | Only re-score runs that intercepted but have no judge verdict — what a judge outage leaves behind |
 | `--eval-results-dir <path>` | `./eval_results` | Where per-task CSV + `summary.json` are written |
 | `--no-eval-results` | off | Skip writing the `eval_results/` artifact |
 | `--models-yaml <path>`, `--sweep-root <path>` | — | Override config / sweep locations |

src/clawbench/eval/rescore.py

@@ -38,11 +38,33 @@
 
 import yaml
 
+# run_support.results holds no container-engine probe, so importing it here
+# keeps rescore usable on a host with neither Docker nor Podman.
+from clawbench.runner.run_support.results import is_judge_inconclusive
+
 JUDGE_FILE = {"strict": "judge.json", "lenient": "judge_llm.json"}
 
 
-def find_run_dirs(root: Path) -> list[Path]:
-    return [p.parent for p in root.rglob("run-meta.json")]
+def find_run_dirs(root: Path, only_unjudged: bool = False) -> list[Path]:
+    """Run directories under `root`, optionally only those still unjudged.
+
+    A judge outage leaves an otherwise complete run with no verdict. Those are
+    the runs worth re-judging after the provider recovers, and re-judging only
+    them costs a fraction of a full sweep.
+    """
+    run_dirs = [p.parent for p in root.rglob("run-meta.json")]
+    if not only_unjudged:
+        return run_dirs
+
+    selected = []
+    for run_dir in run_dirs:
+        try:
+            meta = json.loads((run_dir / "run-meta.json").read_text(encoding="utf-8"))
+        except (OSError, ValueError):
+            continue
+        if isinstance(meta, dict) and is_judge_inconclusive(meta):
+            selected.append(run_dir)
+    return selected
 
 
 def rescore_one(
@@ -242,6 +264,15 @@ def main() -> int:
     )
     p.add_argument("--limit", type=int, default=0)
     p.add_argument("--only-batch", type=Path, default=None)
+    p.add_argument(
+        "--only-unjudged",
+        action="store_true",
+        help=(
+            "Only re-score runs that intercepted but have no judge verdict "
+            "(judge_match null) -- what a judge outage leaves behind. Use "
+            "after the provider recovers to fill in exactly those runs."
+        ),
+    )
     args = p.parse_args()
 
     cfg_all = yaml.safe_load(args.models_yaml.read_text())
@@ -267,11 +298,11 @@ def main() -> int:
 
         judge_funcs["lenient"] = judge_lenient
 
-    run_dirs = (
-        find_run_dirs(args.only_batch)
-        if args.only_batch
-        else find_run_dirs(args.sweep_root)
-    )
+    root = args.only_batch or args.sweep_root
+    run_dirs = find_run_dirs(root, only_unjudged=args.only_unjudged)
+    if args.only_unjudged and not run_dirs:
+        print(f"No runs with a missing judge verdict under {root}")
+        return 0
 
     pending = []
     for rd in run_dirs:

src/clawbench/runner/batch.py

@@ -17,6 +17,7 @@
 
 import yaml
 
+from clawbench.runner.run_support.results import JUDGE_INCONCLUSIVE_EXIT
 from clawbench.utils.paths import ASSET_ROOT, WORKSPACE_ROOT, ensure_workspace_templates
 
 
@@ -194,6 +195,12 @@ class Job:
     proc: asyncio.subprocess.Process | None = field(default=None, repr=False)
 
 
+# Terminal job statuses, in reporting order. "unjudged" is a run that
+# intercepted but whose stage-2 verdict never arrived; it is neither a pass nor
+# a model failure, and it is the set rescore should re-judge.
+JOB_STATUSES = ("passed", "failed", "unjudged", "error", "skipped")
+
+
 def fmt_duration(s: float) -> str:
     m, sec = divmod(int(s), 60)
     return f"{m}m{sec:02d}s"
@@ -326,6 +333,10 @@ async def run_job(
                     job.status = "passed"
                 elif proc.returncode == 1:
                     job.status = "failed"
+                elif proc.returncode == JUDGE_INCONCLUSIVE_EXIT:
+                    # The run is fine; only the verdict is missing. Counting it
+                    # as "failed" silently deflates the batch's reward.
+                    job.status = "unjudged"
                 else:
                     job.status = "error"
             except asyncio.CancelledError:
@@ -409,11 +420,7 @@ def print_summary(
     totals = {}
     for j in jobs:
         totals[j.status] = totals.get(j.status, 0) + 1
-    parts = [
-        f"{totals.get(s, 0)} {s}"
-        for s in ("passed", "failed", "error", "skipped")
-        if totals.get(s)
-    ]
+    parts = [f"{totals.get(s, 0)} {s}" for s in JOB_STATUSES if totals.get(s)]
     print(f"\nTotal: {len(jobs)} jobs | {' | '.join(parts)}")
     print(f"Total elapsed: {fmt_duration(elapsed)} (max_concurrent={max_concurrent})")
 
@@ -574,10 +581,7 @@ def write_summary_json(
             }
             for j in jobs
         ],
-        "totals": {
-            s: sum(1 for j in jobs if j.status == s)
-            for s in ("passed", "failed", "error", "skipped")
-        },
+        "totals": {s: sum(1 for j in jobs if j.status == s) for s in JOB_STATUSES},
     }
     (base_output / "batch-summary.json").write_text(json.dumps(data, indent=2))
 

src/clawbench/runner/run.py

@@ -53,6 +53,7 @@
 from clawbench.runner.run_support.email import create_email, delete_email
 from clawbench.runner.run_support.metadata import make_run_meta, write_run_meta
 from clawbench.runner.run_support.results import (
+    JUDGE_INCONCLUSIVE_EXIT,
     classify_run,
     ensure_interception,
     print_results,
@@ -794,7 +795,12 @@ def handle_sigint(sig, frame):
             f"\nINTERCEPTED but JUDGE {'MISMATCH' if verdict is False else 'INCONCLUSIVE'} "
             f"— results in {output_dir}\n  reason: {reason[:200]}"
         )
-        sys.exit(1)
+        # An inconclusive judge is a missing verdict, not a failed task: the
+        # agent did intercept. Exiting 1 here made a judge outage -- an HTTP
+        # 402 mid-sweep, a timeout -- indistinguishable from the model
+        # failing, and whole batches were recorded as failures. A distinct
+        # code lets batch.py count these apart and rescore target them.
+        sys.exit(1 if verdict is False else JUDGE_INCONCLUSIVE_EXIT)
     if final_pass:
         status = "INTERCEPTED" if args.no_judge else "INTERCEPTED + JUDGE MATCH"
         print(f"\n{status} — results in {output_dir}")

src/clawbench/runner/run_support/results.py

@@ -57,6 +57,30 @@
     " 429 ",
 )
 
+# clawbench-run exit code for "intercepted, but the judge returned no verdict".
+# Distinct from 1 (the agent genuinely failed stage 1 or the judge said no) so a
+# judge outage is not recorded as the model having failed the task.
+JUDGE_INCONCLUSIVE_EXIT = 3
+
+
+def is_judge_inconclusive(meta: dict[str, Any]) -> bool:
+    """Did stage 2 fail to produce a verdict on an otherwise complete run?
+
+    judge.py returns match=None when the call fails after retries -- HTTP 402,
+    a timeout, an unsupported api_type. The run itself is fine; what is missing
+    is the judgement, so it needs re-judging rather than counting against the
+    model.
+
+    A run that was never judged (--no-judge) has no judge_match key and is not
+    inconclusive: nothing was attempted, so nothing is outstanding.
+    """
+    if not meta.get("intercepted"):
+        return False
+    if "judge_match" not in meta:
+        return False
+    return meta["judge_match"] is None
+
+
 NON_MODEL_FAILURE_CATEGORIES = {
     "infra_failure",
     "api_or_credit",

tests/test_judge_inconclusive.py

@@ -0,0 +1,171 @@
+"""A judge outage is a missing verdict, not a failed task.
+
+judge.py returns match=None when the call fails after retries -- HTTP 402, a
+timeout, an unsupported api_type. run.py exited 1 for that, batch.py counts
+exit 1 as "failed", and so a judge account hitting 402 mid-sweep recorded whole
+batches as agent failures, silently deflating reward. The run itself was fine;
+only stage 2 was missing.
+"""
+
+from __future__ import annotations
+
+import inspect
+import json
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+
+from clawbench.eval import rescore
+from clawbench.runner.batch import JOB_STATUSES, Job, write_summary_json
+from clawbench.runner.run_support.results import (
+    JUDGE_INCONCLUSIVE_EXIT,
+    is_judge_inconclusive,
+)
+
+RUN_PY = Path(__file__).resolve().parents[1] / "src" / "clawbench" / "runner" / "run.py"
+
+
+# --- what counts as inconclusive ---------------------------------------------
+
+
+def test_a_judge_outage_on_an_intercepted_run_is_inconclusive() -> None:
+    assert is_judge_inconclusive({"intercepted": True, "judge_match": None})
+
+
+@pytest.mark.parametrize(
+    "meta",
+    [
+        {"intercepted": True, "judge_match": True},
+        {"intercepted": True, "judge_match": False},
+    ],
+)
+def test_a_real_verdict_is_not_inconclusive(meta: dict) -> None:
+    """A judge that said "no" answered the question. Only a judge that could
+    not answer leaves the run outstanding."""
+    assert not is_judge_inconclusive(meta)
+
+
+def test_no_judge_is_not_inconclusive() -> None:
+    """--no-judge attempted nothing, so nothing is outstanding; without this the
+    whole --no-judge corpus would look like it needed re-judging."""
+    assert not is_judge_inconclusive({"intercepted": True})
+
+
+def test_a_run_that_never_intercepted_is_not_inconclusive() -> None:
+    """Stage 1 already decided it. Re-judging cannot change that."""
+    assert not is_judge_inconclusive({"intercepted": False, "judge_match": None})
+
+
+# --- run.py's exit code -------------------------------------------------------
+
+
+def test_the_exit_code_is_distinct_from_agent_failure() -> None:
+    assert JUDGE_INCONCLUSIVE_EXIT not in (0, 1, 2)
+
+
+def test_run_exits_the_distinct_code_only_when_the_verdict_is_missing() -> None:
+    """The branch already printed MISMATCH vs INCONCLUSIVE but exited 1 for
+    both. Read from source rather than imported: run.py pulls in
+    run_support.config, which probes for a container engine at import time and
+    exits when none is installed (#315)."""
+    src = RUN_PY.read_text(encoding="utf-8")
+
+    assert "sys.exit(1 if verdict is False else JUDGE_INCONCLUSIVE_EXIT)" in src
+    assert "JUDGE_INCONCLUSIVE_EXIT," in src  # imported, not redefined
+
+
+# --- batch.py's accounting ----------------------------------------------------
+
+
+def test_unjudged_is_a_status_of_its_own() -> None:
+    assert "unjudged" in JOB_STATUSES
+    for expected in ("passed", "failed", "error", "skipped"):
+        assert expected in JOB_STATUSES, expected
+
+
+def test_the_summary_counts_unjudged_apart_from_failed(tmp_path: Path) -> None:
+    """The reported bug: "N runs unjudged" was indistinguishable from "N runs
+    the model failed" in batch-summary.json."""
+    jobs = [
+        Job(model="glm-5.1", case_dir=Path("c"), case_name="a", status="passed"),
+        Job(model="glm-5.1", case_dir=Path("c"), case_name="b", status="failed"),
+        Job(model="glm-5.1", case_dir=Path("c"), case_name="c", status="unjudged"),
+        Job(model="glm-5.1", case_dir=Path("c"), case_name="d", status="unjudged"),
+    ]
+
+    write_summary_json(jobs, tmp_path, 1.0, 2, "2026-01-01T00:00:00+00:00")
+    totals = json.loads((tmp_path / "batch-summary.json").read_text())["totals"]
+
+    assert totals["unjudged"] == 2
+    assert totals["failed"] == 1
+    assert totals["passed"] == 1
+
+
+def test_batch_maps_the_exit_code_to_unjudged() -> None:
+    """Guard the wiring between the two modules: run.py's code and batch.py's
+    status have to stay in agreement or the count silently reverts to failed."""
+    from clawbench.runner import batch as batch_mod
+
+    src = inspect.getsource(batch_mod.run_job)
+
+    assert "elif proc.returncode == JUDGE_INCONCLUSIVE_EXIT:" in src
+    assert 'job.status = "unjudged"' in src
+
+
+# --- rescore --only-unjudged --------------------------------------------------
+
+
+def _write_run(base: Path, name: str, meta: dict) -> Path:
+    run_dir = base / name
+    run_dir.mkdir(parents=True, exist_ok=True)
+    (run_dir / "run-meta.json").write_text(json.dumps(meta))
+    return run_dir
+
+
+def test_only_unjudged_selects_exactly_the_runs_missing_a_verdict(
+    tmp_path: Path,
+) -> None:
+    outage = _write_run(tmp_path, "outage", {"intercepted": True, "judge_match": None})
+    _write_run(tmp_path, "matched", {"intercepted": True, "judge_match": True})
+    _write_run(tmp_path, "mismatch", {"intercepted": True, "judge_match": False})
+    _write_run(tmp_path, "not-intercepted", {"intercepted": False})
+
+    assert rescore.find_run_dirs(tmp_path, only_unjudged=True) == [outage]
+
+
+def test_without_the_flag_every_run_is_still_returned(tmp_path: Path) -> None:
+    _write_run(tmp_path, "outage", {"intercepted": True, "judge_match": None})
+    _write_run(tmp_path, "matched", {"intercepted": True, "judge_match": True})
+
+    assert len(rescore.find_run_dirs(tmp_path)) == 2
+
+
+def test_an_unreadable_run_meta_is_skipped_not_crashed_on(tmp_path: Path) -> None:
+    bad = _write_run(tmp_path, "bad", {"intercepted": True, "judge_match": None})
+    (bad / "run-meta.json").write_text('{"intercepted": tr')
+
+    assert rescore.find_run_dirs(tmp_path, only_unjudged=True) == []
+
+
+def test_rescore_still_does_not_need_a_container_engine() -> None:
+    """rescore imports run_support.results now. That module must stay free of
+    the import-time engine probe, or a post-hoc scoring tool starts requiring
+    Docker. Imported in a subprocess with an empty PATH so the probe would
+    actually fire if it were reachable."""
+    result = subprocess.run(
+        [sys.executable, "-c", "import clawbench.eval.rescore"],
+        capture_output=True,
+        text=True,
+        env={
+            **os.environ,
+            "PATH": "",
+            "CONTAINER_ENGINE": "",
+            "PYTHONPATH": str(RUN_PY.parents[2]),
+        },
+    )
+
+    assert result.returncode == 0, result.stdout + result.stderr
+    assert "not found on PATH" not in result.stdout

tests/test_run_judge_stage.py

@@ -30,6 +30,8 @@
 
 import pytest
 
+from clawbench.runner.run_support.results import JUDGE_INCONCLUSIVE_EXIT
+
 
 def _import_run_module(monkeypatch: pytest.MonkeyPatch) -> ModuleType:
     for module_name in (
@@ -237,8 +239,10 @@ def explode(*args: Any, **kwargs: Any) -> dict:
     with pytest.raises(SystemExit) as excinfo:
         run_mod.main()
 
-    # Inconclusive judge -> normal exit 1, not an uncaught crash.
-    assert excinfo.value.code == 1
+    # A clean exit, not an uncaught crash. #299 gave the inconclusive verdict
+    # its own code so a judge outage is not counted as the model failing; this
+    # run intercepted, and only stage 2 is missing.
+    assert excinfo.value.code == JUDGE_INCONCLUSIVE_EXIT
     assert docker_calls == ["run"]  # the agent did run
 
     # The core regression: its results must not be silently discarded.