From 4936d982b81af9e725784c3d9d0b19f3b6754fe8 Mon Sep 17 00:00:00 2001 From: Ryan Stewart Date: Tue, 28 Jul 2026 16:18:16 -0400 Subject: [PATCH 01/26] feat: make Legal Agent Bench agent configurable Signed-off-by: Ryan Stewart --- benchmarks/legal_agent_bench/README.md | 221 +++- .../legal_agent_bench/config_claude_code.yaml | 25 + .../legal_agent_bench/config_codex.yaml | 25 + .../legal_agent_bench/config_hermes.yaml | 25 + benchmarks/legal_agent_bench/prepare.py | 6 +- .../legal_agent_bench/tests/test_prepare.py | 109 +- resources_servers/legal_agent_bench/README.md | 147 ++- .../configs/resources_only.yaml | 28 + .../legal_agent_bench/data/example.jsonl | 10 +- .../legal_agent_bench/prepare.py | 50 +- .../legal_agent_bench/tests/test_prepare.py | 27 +- .../scripts/claude_code_agent_deps.sh | 18 +- .../codex_agent/scripts/codex_agent_deps.sh | 33 + responses_api_agents/hermes_agent/README.md | 9 +- responses_api_agents/hermes_agent/app.py | 34 +- .../hermes_agent/tests/test_app.py | 48 +- .../legal_agent_bench_agent/.gitignore | 3 + .../legal_agent_bench_agent/README.md | 85 ++ .../legal_agent_bench_agent/__init__.py | 3 + .../legal_agent_bench_agent/app.py | 1121 +++++++++++++++++ .../legal_agent_bench_claude_code.yaml | 39 + .../configs/legal_agent_bench_codex.yaml | 39 + .../configs/legal_agent_bench_hermes.yaml | 39 + .../legal_agent_bench_agent/requirements.txt | 1 + .../setup_scripts/_portable_python.sh | 37 + .../legal_agent_bench_agent/tests/__init__.py | 2 + .../legal_agent_bench_agent/tests/test_app.py | 937 ++++++++++++++ 27 files changed, 3038 insertions(+), 83 deletions(-) create mode 100644 benchmarks/legal_agent_bench/config_claude_code.yaml create mode 100644 benchmarks/legal_agent_bench/config_codex.yaml create mode 100644 benchmarks/legal_agent_bench/config_hermes.yaml create mode 100644 resources_servers/legal_agent_bench/configs/resources_only.yaml create mode 100755 responses_api_agents/codex_agent/scripts/codex_agent_deps.sh create mode 100644 responses_api_agents/legal_agent_bench_agent/.gitignore create mode 100644 responses_api_agents/legal_agent_bench_agent/README.md create mode 100644 responses_api_agents/legal_agent_bench_agent/__init__.py create mode 100644 responses_api_agents/legal_agent_bench_agent/app.py create mode 100644 responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_claude_code.yaml create mode 100644 responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_codex.yaml create mode 100644 responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_hermes.yaml create mode 100644 responses_api_agents/legal_agent_bench_agent/requirements.txt create mode 100755 responses_api_agents/legal_agent_bench_agent/setup_scripts/_portable_python.sh create mode 100644 responses_api_agents/legal_agent_bench_agent/tests/__init__.py create mode 100644 responses_api_agents/legal_agent_bench_agent/tests/test_app.py diff --git a/benchmarks/legal_agent_bench/README.md b/benchmarks/legal_agent_bench/README.md index c25c1ef25e..ae1907ee51 100644 --- a/benchmarks/legal_agent_bench/README.md +++ b/benchmarks/legal_agent_bench/README.md @@ -1,13 +1,25 @@ # Legal Agent Bench Benchmark -This benchmark registers the existing +This benchmark registers the [Legal Agent Bench resource server](../../resources_servers/legal_agent_bench/README.md) -with Gym's benchmark catalog. It evaluates public Harvey LAB tasks through -the legal_agent_bench resource server's implementation. +with Gym's benchmark catalog. Every variant evaluates the same 1,749 public +Harvey LAB tasks with the same skills and verifier. -Benchmark preparation reuses -the resource server's pinned task and skill caches and writes only a small, -gitignored benchmark index. +This benchmark uses the Harbor integration by default. Three additional variants use the LAB-owned +[configurable runner](../../responses_api_agents/legal_agent_bench_agent/README.md) +with Gym's built-in agent harnesses: + +| Benchmark | Agent | +| --- | --- | +| `legal_agent_bench` | Harbor (default) | +| `legal_agent_bench/config_hermes` | Hermes | +| `legal_agent_bench/config_claude_code` | Claude Code | +| `legal_agent_bench/config_codex` | Codex | + +Benchmark preparation reuses the resource server's pinned task and skill +caches and copies a small, gitignored, deterministic index. Source, example, +and benchmark JSONL rows are agent-neutral. Gym stamps the variant's configured +agent during dataset collation. ## Requirements @@ -17,13 +29,43 @@ gitignored benchmark index. `env.yaml` - At least 10 GB of free working space +Not required: + +- Separate Harbor, Hermes, Claude Code, or Codex installations +- Anthropic or OpenAI vendor subscriptions or CLI logins for the Claude Code + and Codex harnesses + +Gym provisions the pinned harness dependencies automatically. Every harness +uses the configured policy model endpoint. Access to your configured policy and +judge endpoints is still required and may itself be metered or paid. + See the [resource-server README](../../resources_servers/legal_agent_bench/README.md) for endpoint configuration, source and license details, cache locations, and troubleshooting. The initial source download is several hundred MiB, and the first rollout builds a document-tooling Docker image that can take several minutes. -## Prepare +## Set up + +From the repository root: + +```bash +uv venv --python 3.12 +source .venv/bin/activate +uv sync --extra dev +docker info >/dev/null +``` + +Configure the policy and judge endpoints in the gitignored root `env.yaml` as +shown in the +[resource-server README](../../resources_servers/legal_agent_bench/README.md#requirements). +Keep the virtual environment activated for every command below. + +Use the `gym` executable from the activated environment directly. Do not prefix +these server-starting commands with `uv run`: Ray starts components from their +own working directories, which can conflict with uv's project discovery. + +## Prepare and validate From the repository root, run: @@ -31,43 +73,152 @@ From the repository root, run: gym eval prepare --benchmark legal_agent_bench ``` -This validates or prepares the shared task and skill caches, then writes the -deterministic benchmark index to +Preparation is shared by all four variants. It validates or prepares the task +and skill caches, then writes the deterministic benchmark index to `benchmarks/legal_agent_bench/data/legal_agent_bench_benchmark.jsonl`. Repeated preparation reuses valid caches and does not download a second copy of LAB. -## Run +Validate that all four configurations resolve before spending time on a +rollout: + +```bash +gym env validate --model-type vllm_model --benchmark legal_agent_bench +gym env validate --model-type vllm_model --benchmark legal_agent_bench/config_hermes +gym env validate --model-type vllm_model --benchmark legal_agent_bench/config_claude_code +gym env validate --model-type vllm_model --benchmark legal_agent_bench/config_codex +``` + +## Test the various harnesses -For the standard one-shot workflow: +Run these one at a time for each harness you want to test. Each command starts the required Gym services, runs +the first benchmark task, writes one JSONL result row, and stops the services. +Alongside the path passed to `--output`, Gym writes +`_materialized_inputs.jsonl` and +`_aggregate_metrics.json`. + +Harbor: ```bash gym eval run \ --model-type vllm_model \ --benchmark legal_agent_bench \ --split benchmark \ - --output results/legal_agent_bench_benchmark.jsonl \ - --concurrency 1 + --output results/legal_agent_bench_harbor_smoke.jsonl \ + --concurrency 1 \ + --limit 1 +``` + +Hermes: + +```bash +gym eval run \ + --model-type vllm_model \ + --benchmark legal_agent_bench/config_hermes \ + --split benchmark \ + --output results/legal_agent_bench_hermes_smoke.jsonl \ + --concurrency 1 \ + --limit 1 +``` + +Claude Code: + +```bash +gym eval run \ + --model-type vllm_model \ + --benchmark legal_agent_bench/config_claude_code \ + --split benchmark \ + --output results/legal_agent_bench_claude_code_smoke.jsonl \ + --concurrency 1 \ + --limit 1 +``` + +Codex: + +```bash +gym eval run \ + --model-type vllm_model \ + --benchmark legal_agent_bench/config_codex \ + --split benchmark \ + --output results/legal_agent_bench_codex_smoke.jsonl \ + --concurrency 1 \ + --limit 1 ``` -For a one-task smoke test, add `--limit 1`. +Hermes uses chat completions. Claude Code and Codex use their respective CLI +adapters against the configured OpenAI-compatible policy endpoint. The first +run of each configurable harness provisions its pinned portable runtime, so it +can be much slower than a cached run. The LAB runner disables Hermes's optional +pricing and context-metadata lookups because Gym already supplies the model and +its internal policy proxy does not expose `/models`; real model-call access +logging is enabled. + +## Check a configurable result -To manage the servers separately, start them first: +For Hermes, for example, inspect the result row and the persisted artifact bundle with: + +```bash +jq '{ + reward, + criteria_pass_rate, + mask_sample, + agent_failed, + model_connection_failed, + verifier_failed, + judge_error_count, + verifier_error, + artifact_dir +}' results/legal_agent_bench_hermes_smoke.jsonl + +ARTIFACT_DIR=$(jq -r '.artifact_dir' results/legal_agent_bench_hermes_smoke.jsonl) +jq . "$ARTIFACT_DIR/run_summary.json" +jq . "$ARTIFACT_DIR/agent/trajectory.json" +open "$ARTIFACT_DIR/verifier/report.html" # macOS; use xdg-open on Linux +``` + +Substitute the Claude Code or Codex output filename to inspect those runs. +A reliable completed rollout has `mask_sample: false`, no failure flags, +`judge_error_count: 0`, and `verifier_error: 0`. Check `output_files` in +`run_summary.json` for the deliverables. A `reward` of `0.0` can still be a +valid run: the default `full_task` reward requires every rubric criterion to +pass, while `criteria_pass_rate` shows partial success. + +Harbor keeps its detailed trial files under +`results/legal_agent_bench/harbor_jobs`; configurable variants use the +corresponding `hermes_jobs`, `claude_code_jobs`, or `codex_jobs` directory. +Their model directory comes from `policy_model_name`, with path separators and +other unsafe characters normalized for the filesystem. A session directory is +created only when its first rollout starts. Each configurable rollout is stored +under +`_jobs///_/`, where +the task name is normalized and the run ID is an eight-character unique suffix. +Harbor's result row uses the established Harbor response format rather than the +configurable runner's top-level flags. + +## Run a larger evaluation + +Remove `--limit 1` from the desired smoke command. Choose a new output filename +and increase `--concurrency` only after confirming that Docker, the policy +endpoint, and the judge endpoint can sustain it. + +## Manage servers separately + +To manage the servers separately, start the desired variant first: ```bash gym env start \ --model-type vllm_model \ - --benchmark legal_agent_bench + --benchmark legal_agent_bench/config_hermes ``` -Then run against them from a second activated terminal: +Then run against them with `--no-serve` from a second activated terminal: ```bash gym eval run --no-serve \ - --benchmark legal_agent_bench \ - --agent legal_agent_bench_benchmark_harbor_agent \ + --benchmark legal_agent_bench/config_hermes \ + --agent legal_agent_bench_benchmark_hermes_agent \ --input benchmarks/legal_agent_bench/data/legal_agent_bench_benchmark.jsonl \ - --output results/legal_agent_bench_benchmark.jsonl \ + --output results/legal_agent_bench_hermes_smoke.jsonl \ --concurrency 1 \ --limit 1 ``` @@ -86,6 +237,33 @@ pass rate instead, add this override to `gym env start` or the one-shot This changes only the reported reward; it does not change the tasks, agent, or judge criteria. +For configurable variants, the rollout also reports `agent_failed`, +`model_connection_failed`, `agent_timed_out`, `verifier_failed`, +`verifier_timed_out`, `sandbox_failed`, `task_failed`, +`configuration_failed`, `judge_error_count`, and `verifier_error`. +`task_failed` identifies an unsafe, unknown, incomplete, or malformed task; +`configuration_failed` identifies an invalid harness selection or missing +required pin. The runner checks policy-model connectivity from inside the +Docker sandbox before starting the harness. A connectivity or harness failure +is masked and is not sent to the judge. A normal model/task result can still +receive zero reward without those flags. Infrastructure, configuration, +task-loading, or judge failures set `mask_sample`; do not treat those zeroes as +model-quality results. + +Configurable-runner artifacts are grouped by harness, model, dated session, and +task. The default roots are `results/legal_agent_bench/hermes_jobs`, +`results/legal_agent_bench/claude_code_jobs`, and +`results/legal_agent_bench/codex_jobs`. Harbor continues to use +`results/legal_agent_bench/harbor_jobs`. + +Each configurable output row includes `artifact_dir`, `run_summary_path`, +`agent_trace_path`, `agent_stdout_path`, `agent_stderr_path`, +`verifier_report_path`, and `output_dir`. The inspection commands above use +these paths. + +The agent-server log also prints the artifact directory when the rollout is +created and again when it completes or fails. + ## Test Run the benchmark and resource-server tests with: @@ -93,7 +271,8 @@ Run the benchmark and resource-server tests with: ```bash uv run pytest -q \ benchmarks/legal_agent_bench/tests \ - resources_servers/legal_agent_bench/tests + resources_servers/legal_agent_bench/tests \ + responses_api_agents/legal_agent_bench_agent/tests ``` Generated indexes, collation metrics, Harbor jobs, source documents, and skills diff --git a/benchmarks/legal_agent_bench/config_claude_code.yaml b/benchmarks/legal_agent_bench/config_claude_code.yaml new file mode 100644 index 0000000000..fe7dea3da6 --- /dev/null +++ b/benchmarks/legal_agent_bench/config_claude_code.yaml @@ -0,0 +1,25 @@ +config_paths: + - responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_claude_code.yaml + +legal_agent_bench_benchmark_claude_code_resources_server: + _inherit_from: legal_agent_bench + resources_servers: + legal_agent_bench: + description: Configurable-agent integration of Legal Agent Benchmark (LAB) using Claude Code + +legal_agent_bench_benchmark_claude_code_agent: + _inherit_from: legal_agent_bench_claude_code_agent + responses_api_agents: + legal_agent_bench_agent: + resources_server: + type: resources_servers + name: legal_agent_bench_benchmark_claude_code_resources_server + runtime_tasks_dir: ${legal_agent_bench_benchmark_claude_code_resources_server.resources_servers.legal_agent_bench.harbor_tasks_dir} + skills_dir: ${legal_agent_bench_benchmark_claude_code_resources_server.resources_servers.legal_agent_bench.harness_skills_dir} + datasets: + - name: legal_agent_bench + type: benchmark + jsonl_fpath: benchmarks/legal_agent_bench/data/legal_agent_bench_benchmark.jsonl + prompt_config: null + prepare_script: benchmarks/legal_agent_bench/prepare.py + num_repeats: 1 diff --git a/benchmarks/legal_agent_bench/config_codex.yaml b/benchmarks/legal_agent_bench/config_codex.yaml new file mode 100644 index 0000000000..5b827e2ea3 --- /dev/null +++ b/benchmarks/legal_agent_bench/config_codex.yaml @@ -0,0 +1,25 @@ +config_paths: + - responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_codex.yaml + +legal_agent_bench_benchmark_codex_resources_server: + _inherit_from: legal_agent_bench + resources_servers: + legal_agent_bench: + description: Configurable-agent integration of Legal Agent Benchmark (LAB) using Codex + +legal_agent_bench_benchmark_codex_agent: + _inherit_from: legal_agent_bench_codex_agent + responses_api_agents: + legal_agent_bench_agent: + resources_server: + type: resources_servers + name: legal_agent_bench_benchmark_codex_resources_server + runtime_tasks_dir: ${legal_agent_bench_benchmark_codex_resources_server.resources_servers.legal_agent_bench.harbor_tasks_dir} + skills_dir: ${legal_agent_bench_benchmark_codex_resources_server.resources_servers.legal_agent_bench.harness_skills_dir} + datasets: + - name: legal_agent_bench + type: benchmark + jsonl_fpath: benchmarks/legal_agent_bench/data/legal_agent_bench_benchmark.jsonl + prompt_config: null + prepare_script: benchmarks/legal_agent_bench/prepare.py + num_repeats: 1 diff --git a/benchmarks/legal_agent_bench/config_hermes.yaml b/benchmarks/legal_agent_bench/config_hermes.yaml new file mode 100644 index 0000000000..bd5d6e6ec2 --- /dev/null +++ b/benchmarks/legal_agent_bench/config_hermes.yaml @@ -0,0 +1,25 @@ +config_paths: + - responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_hermes.yaml + +legal_agent_bench_benchmark_hermes_resources_server: + _inherit_from: legal_agent_bench + resources_servers: + legal_agent_bench: + description: Configurable-agent integration of Legal Agent Benchmark (LAB) using Hermes + +legal_agent_bench_benchmark_hermes_agent: + _inherit_from: legal_agent_bench_hermes_agent + responses_api_agents: + legal_agent_bench_agent: + resources_server: + type: resources_servers + name: legal_agent_bench_benchmark_hermes_resources_server + runtime_tasks_dir: ${legal_agent_bench_benchmark_hermes_resources_server.resources_servers.legal_agent_bench.harbor_tasks_dir} + skills_dir: ${legal_agent_bench_benchmark_hermes_resources_server.resources_servers.legal_agent_bench.harness_skills_dir} + datasets: + - name: legal_agent_bench + type: benchmark + jsonl_fpath: benchmarks/legal_agent_bench/data/legal_agent_bench_benchmark.jsonl + prompt_config: null + prepare_script: benchmarks/legal_agent_bench/prepare.py + num_repeats: 1 diff --git a/benchmarks/legal_agent_bench/prepare.py b/benchmarks/legal_agent_bench/prepare.py index 6a2345a580..1a0c67cda4 100644 --- a/benchmarks/legal_agent_bench/prepare.py +++ b/benchmarks/legal_agent_bench/prepare.py @@ -15,7 +15,6 @@ BENCHMARK_DIR = Path(__file__).resolve().parent DATA_DIR = BENCHMARK_DIR / "data" OUTPUT_FPATH = DATA_DIR / "legal_agent_bench_benchmark.jsonl" -BENCHMARK_AGENT_NAME = "legal_agent_bench_benchmark_harbor_agent" def _render_benchmark_index(source_index: Path) -> str: @@ -31,10 +30,7 @@ def _render_benchmark_index(source_index: Path) -> str: if not isinstance(row, dict): raise ValueError(f"LAB task index line {line_number} must contain a JSON object") - row["agent_ref"] = { - "name": BENCHMARK_AGENT_NAME, - "type": "responses_api_agents", - } + row.pop("agent_ref", None) rendered_rows.append(json.dumps(row, ensure_ascii=False, sort_keys=True) + "\n") if len(rendered_rows) != EXPECTED_TASK_COUNT: diff --git a/benchmarks/legal_agent_bench/tests/test_prepare.py b/benchmarks/legal_agent_bench/tests/test_prepare.py index 7aecea87c2..391cd8386a 100644 --- a/benchmarks/legal_agent_bench/tests/test_prepare.py +++ b/benchmarks/legal_agent_bench/tests/test_prepare.py @@ -8,11 +8,13 @@ from pathlib import Path import pytest -from omegaconf import OmegaConf +from omegaconf import DictConfig, OmegaConf from benchmarks.legal_agent_bench import prepare as benchmark_prepare from nemo_gym.benchmarks import BenchmarkConfig +from nemo_gym.config_types import ResponsesAPIAgentServerInstanceConfig from nemo_gym.global_config import GlobalConfigDictParser, GlobalConfigDictParserConfig +from nemo_gym.train_data_utils import TrainDataProcessor from resources_servers.legal_agent_bench.prepare import EXPECTED_TASK_COUNT, INDEX_FILENAME @@ -28,10 +30,6 @@ def _write_task_index(parent: Path, count: int) -> tuple[Path, list[str]]: for task_name in task_names: rows.append( { - "agent_ref": { - "name": "legal_agent_bench_harbor_agent", - "type": "responses_api_agents", - }, "instance_id": f"legal_agent_bench::{task_name}", "responses_create_params": { "input": [], @@ -73,14 +71,7 @@ def test_prepare_writes_deterministic_complete_benchmark_index(monkeypatch, tmp_ rows = [json.loads(line) for line in output_path.read_text(encoding="utf-8").splitlines()] assert len(rows) == EXPECTED_TASK_COUNT assert [row["instance_id"].split("::", 1)[1] for row in rows] == task_names - assert all( - row["agent_ref"] - == { - "name": benchmark_prepare.BENCHMARK_AGENT_NAME, - "type": "responses_api_agents", - } - for row in rows - ) + assert all("agent_ref" not in row for row in rows) def test_wrong_row_count_does_not_replace_existing_output(monkeypatch, tmp_path) -> None: @@ -150,3 +141,95 @@ def test_benchmark_config_is_isolated_and_resolves_shared_cache_paths() -> None: assert agent.harbor_agent_kwargs.max_turns == 60 assert len(agent.datasets) == 1 assert agent.datasets[0].type == "benchmark" + + +@pytest.mark.parametrize( + ("filename", "expected_agent", "expected_module"), + [ + ("config_hermes.yaml", "legal_agent_bench_benchmark_hermes_agent", "responses_api_agents.hermes_agent.app"), + ( + "config_claude_code.yaml", + "legal_agent_bench_benchmark_claude_code_agent", + "responses_api_agents.claude_code_agent.app", + ), + ("config_codex.yaml", "legal_agent_bench_benchmark_codex_agent", "responses_api_agents.codex_agent.app"), + ], +) +def test_configurable_benchmark_variants_resolve(filename, expected_agent, expected_module) -> None: + config_path = BENCHMARK_DIR / filename + benchmark = BenchmarkConfig.from_config_path(config_path, strict=False) + assert benchmark is not None + assert benchmark.name == "legal_agent_bench" + assert benchmark.agent_name == expected_agent + assert benchmark.dataset.jsonl_fpath == Path("benchmarks/legal_agent_bench/data/legal_agent_bench_benchmark.jsonl") + + initial_config = OmegaConf.merge( + OmegaConf.load(config_path), + GlobalConfigDictParserConfig.NO_MODEL_GLOBAL_CONFIG_DICT, + ) + resolved = GlobalConfigDictParser().parse_no_environment(initial_global_config_dict=initial_config) + agent_names = [ + name for name, value in resolved.items() if isinstance(value, DictConfig) and "responses_api_agents" in value + ] + resource_names = [ + name for name, value in resolved.items() if isinstance(value, DictConfig) and "resources_servers" in value + ] + assert agent_names == [expected_agent] + assert resource_names == [f"{expected_agent.removesuffix('_agent')}_resources_server"] + agent = resolved[expected_agent].responses_api_agents.legal_agent_bench_agent + assert agent.agent_server_module == expected_module + if expected_module == "responses_api_agents.claude_code_agent.app": + assert agent.agent_kwargs.claude_code_version == "2.1.211" + elif expected_module == "responses_api_agents.codex_agent.app": + assert agent.agent_kwargs.codex_version == "0.144.4" + assert agent.datasets[0].type == "benchmark" + assert agent.runtime_tasks_dir.endswith("data/runtime/harbor_tasks/legal_agent_bench") + + +@pytest.mark.parametrize( + "agent_name", + [ + "legal_agent_bench_benchmark_harbor_agent", + "legal_agent_bench_benchmark_hermes_agent", + "legal_agent_bench_benchmark_claude_code_agent", + "legal_agent_bench_benchmark_codex_agent", + ], +) +def test_benchmark_collation_stamps_selected_agent_without_changing_source(tmp_path, agent_name) -> None: + source = tmp_path / "legal_agent_bench.jsonl" + neutral_row = { + "instance_id": "legal_agent_bench::corporate__task", + "responses_create_params": {"input": []}, + } + source.write_text(json.dumps(neutral_row) + "\n", encoding="utf-8") + agent_config = { + "responses_api_agents": { + "agent": { + "host": "127.0.0.1", + "port": 12345, + "entrypoint": "app.py", + "resources_server": {"type": "resources_servers", "name": "legal_agent_bench"}, + "model_server": {"type": "responses_api_models", "name": "policy_model"}, + "datasets": [ + { + "name": "legal_agent_bench", + "type": "benchmark", + "jsonl_fpath": str(source), + "prepare_script": "benchmarks/legal_agent_bench/prepare.py", + "num_repeats": 1, + } + ], + } + } + } + instance = ResponsesAPIAgentServerInstanceConfig( + name=agent_name, + server_type_config_dict=DictConfig(agent_config), + responses_api_agents=agent_config["responses_api_agents"], + ) + + prepared = TrainDataProcessor()._collate_samples_single_type("benchmark", [instance])[0] + collated_row = json.loads(prepared.read_text(encoding="utf-8")) + + assert json.loads(source.read_text(encoding="utf-8")) == neutral_row + assert collated_row["agent_ref"] == {"type": "responses_api_agents", "name": agent_name} diff --git a/resources_servers/legal_agent_bench/README.md b/resources_servers/legal_agent_bench/README.md index 4ff4a739f2..33a73b80ed 100644 --- a/resources_servers/legal_agent_bench/README.md +++ b/resources_servers/legal_agent_bench/README.md @@ -1,14 +1,18 @@ # Legal Agent Bench -This resource server runs the public +This resource server runs the public Harvey [Legal Agent Benchmark (LAB)](https://github.com/harveyai/harvey-labs/tree/f46ef86e4788545622db25dcffa3aebb7a139929) -through NeMo Gym and Harbor. The integration is pinned to upstream commit +through NeMo Gym. The compatibility configuration uses Harbor, and optional +configurations run Gym's Hermes, Claude Code, or Codex harness through a +LAB-owned Docker runner. The integration is pinned to upstream commit `f46ef86e4788545622db25dcffa3aebb7a139929`: 1,749 tasks and the public `docx`, `pptx`, and `xlsx` skills. -NeMo Gym schedules rollouts, the custom Harbor agent works with each task's +NeMo Gym schedules rollouts, the selected agent works with each task's documents inside Docker, and the task-local verifier scores every rubric -criterion with an OpenAI-compatible judge model. +criterion with an OpenAI-compatible judge model. The default configuration uses +the Harbor agent harness, but it can easily be overriden through configuration +settings. ## Requirements @@ -17,6 +21,16 @@ criterion with an OpenAI-compatible judge model. - An OpenAI-compatible policy endpoint and judge endpoint - At least 10 GB of free working space for preparation and the first Docker build +Not required: + +- Separate Harbor, Hermes, Claude Code, or Codex installations +- Anthropic or OpenAI vendor subscriptions or CLI logins for the Claude Code + and Codex harnesses + +Gym provisions the pinned harness dependencies automatically. Every harness +uses the configured policy model endpoint. Access to the configured policy and +judge endpoints is still required and may itself be metered or paid. + The pinned source download is about 579 MiB; allow a few GiB of free working space during preparation. The first task also builds a document-tooling Docker image and can take several minutes. Later tasks reuse @@ -31,7 +45,12 @@ uv sync --extra dev docker info >/dev/null ``` -Add endpoint settings to the gitignored root `env.yaml`: +Keep this environment activated when running LAB commands. Invoke `gym` +directly rather than `uv run gym` for workflows that start servers: Ray starts +components from their own working directories, which can conflict with uv's +project discovery. + +Create a `env.yaml` file in the root `Gym/` directory and add your endpoint settings (note: `env.yaml` is .gitignored): ```yaml policy_base_url: https://your-policy-endpoint.example/v1 @@ -44,7 +63,9 @@ judge_model_name: your-judge-model ``` The judge credentials are injected only into the regenerated, gitignored -runtime task tree. They are never written into the cache. +runtime task tree. They are never written into the cache. The configurable +runner does not mount rubric files or pass judge credentials during the agent +phase; both are staged only after the agent exits. ## Prepare explicitly (recommended) @@ -57,7 +78,7 @@ python resources_servers/legal_agent_bench/prepare.py The command downloads the pinned LAB source archive from GitHub with retries and visible progress, verifies SHA-256 `e45cbdf3236b22866e034bcc62fb23bf00ef2f2e49db7a0cd8a4b07dbae9212c`, -rejects unsafe archive entries, generates deterministic Harbor tasks, and +rejects unsafe archive entries, generates deterministic runtime tasks, and builds each cache in staging before replacing the previous valid cache. A handled preparation failure leaves the previous cache in place. @@ -84,7 +105,7 @@ gym dataset collate \ --mode example_validation ``` -Preparation generates the full 1,749-row task index inside the task cache. Prepare the assets before collating the full validation dataset: +Preparation using `prepare.py` generates the full 1,749-row task index inside the task cache. Prepare the assets before collating the full validation dataset: ```bash python resources_servers/legal_agent_bench/prepare.py @@ -108,7 +129,13 @@ gym env test --resources-server legal_agent_bench ## Run and smoke test -Start the servers: +For the shortest copy-paste path that prepares and tests Harbor, Hermes, Claude +Code, and Codex as benchmark variants, use the +[benchmark README](../../benchmarks/legal_agent_bench/README.md#smoke-test-every-harness). +The commands below describe the lower-level resource-server and standalone +agent workflow. + +The default configuration starts the Harbor agent: ```bash gym env start \ @@ -131,6 +158,42 @@ gym eval run --no-serve \ --limit 1 ``` +The generated and example JSONL files do not contain `agent_ref`. Direct runs +must therefore specify the desired agent. The available standalone +configurations and agent names are: + +| Config path | Direct-run agent | +| --- | --- | +| `resources_servers/legal_agent_bench/configs/legal_agent_bench.yaml` | `legal_agent_bench_harbor_agent` | +| `responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_hermes.yaml` | `legal_agent_bench_hermes_agent` | +| `responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_claude_code.yaml` | `legal_agent_bench_claude_code_agent` | +| `responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_codex.yaml` | `legal_agent_bench_codex_agent` | + +The configurable files include +`resources_servers/legal_agent_bench/configs/resources_only.yaml`, which starts +the LAB resource server without also launching the compatibility Harbor agent. + +For example, start and run Hermes with: + +```bash +gym env start \ + --config responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_hermes.yaml \ + --model-type vllm_model + +gym eval run --no-serve \ + --config responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_hermes.yaml \ + --agent legal_agent_bench_hermes_agent \ + --input resources_servers/legal_agent_bench/data/example.jsonl \ + --output results/legal_agent_bench_hermes_smoke.jsonl \ + --concurrency 1 \ + --limit 1 +``` + +Use the corresponding config and agent name for Claude Code or Codex. All +three harnesses use the configured Gym policy endpoint. That endpoint must +support the selected harness's protocol. Their CLI/runtime dependencies are +pinned and provisioned into a portable cache on the first rollout. + The default `full_task` reward is LAB's official all-criteria score: a task earns `1.0` only when every criterion passes. For diagnostic partial credit, start with: @@ -142,8 +205,8 @@ gym env start \ +legal_agent_bench.resources_servers.legal_agent_bench.reward_mode=criteria_pass_rate ``` -The verifier evaluates up to six criteria concurrently, matching the upstream LAB -default. Adjust the resource setting when the judge endpoint has a lower +The verifier evaluates up to six criteria concurrently by default. +Adjust the resource setting when the judge endpoint has a lower concurrency limit: ```bash @@ -156,10 +219,8 @@ gym env start \ This setting is forwarded to each task verifier as `LAB_JUDGE_PARALLELISM`. -The task container requires network access because its verifier calls the -configured judge endpoint. The agent and verifier share that container, so the -agent also has network access during a rollout. This differs from the upstream LAB -closed-network reference sandbox and should be recorded when comparing runs. +The task container requires network access because the policy agent and +verifier call configured endpoints. ## Caches and outputs @@ -169,6 +230,10 @@ The default paths are: - Public skills: `data/cache/harness/skills` - Credential-bearing runtime tasks: `data/runtime/harbor_tasks/legal_agent_bench` - Harbor jobs: `results/legal_agent_bench/harbor_jobs` +- Configurable agent runtimes: `responses_api_agents/legal_agent_bench_agent/.deps` +- Hermes rollout artifacts: `results/legal_agent_bench/hermes_jobs` +- Claude Code rollout artifacts: `results/legal_agent_bench/claude_code_jobs` +- Codex rollout artifacts: `results/legal_agent_bench/codex_jobs` - Rollout output: the path passed to `gym eval run` The runtime tree hardlinks immutable documents from the cache when the @@ -181,15 +246,63 @@ Each successful Harbor trial contains `result.json`, `verifier/reward.json`, and `agent/artifacts/lab-run/transcript.jsonl`. The agent config artifact should list exactly `docx`, `pptx`, and `xlsx`. +Each configurable trial contains the inner Gym trajectory, agent stdout and +stderr, LAB `config.json` and `metrics.json`, completed deliverables under +`agent/artifacts/lab-run/output`, downloaded verifier artifacts, and a compact +top-level `run_summary.json`. The rollout JSONL row exposes direct paths to the +summary, trajectory, stdout, stderr, output directory, and verifier report. +Trials are grouped as +`_jobs///_` so runs +are browsable and safe under concurrent execution. `` is the configured +`policy_model_name`, normalized into one safe path segment. The task name is +normalized, and the run ID is an eight-character unique suffix. Starting or +validating an agent server does not create an empty session directory; the +directory is created by its first rollout. +The Docker image is content-addressed from the task environment and reused +across harnesses. Agent and verifier phases use separate containers: the agent +sees only its selected Gym package and public task inputs, while the verifier +starts after that container is destroyed and receives the completed LAB run +read-only. + +To inspect a configurable smoke run: + +```bash +ARTIFACT_DIR=$(jq -r '.artifact_dir' results/legal_agent_bench_hermes_smoke.jsonl) +jq . "$ARTIFACT_DIR/run_summary.json" +jq . "$ARTIFACT_DIR/agent/trajectory.json" +open "$ARTIFACT_DIR/verifier/report.html" # macOS; use xdg-open on Linux +``` + +In `run_summary.json`, a reliable completed rollout has `mask_sample: false`, +all failure flags false, `judge_error_count: 0`, and `verifier_error: 0`. +`output_files` lists the deliverables and `agent/trajectory.json` is the inner +harness trace. A zero `full_task` reward can still be a valid rollout when one +or more criteria fail; use `criteria_pass_rate` to see partial success. + ## Troubleshooting -- A checksum, corrupt archive, unsafe path, wrong task count, or missing skill - fails before replacing an existing valid cache. - A missing judge setting produces a verifier error in the trial artifacts. Confirm the endpoint permits the exact `judge_model_name`. - Treat a nonzero `judge_error_count` or `verifier_error` as a judge or infrastructure failure, not an ordinary model failure, even though Harbor receives a numeric zero reward so it can preserve a complete trial result. +- Configurable rollouts additionally expose `agent_failed`, + `model_connection_failed`, `agent_timed_out`, `verifier_failed`, + `verifier_timed_out`, `sandbox_failed`, `task_failed`, + `configuration_failed`, and `mask_sample`. Before the harness starts, the + runner checks the policy endpoint from inside Docker. On Docker Desktop, + derived loopback model URLs are automatically routed through + `host.docker.internal`; `sandbox_model_base_url` is an explicit override. + Connectivity and harness failures are masked and skip judging. + Task-loading and harness-configuration failures are kept separate from Docker + sandbox failures. A zero reward without an infrastructure/judge flag is an + ordinary model/task result; a flagged result should be excluded from + model-quality comparisons. +- Hermes normally probes `/v1/models` and `/models` for optional pricing and + context metadata. The LAB runner disables those lookups because Gym supplies + the model explicitly and its internal policy proxy does not implement model + discovery. This does not suppress access logs for actual chat-completion + requests. - If Docker appears idle on the first rollout, inspect `docker ps` and the `gym env start` terminal; Harbor is normally building the task image. - Do not copy or publish `data/runtime/`: it can contain local judge credentials. diff --git a/resources_servers/legal_agent_bench/configs/resources_only.yaml b/resources_servers/legal_agent_bench/configs/resources_only.yaml new file mode 100644 index 0000000000..97d3814ee6 --- /dev/null +++ b/resources_servers/legal_agent_bench/configs/resources_only.yaml @@ -0,0 +1,28 @@ +legal_agent_bench: + resources_servers: + legal_agent_bench: + entrypoint: app.py + domain: agent + verified: false + description: Resource-only Legal Agent Benchmark (LAB) configuration for configurable Gym agents + value: Improve legal-agent document review, drafting, and analysis capability + harbor_tasks_cache_dir: ${oc.env:LEGAL_AGENT_BENCH_TASK_CACHE_DIR,resources_servers/legal_agent_bench/data/cache/harbor_tasks/legal_agent_bench} + harbor_tasks_dir: ${oc.env:LEGAL_AGENT_BENCH_RUNTIME_TASKS_DIR,resources_servers/legal_agent_bench/data/runtime/harbor_tasks/legal_agent_bench} + harness_skills_dir: ${oc.env:LEGAL_AGENT_BENCH_SKILLS_DIR,resources_servers/legal_agent_bench/data/cache/harness/skills} + auto_prepare_assets: true + reward_mode: full_task + judge_base_url: ${oc.select:judge_base_url,null} + judge_api_key: ${oc.select:judge_api_key,null} + judge_model_name: ${oc.select:judge_model_name,null} + judge_temperature: null + judge_request_timeout_seconds: 90 + judge_max_retries: 1 + judge_structured_output: true + judge_parse_repair_attempts: 1 + judge_repair_max_tokens: 4096 + judge_max_tokens: 4096 + judge_parallelism: 6 + +responses_create_params: + temperature: 1.0 + top_p: 0.95 diff --git a/resources_servers/legal_agent_bench/data/example.jsonl b/resources_servers/legal_agent_bench/data/example.jsonl index 425bace0cc..0e90ca5120 100644 --- a/resources_servers/legal_agent_bench/data/example.jsonl +++ b/resources_servers/legal_agent_bench/data/example.jsonl @@ -1,5 +1,5 @@ -{"agent_ref": {"name": "legal_agent_bench_harbor_agent", "type": "responses_api_agents"}, "instance_id": "legal_agent_bench::trusts-estates-private-client__compare-trust-documents-against-client-instructions", "responses_create_params": {"input": [], "temperature": 1.0, "top_p": 0.95}} -{"agent_ref": {"name": "legal_agent_bench_harbor_agent", "type": "responses_api_agents"}, "instance_id": "legal_agent_bench::corporate-ma__analyze-transition-services-agreement-markup", "responses_create_params": {"input": [], "temperature": 1.0, "top_p": 0.95}} -{"agent_ref": {"name": "legal_agent_bench_harbor_agent", "type": "responses_api_agents"}, "instance_id": "legal_agent_bench::healthcare-life-sciences__analyze-compliance-program-gaps", "responses_create_params": {"input": [], "temperature": 1.0, "top_p": 0.95}} -{"agent_ref": {"name": "legal_agent_bench_harbor_agent", "type": "responses_api_agents"}, "instance_id": "legal_agent_bench::employment-labor__analyze-reasonable-accommodation-request-under-ada-requirements", "responses_create_params": {"input": [], "temperature": 1.0, "top_p": 0.95}} -{"agent_ref": {"name": "legal_agent_bench_harbor_agent", "type": "responses_api_agents"}, "instance_id": "legal_agent_bench::litigation-dispute-resolution__categorize-document-production-set-by-relevance-and-privilege", "responses_create_params": {"input": [], "temperature": 1.0, "top_p": 0.95}} +{"instance_id": "legal_agent_bench::trusts-estates-private-client__compare-trust-documents-against-client-instructions", "responses_create_params": {"input": [], "temperature": 1.0, "top_p": 0.95}} +{"instance_id": "legal_agent_bench::corporate-ma__analyze-transition-services-agreement-markup", "responses_create_params": {"input": [], "temperature": 1.0, "top_p": 0.95}} +{"instance_id": "legal_agent_bench::healthcare-life-sciences__analyze-compliance-program-gaps", "responses_create_params": {"input": [], "temperature": 1.0, "top_p": 0.95}} +{"instance_id": "legal_agent_bench::employment-labor__analyze-reasonable-accommodation-request-under-ada-requirements", "responses_create_params": {"input": [], "temperature": 1.0, "top_p": 0.95}} +{"instance_id": "legal_agent_bench::litigation-dispute-resolution__categorize-document-production-set-by-relevance-and-privilege", "responses_create_params": {"input": [], "temperature": 1.0, "top_p": 0.95}} diff --git a/resources_servers/legal_agent_bench/prepare.py b/resources_servers/legal_agent_bench/prepare.py index 20264afaa0..8e72876df4 100644 --- a/resources_servers/legal_agent_bench/prepare.py +++ b/resources_servers/legal_agent_bench/prepare.py @@ -242,7 +242,10 @@ def prepare_assets( print(f"Using cached Legal Agent Bench {name} in {prepared[name]}", flush=True) continue except (FileNotFoundError, ValueError): - pass + if name == "tasks" and _migrate_legacy_agent_index(targets[name]): + prepared[name] = validators[name](targets[name]) + print(f"Migrated cached Legal Agent Bench {name} index in {prepared[name]}", flush=True) + continue missing.append(name) if not missing: @@ -479,10 +482,6 @@ def _render_task_index(source_ids: Iterable[str]) -> str: rows = [] for source_id in sorted(source_ids): row = { - "agent_ref": { - "name": "legal_agent_bench_harbor_agent", - "type": "responses_api_agents", - }, "instance_id": f"legal_agent_bench::{flatten_task_id(source_id)}", "responses_create_params": { "input": [], @@ -675,6 +674,47 @@ def _replace_directory(source: Path, target: Path) -> None: shutil.rmtree(backup) +def _migrate_legacy_agent_index(tasks_dir: Path) -> bool: + """Atomically remove the retired Harbor agent_ref from an otherwise exact cached index.""" + index_path = tasks_dir / INDEX_FILENAME + try: + rows = [json.loads(line) for line in index_path.read_text(encoding="utf-8").splitlines()] + except (OSError, json.JSONDecodeError): + return False + legacy_ref = { + "name": "legal_agent_bench_harbor_agent", + "type": "responses_api_agents", + } + if not rows or any(not isinstance(row, dict) or row.get("agent_ref") != legacy_ref for row in rows): + return False + candidate_rows = [] + for row in rows: + row = dict(row) + row.pop("agent_ref") + candidate_rows.append(json.dumps(row, ensure_ascii=False, sort_keys=True) + "\n") + candidate = "".join(candidate_rows) + + source_ids = [] + try: + for task_dir in sorted(child for child in tasks_dir.iterdir() if child.is_dir()): + task = json.loads((task_dir / "task.json").read_text(encoding="utf-8")) + source_ids.append(str(task["metadata"]["lab_task_id"])) + except (OSError, KeyError, TypeError, ValueError, json.JSONDecodeError): + return False + if candidate != _render_task_index(source_ids): + return False + + file_descriptor, temp_name = tempfile.mkstemp(dir=index_path.parent, prefix=f".{index_path.name}.") + os.close(file_descriptor) + temp_path = Path(temp_name) + try: + temp_path.write_text(candidate, encoding="utf-8") + os.replace(temp_path, index_path) + finally: + temp_path.unlink(missing_ok=True) + return True + + def _publish_task_index(tasks_dir: Path) -> Path: """Atomically publish the prepared index at Gym's stable dataset path.""" source = tasks_dir / INDEX_FILENAME diff --git a/resources_servers/legal_agent_bench/tests/test_prepare.py b/resources_servers/legal_agent_bench/tests/test_prepare.py index 973f53cabb..c105e64b1a 100644 --- a/resources_servers/legal_agent_bench/tests/test_prepare.py +++ b/resources_servers/legal_agent_bench/tests/test_prepare.py @@ -90,7 +90,7 @@ def test_generated_task_cache_is_deterministic_and_credential_free(monkeypatch, "legal_agent_bench::area__task-group__scenario-01", "legal_agent_bench::area__task-one", ] - assert all(row["agent_ref"]["name"] == "legal_agent_bench_harbor_agent" for row in rows) + assert all("agent_ref" not in row for row in rows) for toml in first.glob("*/task.toml"): text = toml.read_text(encoding="utf-8") assert "[verifier.env]" not in text @@ -112,6 +112,31 @@ def test_existing_valid_caches_skip_network(monkeypatch, tmp_path) -> None: assert (tmp_path / "all.jsonl").read_bytes() == (tasks / prepare.INDEX_FILENAME).read_bytes() +def test_legacy_harbor_index_migrates_without_network(monkeypatch, tmp_path) -> None: + _source, tasks, _skills = _build_caches(monkeypatch, tmp_path) + index_path = tasks / prepare.INDEX_FILENAME + legacy_rows = [] + for line in index_path.read_text(encoding="utf-8").splitlines(): + row = json.loads(line) + row["agent_ref"] = { + "name": "legal_agent_bench_harbor_agent", + "type": "responses_api_agents", + } + legacy_rows.append(json.dumps(row, sort_keys=True) + "\n") + index_path.write_text("".join(legacy_rows), encoding="utf-8") + monkeypatch.setattr( + prepare, + "_download_source_archive", + lambda _path: pytest.fail("the exact legacy index must migrate without a source download"), + ) + + result = prepare.prepare_assets("tasks", tasks_dir=tasks) + + assert result == {"tasks": tasks} + assert all("agent_ref" not in json.loads(line) for line in index_path.read_text().splitlines()) + prepare.validate_harbor_tasks(tasks) + + def test_missing_assets_download_extract_and_install(monkeypatch, tmp_path) -> None: _configure_small_snapshot(monkeypatch, tmp_path) source = _write_source(tmp_path / "source") diff --git a/responses_api_agents/claude_code_agent/scripts/claude_code_agent_deps.sh b/responses_api_agents/claude_code_agent/scripts/claude_code_agent_deps.sh index 3a12fca8e4..99d739cbb3 100755 --- a/responses_api_agents/claude_code_agent/scripts/claude_code_agent_deps.sh +++ b/responses_api_agents/claude_code_agent/scripts/claude_code_agent_deps.sh @@ -9,16 +9,26 @@ source "${PORTABLE_PYTHON_SH:-$SCRIPT_DIR/_portable_python.sh}" : "${DEPS_DIR:?DEPS_DIR must be set}" : "${NEMO_GYM_ROOT:?NEMO_GYM_ROOT must be set}" -NODE_VERSION="${NODE_VERSION:-20.18.1}" -CLAUDE_SPEC="${CLAUDE_SPEC:-@anthropic-ai/claude-code}" +NODE_VERSION="${NODE_VERSION:-22.15.0}" +: "${CLAUDE_SPEC:?CLAUDE_SPEC must select a pinned @anthropic-ai/claude-code version}" +if [ -z "${NODE_ARCH:-}" ]; then + case "$(uname -m)" in + x86_64) NODE_ARCH="x64" ;; + aarch64|arm64) NODE_ARCH="arm64" ;; + *) + echo "Unsupported Node architecture: $(uname -m)" >&2 + exit 1 + ;; + esac +fi install_portable_python install_nemo_gym_deps if [ ! -x "$DEPS_DIR/bin/node" ]; then - node_url="https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-x64.tar.xz" + node_url="https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-${NODE_ARCH}.tar.gz" echo "Downloading portable node: $node_url" - curl -fsSL "$node_url" | tar xJ -C "$DEPS_DIR" --strip-components=1 + curl -fsSL "$node_url" | tar xz -C "$DEPS_DIR" --strip-components=1 fi export PATH="$DEPS_DIR/bin:$PATH" diff --git a/responses_api_agents/codex_agent/scripts/codex_agent_deps.sh b/responses_api_agents/codex_agent/scripts/codex_agent_deps.sh new file mode 100755 index 0000000000..c487ab5c1c --- /dev/null +++ b/responses_api_agents/codex_agent/scripts/codex_agent_deps.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# Install Codex agent dependencies into a portable prefix mounted in a task sandbox. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${PORTABLE_PYTHON_SH:-$SCRIPT_DIR/_portable_python.sh}" + +: "${DEPS_DIR:?DEPS_DIR must be set}" +: "${NEMO_GYM_ROOT:?NEMO_GYM_ROOT must be set}" +NODE_VERSION="${NODE_VERSION:-22.15.0}" +: "${CODEX_SPEC:?CODEX_SPEC must select a pinned @openai/codex version}" +if [ -z "${NODE_ARCH:-}" ]; then + case "$(uname -m)" in + x86_64) NODE_ARCH="x64" ;; + aarch64|arm64) NODE_ARCH="arm64" ;; + *) + echo "Unsupported Node architecture: $(uname -m)" >&2 + exit 1 + ;; + esac +fi + +install_portable_python +install_nemo_gym_deps + +if [ ! -x "$DEPS_DIR/bin/node" ]; then + node_url="https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-${NODE_ARCH}.tar.gz" + curl -fsSL "$node_url" | tar xz -C "$DEPS_DIR" --strip-components=1 +fi + +export PATH="$DEPS_DIR/bin:$PATH" +npm install -g --prefix "$DEPS_DIR" "$CODEX_SPEC" +"$DEPS_DIR/bin/codex" --version diff --git a/responses_api_agents/hermes_agent/README.md b/responses_api_agents/hermes_agent/README.md index 76c6de5090..f4cdc543d0 100644 --- a/responses_api_agents/hermes_agent/README.md +++ b/responses_api_agents/hermes_agent/README.md @@ -41,10 +41,7 @@ Key metrics for math_with_judge_hermes_agent: "mean/reward": 0.2, "mean/turns_used": 1.6, "mean/finished_naturally": 1.0, - "mean/library_reward": 0.2, - "mean/input_tokens": 0.0, - "mean/output_tokens": 0.0, - "mean/total_tokens": 0.0 + "mean/library_reward": 0.2 } Finished rollout collection! View results at: Fully materialized inputs: responses_api_agents/hermes_agent/data/example_math_rollouts_materialized_inputs.jsonl @@ -52,6 +49,10 @@ Rollouts: responses_api_agents/hermes_agent/data/example_math_rollouts.jsonl Aggregate metrics: responses_api_agents/hermes_agent/data/example_math_rollouts_aggregate_metrics.json ``` +Current runs also report nonzero `mean/input_tokens`, +`mean/output_tokens`, and `mean/total_tokens` from Hermes' cumulative session +counters. + Example training reward for small multi environment test is shown [here](https://github.com/NVIDIA-NeMo/Gym/pull/1033#issuecomment-4399509664). ## Description diff --git a/responses_api_agents/hermes_agent/app.py b/responses_api_agents/hermes_agent/app.py index 4d28b7d44c..b935bec52d 100644 --- a/responses_api_agents/hermes_agent/app.py +++ b/responses_api_agents/hermes_agent/app.py @@ -57,6 +57,32 @@ from responses_api_agents.hermes_agent.observability import HermesAgentObserver +def _usage_from_result(result: dict[str, Any]) -> NeMoGymResponseUsage: + """Map Hermes' cumulative session counters to the Responses API usage schema.""" + + def token_count(key: str) -> int: + try: + return max(0, int(result.get(key) or 0)) + except (TypeError, ValueError): + return 0 + + cache_read_tokens = token_count("cache_read_tokens") + cache_write_tokens = token_count("cache_write_tokens") + input_tokens = token_count("prompt_tokens") + if input_tokens == 0: + input_tokens = token_count("input_tokens") + cache_read_tokens + cache_write_tokens + output_tokens = token_count("output_tokens") or token_count("completion_tokens") + total_tokens = token_count("total_tokens") or input_tokens + output_tokens + + return NeMoGymResponseUsage( + input_tokens=input_tokens, + input_tokens_details=NeMoGymResponseInputTokensDetails(cached_tokens=cache_read_tokens), + output_tokens=output_tokens, + output_tokens_details=NeMoGymResponseOutputTokensDetails(reasoning_tokens=token_count("reasoning_tokens")), + total_tokens=total_tokens, + ) + + def _trajectory_to_output_items(messages, n_input): output_items = [] for item in messages[n_input:]: @@ -410,13 +436,7 @@ def _patched_build_api_kwargs(api_messages): tool_choice=body.tool_choice, tools=body.tools, parallel_tool_calls=body.parallel_tool_calls, - usage=NeMoGymResponseUsage( - input_tokens=0, - input_tokens_details=NeMoGymResponseInputTokensDetails(cached_tokens=0), - output_tokens=0, - output_tokens_details=NeMoGymResponseOutputTokensDetails(reasoning_tokens=0), - total_tokens=0, - ), + usage=_usage_from_result(result), ) async def responses( diff --git a/responses_api_agents/hermes_agent/tests/test_app.py b/responses_api_agents/hermes_agent/tests/test_app.py index ae0aacd3ec..37ebb3bf18 100644 --- a/responses_api_agents/hermes_agent/tests/test_app.py +++ b/responses_api_agents/hermes_agent/tests/test_app.py @@ -35,6 +35,7 @@ ResourcesServerRef, _split_input_to_user_and_history, _trajectory_to_output_items, + _usage_from_result, ) from responses_api_agents.hermes_agent.observability import HermesAgentObserver @@ -273,6 +274,43 @@ def test_skips_non_dict_items(self) -> None: assert len(out) == 1 +class TestUsageFromResult: + def test_maps_hermes_session_counters(self) -> None: + usage = _usage_from_result( + { + "input_tokens": 100, + "cache_read_tokens": 20, + "cache_write_tokens": 5, + "prompt_tokens": 125, + "output_tokens": 40, + "reasoning_tokens": 10, + "total_tokens": 165, + } + ) + + assert usage.input_tokens == 125 + assert usage.input_tokens_details.cached_tokens == 20 + assert usage.output_tokens == 40 + assert usage.output_tokens_details.reasoning_tokens == 10 + assert usage.total_tokens == 165 + + def test_reconstructs_totals_when_aggregate_counters_are_missing(self) -> None: + usage = _usage_from_result( + { + "input_tokens": 100, + "cache_read_tokens": 20, + "cache_write_tokens": 5, + "completion_tokens": 40, + } + ) + + assert usage.input_tokens == 125 + assert usage.input_tokens_details.cached_tokens == 20 + assert usage.output_tokens == 40 + assert usage.output_tokens_details.reasoning_tokens == 0 + assert usage.total_tokens == 165 + + class TestRolloutCorrelation: def test_responses_applies_rollout_prefix(self, monkeypatch) -> None: from fastapi.testclient import TestClient @@ -296,7 +334,12 @@ def __init__(self, **kwargs) -> None: self.compression_enabled = True def run_conversation(self, *args, **kwargs) -> dict: - return {"messages": [{"role": "assistant", "content": "ok"}]} + return { + "messages": [{"role": "assistant", "content": "ok"}], + "prompt_tokens": 12, + "output_tokens": 3, + "total_tokens": 15, + } monkeypatch.setattr("run_agent.AIAgent", _StubAIAgent) client = TestClient(agent.setup_webserver()) @@ -307,6 +350,9 @@ def run_conversation(self, *args, **kwargs) -> dict: direct = asyncio.run(agent.responses(request=None, body=NeMoGymResponseCreateParamsNonStreaming(input="hi"))) assert seen["base_url"] == "http://h:1/v1" assert "_ng_agent_observations" not in direct.model_dump(mode="json") + assert direct.usage.input_tokens == 12 + assert direct.usage.output_tokens == 3 + assert direct.usage.total_tokens == 15 episode = asyncio.run( agent._create_episode( diff --git a/responses_api_agents/legal_agent_bench_agent/.gitignore b/responses_api_agents/legal_agent_bench_agent/.gitignore new file mode 100644 index 0000000000..86e51f5891 --- /dev/null +++ b/responses_api_agents/legal_agent_bench_agent/.gitignore @@ -0,0 +1,3 @@ +.deps/ +results/ +__pycache__/ diff --git a/responses_api_agents/legal_agent_bench_agent/README.md b/responses_api_agents/legal_agent_bench_agent/README.md new file mode 100644 index 0000000000..3aaefac3fe --- /dev/null +++ b/responses_api_agents/legal_agent_bench_agent/README.md @@ -0,0 +1,85 @@ +# Legal Agent Bench Configurable Agent + +This task-driven Gym agent runs a configured Gym Responses API agent inside the +Legal Agent Bench (LAB) Docker environment. It keeps the benchmark task set, +skills, and verifier fixed while allowing the agent harness to vary. + +The committed configurations select Gym's Hermes, Claude Code, and Codex agents +through: + +- `agent_server_module` +- `agent_server_class` +- `agent_config_class` +- `agent_kwargs` + +The runner also accepts the policy `model_server`, LAB runtime task and skill +paths, concurrency and timeout settings, Docker network and model URL +overrides, and a result directory. See `configs/` for complete examples. +Those configurations include the LAB `resources_only.yaml` definition so Gym +starts one resource server and only the selected agent; the combined +compatibility config is reserved for Harbor. + +## Execution boundary + +For each rollout, the runner validates `instance_id`, builds or reuses the +content-addressed LAB image, provisions a cached portable Linux runtime for the +selected harness inside that image, and starts an agent-only Docker sandbox. +Provisioning selects ARM64 or x86-64 dependencies from the container +architecture, so the host does not need to execute container binaries. The +runtime cache is invalidated by changes to its setup script, requirements, +configured CLI pin, LAB image, Gym packaging metadata, or installed `nemo_gym` +source. Only the selected Gym agent package, source documents, skills, and +portable dependencies are mounted read-only; the repository and LAB task cache +are not mounted. Output and scratch directories are writable. + +Only the task instructions and public skill manuals are supplied to the +configured agent. After it exits, its container is destroyed before a fresh +verifier-only sandbox starts. That sandbox receives the completed `lab-run` +directory read-only, then receives the rubric tests and judge credentials. +The verifier sees deliverables at `lab-run/output`, preserving LAB's contract +without sharing a process namespace with the agent. + +The runner translates derived host-loopback model URLs to +`host.docker.internal` on Docker Desktop and registers the equivalent host +gateway for Linux bridge networking. An explicit `sandbox_model_base_url` is +used unchanged. Before importing the selected harness, the in-container runner +performs a bounded TCP connectivity check and writes +`runtime/runner_status.json`. Connectivity or harness failures set the +`mask_sample` flag on the row, skip the judge, and are distinct from an ordinary zero-reward +model result. For Hermes, the runner also disables its optional `/models` +pricing and context-metadata probes; Gym supplies the selected model explicitly, +and actual model-call access logging is enabled. + +Artifacts are written below +`results/legal_agent_bench/_jobs///_/`. +For example, Hermes uses `results/legal_agent_bench/hermes_jobs`. The model +segment is the configured `policy_model_name`, normalized into one safe path +segment. The task name is also normalized, and the run ID is an eight-character +unique suffix. A session directory is created only when its first rollout starts. +Each trial includes the inner Gym trajectory, harness logs, LAB run +configuration and metrics, completed output files, verifier artifacts, and a +top-level `run_summary.json`. The rollout response also includes direct paths +for the summary, trajectory, stdout, stderr, output directory, and verifier +report. + +For setup, four copy-paste smoke commands, and result inspection, see the +[benchmark README](../../benchmarks/legal_agent_bench/README.md#smoke-test-every-harness). +Run `gym` from an activated repository environment rather than prefixing +server-starting commands with `uv run`. + +## Adding a harness + +A harness can be selected without changing this runner when it: + +1. is implemented as a Gym Responses API agent; +2. can run inside the LAB Linux container; +3. has a portable dependency script at + `responses_api_agents//scripts/_deps.sh`; and +4. can use the configured OpenAI-compatible policy endpoint. + +Add a standalone configuration under `configs/`, then add a benchmark variant +that inherits it. CLI versions belong in the harness configuration's +`agent_kwargs`; provisioning derives its exact package specification from that +pin and rejects unpinned Claude Code or Codex configurations. + +The runner is compatible with Docker only. \ No newline at end of file diff --git a/responses_api_agents/legal_agent_bench_agent/__init__.py b/responses_api_agents/legal_agent_bench_agent/__init__.py new file mode 100644 index 0000000000..353bc52e78 --- /dev/null +++ b/responses_api_agents/legal_agent_bench_agent/__init__.py @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Configurable Gym-agent runner for Legal Agent Bench.""" diff --git a/responses_api_agents/legal_agent_bench_agent/app.py b/responses_api_agents/legal_agent_bench_agent/app.py new file mode 100644 index 0000000000..f2fd5621b3 --- /dev/null +++ b/responses_api_agents/legal_agent_bench_agent/app.py @@ -0,0 +1,1121 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Run a config-selected Gym agent inside a Legal Agent Bench task sandbox.""" + +from __future__ import annotations + +import asyncio +import hashlib +import json +import os +import shutil +import subprocess +import sys +import tarfile +import tempfile +import time +from collections.abc import Mapping +from pathlib import Path, PurePosixPath +from typing import Any, Optional +from urllib.parse import urlsplit, urlunsplit +from uuid import uuid4 + +from fastapi import Body, Request +from pydantic import ConfigDict, Field, PrivateAttr + +from nemo_gym import PARENT_DIR +from nemo_gym.base_resources_server import BaseRunRequest, BaseVerifyResponse +from nemo_gym.base_responses_api_agent import BaseResponsesAPIAgentConfig, SimpleResponsesAPIAgent +from nemo_gym.config_types import ModelServerRef, ResourcesServerRef +from nemo_gym.global_config import get_first_server_config_dict +from nemo_gym.openai_utils import NeMoGymResponse, NeMoGymResponseCreateParamsNonStreaming +from nemo_gym.sandbox import AsyncSandbox, SandboxResources, SandboxSpec +from nemo_gym.sandbox.providers.docker import DockerProvider +from nemo_gym.server_utils import apply_rollout_prefix +from resources_servers.legal_agent_bench.prepare import ( + DEFAULT_RUNTIME_TASKS_DIR, + DEFAULT_SKILLS_DIR, + REQUIRED_SKILLS, + resolve_repo_path, + validate_harness_skills, +) + + +PACKAGE_DIR = Path(__file__).resolve().parent +PORTABLE_PYTHON_SH = PACKAGE_DIR / "setup_scripts" / "_portable_python.sh" +DATASET_ALIAS = "legal_agent_bench" +INITIAL_USER_PROMPT = "Please begin working on the task described in the system prompt." +AGENT_CLI_PINS = { + "claude_code_agent": ("claude_code_version", "CLAUDE_SPEC", "@anthropic-ai/claude-code"), + "codex_agent": ("codex_version", "CODEX_SPEC", "@openai/codex"), +} + +GENERIC_HARNESS_PREAMBLE = """\ +You are an AI agent running in an automated Legal Agent Bench evaluation. + +## Workspace layout + +- Source documents are under `/workspace/vdr` and `$VDR_DIR`. Treat them as read-only. +- Write every final deliverable under `/workspace/output` and `$OUTPUT_DIR`. +- Use `/workspace/workspace` for scratch files. +- Skill manuals and their supporting assets are under `/workspace/skills`. +- Do not search for task configuration, rubric, verifier, test, or judge files. They are intentionally + unavailable while you work. + +Use the terminal and filesystem tools provided by your agent harness. The image already contains the +document tooling required by the skills; do not install packages during the task. Read the skill manuals +included below before creating deliverables. +""" + +_RUNNER_SOURCE = r"""#!/usr/bin/env python3 +import asyncio +import inspect +import json +import os +import socket +import sys +from pathlib import Path +from urllib.parse import urlsplit + +sys.path.insert(0, "/agent_source_mount") +os.environ["PATH"] = os.environ.get("PATH", "") + os.pathsep + "/agent_deps_mount/bin" +os.environ["VDR_DIR"] = "/workspace/vdr" +os.environ["OUTPUT_DIR"] = "/workspace/output" +os.environ["WORKSPACE_DIR"] = "/workspace/workspace" +os.environ["SKILLS_DIR"] = "/workspace/skills" +os.environ["HOME"] = "/workspace/workspace" +os.chdir("/workspace/output") + +from nemo_gym.config_types import ModelServerRef, ResourcesServerRef +from nemo_gym.openai_utils import NeMoGymResponseCreateParamsNonStreaming +from nemo_gym.server_utils import ServerClient + +runner = json.loads(Path("/trajectories_mount/runner.json").read_text()) +model_url = runner["model_url"].rstrip("/") +model_url_v1 = model_url if model_url.endswith("/v1") else model_url + "/v1" +status_path = Path("/trajectories_mount/runner_status.json") + + +def write_status(*, ok, phase, error=None): + status_path.write_text(json.dumps({"ok": ok, "phase": phase, "error": error}, indent=2)) + + +try: + parsed_model_url = urlsplit(model_url) + if not parsed_model_url.hostname: + raise ValueError(f"Policy model URL has no hostname: {model_url!r}") + model_port = parsed_model_url.port or (443 if parsed_model_url.scheme == "https" else 80) + with socket.create_connection( + (parsed_model_url.hostname, model_port), + timeout=runner.get("model_connect_timeout_seconds", 10), + ): + pass +except Exception as exc: + message = f"Policy model is unreachable from the LAB sandbox: {model_url} ({type(exc).__name__}: {exc})" + write_status(ok=False, phase="model_connectivity", error=message) + raise RuntimeError(message) from exc + +try: + module = __import__( + runner["agent_server_module"], + fromlist=[runner["agent_server_class"], runner["agent_config_class"]], + ) + agent_class = getattr(module, runner["agent_server_class"]) + config_class = getattr(module, runner["agent_config_class"]) + + if runner.get("disable_endpoint_metadata_probe"): + # Hermes probes /models for optional pricing and context metadata. + # Gym already supplies the selected model and its internal proxy does not + # expose model discovery, so skip only these optional metadata lookups. + from agent import model_metadata as hermes_model_metadata + from agent import usage_pricing as hermes_usage_pricing + + def empty_endpoint_model_metadata(*args, **kwargs): + return {} + + hermes_model_metadata.fetch_endpoint_model_metadata = empty_endpoint_model_metadata + hermes_usage_pricing.fetch_endpoint_model_metadata = empty_endpoint_model_metadata + + client = ServerClient.model_construct(global_config_dict={}) + client._build_server_base_url = lambda _cfg: model_url + + base = { + "host": "0.0.0.0", + "port": 0, + "name": "legal_agent_bench_inner_agent", + "entrypoint": "app.py", + "resources_server": ResourcesServerRef(name="legal_agent_bench", type="resources_servers"), + "model_server": ModelServerRef(name="policy_model", type="responses_api_models"), + } + kwargs = {key: value for key, value in base.items() if key in config_class.model_fields} + kwargs.update(runner.get("agent_kwargs") or {}) + config = config_class(**kwargs) + agent = agent_class(config=config, server_client=client) + + if hasattr(agent, "resolve_model_base_url"): + object.__setattr__(agent, "resolve_model_base_url", lambda *args, **kwargs: model_url_v1) + if hasattr(agent, "_resolve_model_base_url"): + object.__setattr__(agent, "_resolve_model_base_url", lambda *args, **kwargs: model_url_v1) + if hasattr(agent, "_resolve_base_url"): + object.__setattr__(agent, "_resolve_base_url", lambda *args, **kwargs: model_url) + + body = NeMoGymResponseCreateParamsNonStreaming.model_validate(runner["responses_create_params"]) + response_kwargs = {"body": body} + if "request" in inspect.signature(agent.responses).parameters: + response_kwargs["request"] = None + response = asyncio.run(agent.responses(**response_kwargs)) + Path("/trajectories_mount/response.json").write_text(response.model_dump_json()) +except Exception as exc: + write_status(ok=False, phase="agent_execution", error=f"{type(exc).__name__}: {exc}") + raise +else: + write_status(ok=True, phase="complete") +""" + + +class LegalAgentBenchAgentConfig(BaseResponsesAPIAgentConfig): + resources_server: ResourcesServerRef + model_server: ModelServerRef + agent_server_module: str + agent_server_class: str + agent_config_class: str + agent_kwargs: dict[str, Any] = Field(default_factory=dict) + + runtime_tasks_dir: str = str(DEFAULT_RUNTIME_TASKS_DIR) + skills_dir: str = str(DEFAULT_SKILLS_DIR) + concurrency: int = Field(default=1, ge=1) + agent_timeout_seconds: int = Field(default=10800, ge=1) + model_connect_timeout_seconds: int = Field(default=10, ge=1) + verifier_timeout_seconds: int = Field(default=3600, ge=1) + runtime_build_timeout_seconds: int = Field(default=3600, ge=1) + image_build_timeout_seconds: int = Field(default=3600, ge=1) + sandbox_ttl_seconds: int = Field(default=14400, ge=1) + docker_network: Optional[str] = "host" + sandbox_model_base_url: Optional[str] = None + image_repository: str = "nemo-gym-legal-agent-bench" + results_dir: str = "results/legal_agent_bench" + + +class LegalAgentBenchRunRequest(BaseRunRequest): + model_config = ConfigDict(extra="allow") + instance_id: str + + +class LegalAgentBenchAgentResponse(BaseVerifyResponse): + model_config = ConfigDict(extra="allow") + instance_id: str + criteria_pass_rate: float = 0.0 + judge_error_count: int = 0 + verifier_error: int = 0 + mask_sample: bool = False + agent_failed: bool = False + model_connection_failed: bool = False + agent_timed_out: bool = False + verifier_failed: bool = False + verifier_timed_out: bool = False + sandbox_failed: bool = False + task_failed: bool = False + configuration_failed: bool = False + failure_reason: Optional[str] = None + artifact_dir: Optional[str] = None + run_summary_path: Optional[str] = None + agent_trace_path: Optional[str] = None + agent_stdout_path: Optional[str] = None + agent_stderr_path: Optional[str] = None + verifier_report_path: Optional[str] = None + output_dir: Optional[str] = None + + +class LegalAgentBenchTaskError(ValueError): + """The requested LAB task is unsafe, unknown, incomplete, or malformed.""" + + +class LegalAgentBenchConfigurationError(ValueError): + """The selected Gym agent or its LAB configuration is invalid.""" + + +def agent_key(agent_server_module: str) -> str: + parts = agent_server_module.split(".") + if len(parts) < 2 or parts[-1] != "app": + raise LegalAgentBenchConfigurationError(f"Agent module must end in '.app': {agent_server_module!r}") + key = parts[-2] + if not key.replace("_", "").isalnum(): + raise LegalAgentBenchConfigurationError(f"Invalid agent module key: {key!r}") + return key + + +def _results_segment(value: str, *, fallback: str) -> str: + segment = "".join( + character if character.isascii() and (character.isalnum() or character in "._-") else "-" + for character in value + ) + segment = segment.strip("._-") + return segment[:120] or fallback + + +def _results_session_dir( + results_root: Path, + *, + agent_server_module: str, + model_name: str, + timestamp: float, + session_id: str, +) -> Path: + harness = agent_key(agent_server_module).removesuffix("_agent") + date = time.strftime("%Y%m%d", time.localtime(timestamp)) + clock = time.strftime("%H%M%S", time.localtime(timestamp)) + model = _results_segment(model_name, fallback="unknown_model") + return results_root / f"{harness}_jobs" / model / f"{date}-{clock}_{session_id}" + + +def resolve_agent_setup_script(agent_server_module: str) -> Path: + key = agent_key(agent_server_module) + script = PARENT_DIR / "responses_api_agents" / key / "scripts" / f"{key}_deps.sh" + if not script.is_file(): + raise LegalAgentBenchConfigurationError( + f"Configurable LAB agent {key!r} requires dependency setup script {script.relative_to(PARENT_DIR)}" + ) + return script + + +def _recipe_hash(paths: list[Path], *, values: list[str] | None = None) -> str: + digest = hashlib.sha256() + for path in paths: + digest.update(str(path).encode()) + if path.is_file(): + digest.update(path.read_bytes()) + elif path.is_dir(): + for child in sorted(item for item in path.rglob("*") if item.is_file()): + if "__pycache__" in child.parts or child.suffix in {".pyc", ".pyo"}: + continue + digest.update(child.relative_to(path).as_posix().encode()) + digest.update(child.read_bytes()) + for value in values or []: + digest.update(value.encode()) + return digest.hexdigest() + + +def agent_runtime_env(agent_server_module: str, agent_kwargs: dict[str, Any]) -> dict[str, str]: + key = agent_key(agent_server_module) + pin = AGENT_CLI_PINS.get(key) + if pin is None: + return {} + field, environment_variable, package = pin + version = agent_kwargs.get(field) + if not isinstance(version, str) or not version.strip(): + raise LegalAgentBenchConfigurationError( + f"Configurable LAB agent {key!r} requires a pinned agent_kwargs.{field}" + ) + return {environment_variable: f"{package}@{version.strip()}"} + + +def ensure_agent_runtime( + agent_server_module: str, + *, + agent_kwargs: dict[str, Any], + image: str, + docker_network: Optional[str], + timeout_seconds: int, +) -> Path: + key = agent_key(agent_server_module) + script = resolve_agent_setup_script(agent_server_module) + requirements = PARENT_DIR / "responses_api_agents" / key / "requirements.txt" + runtime_env = agent_runtime_env(agent_server_module, agent_kwargs) + docker = shutil.which("docker") + if not docker: + raise FileNotFoundError("Docker CLI is required to provision Legal Agent Bench agent dependencies") + image_info = subprocess.run( + [docker, "image", "inspect", "--format", "{{.Id}}:{{.Os}}/{{.Architecture}}", image], + check=True, + capture_output=True, + text=True, + timeout=60, + ).stdout.strip() + recipe = _recipe_hash( + [ + PORTABLE_PYTHON_SH, + script, + requirements, + PARENT_DIR / "pyproject.toml", + PARENT_DIR / "nemo_gym", + ], + values=[image_info, json.dumps(runtime_env, sort_keys=True)], + ) + deps_dir = PACKAGE_DIR / ".deps" / key + sentinel = deps_dir / ".installed" + if sentinel.is_file() and sentinel.read_text().strip() == recipe: + return deps_dir + + if deps_dir.exists(): + shutil.rmtree(deps_dir) + deps_dir.mkdir(parents=True) + env = { + "PORTABLE_PYTHON_SH": ( + "/nemo_gym_mount/responses_api_agents/legal_agent_bench_agent/setup_scripts/_portable_python.sh" + ), + "DEPS_DIR": "/agent_deps", + "NEMO_GYM_ROOT": "/nemo_gym_mount", + "HOME": "/tmp", + **runtime_env, + } + command = [ + docker, + "run", + "--rm", + "--user", + f"{os.getuid()}:{os.getgid()}", + "--volume", + f"{PARENT_DIR}:/nemo_gym_mount:ro", + "--volume", + f"{deps_dir}:/agent_deps", + ] + if docker_network: + command.extend(["--network", docker_network]) + for name, value in env.items(): + command.extend(["--env", f"{name}={value}"]) + command.extend([image, "bash", f"/nemo_gym_mount/{script.relative_to(PARENT_DIR).as_posix()}"]) + subprocess.run(command, check=True, timeout=timeout_seconds) + sentinel.write_text(recipe) + return deps_dir + + +def _empty_response(model_name: str) -> NeMoGymResponse: + return NeMoGymResponse.model_validate( + { + "id": f"resp_{uuid4().hex}", + "created_at": int(time.time()), + "model": model_name or "policy_model", + "object": "response", + "output": [], + "parallel_tool_calls": False, + "tool_choice": "auto", + "tools": [], + } + ) + + +def sandbox_model_url( + model_url: str, + *, + docker_network: Optional[str], + platform_name: Optional[str] = None, +) -> str: + """Translate host-loopback model URLs into addresses reachable from a Docker sandbox.""" + parsed = urlsplit(model_url) + hostname = (parsed.hostname or "").lower() + platform_name = platform_name or sys.platform + docker_desktop = platform_name == "darwin" or platform_name.startswith("win") + uses_bridge = docker_network != "host" + if hostname not in {"localhost", "127.0.0.1", "0.0.0.0", "::1", "::"} or not (docker_desktop or uses_bridge): + return model_url + + userinfo = parsed.netloc.rsplit("@", 1)[0] + "@" if "@" in parsed.netloc else "" + port = f":{parsed.port}" if parsed.port is not None else "" + return urlunsplit(parsed._replace(netloc=f"{userinfo}host.docker.internal{port}")) + + +def _response_output_text(response: NeMoGymResponse) -> str: + parts: list[str] = [] + for item in response.output: + if getattr(item, "type", None) != "message": + continue + for content in getattr(item, "content", []) or []: + text = getattr(content, "text", None) + if text: + parts.append(str(text)) + return "\n".join(parts).strip() + + +def agent_response_failure(response: NeMoGymResponse, agent_server_module: str) -> Optional[str]: + """Return a harness-failure reason without treating normal task-quality failures as infrastructure.""" + if response.error is not None: + return f"Agent returned an error response: {response.error}" + if response.incomplete_details is not None: + return f"Agent returned an incomplete response: {response.incomplete_details}" + if not response.output: + return "Agent produced an empty trajectory" + + key = agent_key(agent_server_module) + if key == "hermes_agent": + message_items = [item for item in response.output if getattr(item, "type", None) == "message"] + synthetic_messages = bool(message_items) and all( + getattr(item, "prompt_token_ids", None) == [0] and getattr(item, "generation_token_ids", None) == [0] + for item in message_items + ) + has_tool_activity = any(getattr(item, "type", None) != "message" for item in response.output) + if synthetic_messages and not has_tool_activity: + detail = _response_output_text(response) or "no assistant message" + return f"Hermes produced no model trajectory: {detail}" + + usage = response.usage + total_tokens = int(getattr(usage, "total_tokens", 0) or 0) if usage is not None else 0 + has_tool_activity = any(getattr(item, "type", None) != "message" for item in response.output) + if total_tokens == 0 and not has_tool_activity and not _response_output_text(response): + return f"{key} produced no model activity" + return None + + +def _task_name(instance_id: str) -> str: + alias, separator, task_name = instance_id.partition("::") + if separator != "::" or alias != DATASET_ALIAS or not task_name: + raise LegalAgentBenchTaskError(f"instance_id must be '{DATASET_ALIAS}::', got {instance_id!r}") + path = PurePosixPath(task_name) + if len(path.parts) != 1 or path.parts[0] in {"", ".", ".."}: + raise LegalAgentBenchTaskError(f"Unsafe Legal Agent Bench task name: {task_name!r}") + return task_name + + +def resolve_task_dir(runtime_tasks_dir: str | Path, instance_id: str) -> Path: + root = resolve_repo_path(runtime_tasks_dir) + task_dir = (root / _task_name(instance_id)).resolve() + if task_dir.parent != root or not task_dir.is_dir(): + raise LegalAgentBenchTaskError(f"Legal Agent Bench runtime task not found: {task_dir}") + required = ("instruction.md", "task.json", "task.toml", "documents", "environment", "tests") + missing = [name for name in required if not (task_dir / name).exists()] + if missing: + raise LegalAgentBenchTaskError( + f"Legal Agent Bench task {task_dir.name} is incomplete; missing: {', '.join(missing)}" + ) + return task_dir + + +def _load_skill_prompt(skills_dir: Path) -> str: + try: + validate_harness_skills(skills_dir) + sections = [] + for name in REQUIRED_SKILLS: + skill_path = skills_dir / name / "SKILL.md" + sections.append(f"\n\n## Skill: {name}\n\n{skill_path.read_text(encoding='utf-8')}") + except (FileNotFoundError, ValueError) as exc: + raise LegalAgentBenchConfigurationError(f"Invalid LAB skills configuration: {exc}") from exc + return "".join(sections) + + +def compose_agent_input( + task_dir: Path, + skills_dir: Path, + params: NeMoGymResponseCreateParamsNonStreaming, +) -> NeMoGymResponseCreateParamsNonStreaming: + try: + task = json.loads((task_dir / "task.json").read_text(encoding="utf-8")) + title = task["title"] + instructions = task["instructions"] + except (OSError, KeyError, TypeError, json.JSONDecodeError) as exc: + raise LegalAgentBenchTaskError(f"Invalid LAB task configuration in {task_dir}: {exc}") from exc + system_prompt = ( + GENERIC_HARNESS_PREAMBLE + _load_skill_prompt(skills_dir) + "\n\n## Task\n\n" + f"# {title}\n\n{instructions}" + ) + return params.model_copy( + update={ + "input": [ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": INITIAL_USER_PROMPT}, + ] + } + ) + + +def _task_toml(task_dir: Path) -> dict[str, Any]: + import tomllib + + try: + with (task_dir / "task.toml").open("rb") as handle: + return tomllib.load(handle) + except (OSError, tomllib.TOMLDecodeError) as exc: + raise LegalAgentBenchTaskError(f"Invalid LAB task.toml in {task_dir}: {exc}") from exc + + +def _verifier_env(task_dir: Path) -> dict[str, str]: + return {str(key): str(value) for key, value in (_task_toml(task_dir).get("verifier", {}).get("env") or {}).items()} + + +def _sandbox_resources(task_dir: Path) -> SandboxResources: + environment = _task_toml(task_dir).get("environment") or {} + return SandboxResources( + cpu=float(environment["cpus"]) if environment.get("cpus") is not None else None, + memory_mib=int(environment["memory_mb"]) if environment.get("memory_mb") is not None else None, + disk_gib=( + max(1, int(environment["storage_mb"]) // 1024) if environment.get("storage_mb") is not None else None + ), + gpu=int(environment.get("gpus") or 0), + ) + + +def _environment_hash(environment_dir: Path) -> str: + digest = hashlib.sha256() + for path in sorted(item for item in environment_dir.rglob("*") if item.is_file()): + digest.update(path.relative_to(environment_dir).as_posix().encode()) + digest.update(path.read_bytes()) + return digest.hexdigest() + + +async def _run_process(args: list[str], *, cwd: Path, timeout: int) -> tuple[int, str, str]: + process = await asyncio.create_subprocess_exec( + *args, + cwd=cwd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + try: + stdout, stderr = await asyncio.wait_for(process.communicate(), timeout=timeout) + except asyncio.TimeoutError: + process.kill() + await process.communicate() + raise TimeoutError(f"Command timed out after {timeout}s: {args}") + return process.returncode or 0, stdout.decode(errors="replace"), stderr.decode(errors="replace") + + +class LegalAgentBenchAgent(SimpleResponsesAPIAgent): + config: LegalAgentBenchAgentConfig + model_config = ConfigDict(arbitrary_types_allowed=True) + + _sem: asyncio.Semaphore = PrivateAttr() + _image_lock: asyncio.Lock = PrivateAttr() + _runtime_lock: asyncio.Lock = PrivateAttr() + _session_results_dir: Path = PrivateAttr() + _deps_dir: Optional[Path] = PrivateAttr(default=None) + + def model_post_init(self, context: Any) -> None: + self._sem = asyncio.Semaphore(self.config.concurrency) + self._image_lock = asyncio.Lock() + self._runtime_lock = asyncio.Lock() + results_root = resolve_repo_path(self.config.results_dir) + timestamp = time.time() + self._session_results_dir = _results_session_dir( + results_root, + agent_server_module=self.config.agent_server_module, + model_name=self._model_name(), + timestamp=timestamp, + session_id=uuid4().hex[:8], + ) + super().model_post_init(context) + + async def responses( + self, + body: NeMoGymResponseCreateParamsNonStreaming = Body(), + ) -> NeMoGymResponse: + raise NotImplementedError("LegalAgentBenchAgent is task-driven through /run, not /v1/responses") + + async def _ensure_runtime(self, image: str) -> Path: + async with self._runtime_lock: + if self._deps_dir is None: + self._deps_dir = await asyncio.to_thread( + ensure_agent_runtime, + self.config.agent_server_module, + agent_kwargs=self.config.agent_kwargs, + image=image, + docker_network=self.config.docker_network, + timeout_seconds=self.config.runtime_build_timeout_seconds, + ) + return self._deps_dir + + async def _ensure_image(self, task_dir: Path) -> str: + environment_dir = task_dir / "environment" + image = f"{self.config.image_repository}:{_environment_hash(environment_dir)[:16]}" + async with self._image_lock: + docker = shutil.which("docker") + if not docker: + raise FileNotFoundError("Docker CLI is required for Legal Agent Bench") + inspect, _stdout, _stderr = await _run_process( + [docker, "image", "inspect", image], + cwd=environment_dir, + timeout=60, + ) + if inspect == 0: + return image + code, _stdout, stderr = await _run_process( + [docker, "build", "--tag", image, "."], + cwd=environment_dir, + timeout=self.config.image_build_timeout_seconds, + ) + if code != 0: + raise RuntimeError(f"Legal Agent Bench image build failed: {stderr[-2000:]}") + return image + + def _model_url(self, body: LegalAgentBenchRunRequest) -> str: + if self.config.sandbox_model_base_url: + return self.config.sandbox_model_base_url + try: + model_config = get_first_server_config_dict( + self.server_client.global_config_dict, + self.config.model_server.name, + ) + base_url = self.server_client._build_server_base_url(model_config) + except (AttributeError, KeyError, TypeError, ValueError) as exc: + raise LegalAgentBenchConfigurationError( + f"Unable to resolve policy model server {self.config.model_server.name!r}: {exc}" + ) from exc + rollout_id = self.rollout_id_from_run(body) + prefixed_url = apply_rollout_prefix(base_url, rollout_id) if rollout_id else base_url + return sandbox_model_url(prefixed_url, docker_network=self.config.docker_network) + + def _model_name(self) -> str: + global_config = getattr(getattr(self, "server_client", None), "global_config_dict", None) + configured_name = global_config.get("policy_model_name") if isinstance(global_config, Mapping) else None + return str(configured_name or self.config.model_server.name) + + def _run_dirs(self, task_name: str) -> dict[str, Path]: + task_segment = _results_segment(task_name, fallback="unknown_task") + root = self._session_results_dir / f"{task_segment}_{uuid4().hex[:8]}" + paths = { + "root": root, + "runtime": root / "runtime", + "agent_source": root / "agent_source", + "agent": root / "agent", + "lab_run": root / "agent" / "artifacts" / "lab-run", + "output": root / "agent" / "artifacts" / "lab-run" / "output", + "workspace": root / "workspace", + "verifier": root / "verifier", + } + for path in paths.values(): + path.mkdir(parents=True, exist_ok=True) + print(f"LAB rollout artifacts: {root}", flush=True) + return paths + + def _stage_agent_source(self, paths: dict[str, Path]) -> None: + key = agent_key(self.config.agent_server_module) + source = PARENT_DIR / "responses_api_agents" / key + if not source.is_dir(): + raise LegalAgentBenchConfigurationError(f"Configured Gym agent source not found: {source}") + destination = paths["agent_source"] / "responses_api_agents" / key + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copytree( + source, + destination, + ignore=shutil.ignore_patterns( + "__pycache__", + ".pytest_cache", + ".venv", + ".deps", + ".claude_node", + ".codex_node", + "configs", + "data", + "scripts", + "tests", + ), + ) + + def _write_runner_config( + self, + paths: dict[str, Path], + params: NeMoGymResponseCreateParamsNonStreaming, + model_url: str, + ) -> None: + (paths["runtime"] / "agent_runner.py").write_text(_RUNNER_SOURCE) + runner = { + "agent_server_module": self.config.agent_server_module, + "agent_server_class": self.config.agent_server_class, + "agent_config_class": self.config.agent_config_class, + "agent_kwargs": self.config.agent_kwargs, + "model_url": model_url, + "model_connect_timeout_seconds": self.config.model_connect_timeout_seconds, + "disable_endpoint_metadata_probe": agent_key(self.config.agent_server_module) == "hermes_agent", + "responses_create_params": params.model_dump(mode="json", exclude_none=True), + } + (paths["runtime"] / "runner.json").write_text(json.dumps(runner, indent=2)) + + def _provider(self) -> DockerProvider: + create: dict[str, Any] = {"network": self.config.docker_network, "pids_limit": 4096} + if sys.platform == "linux" and self.config.docker_network != "host": + create["extra_run_args"] = ["--add-host", "host.docker.internal:host-gateway"] + return DockerProvider( + create=create, + exec={"default_timeout_s": self.config.agent_timeout_seconds, "concurrency": 8}, + ) + + def _agent_sandbox( + self, + *, + image: str, + task_dir: Path, + skills_dir: Path, + deps_dir: Path, + paths: dict[str, Path], + ) -> AsyncSandbox: + volumes = [ + f"{paths['agent_source']}:/agent_source_mount:ro", + f"{deps_dir}:/agent_deps_mount:ro", + f"{task_dir / 'documents'}:/workspace/vdr:ro", + f"{skills_dir}:/workspace/skills:ro", + f"{paths['runtime']}:/trajectories_mount", + f"{paths['agent']}:/logs/agent", + f"{paths['output']}:/workspace/output", + f"{paths['workspace']}:/workspace/workspace", + ] + return AsyncSandbox( + self._provider(), + SandboxSpec( + image=image, + ttl_s=self.config.sandbox_ttl_seconds, + workdir="/workspace/output", + resources=_sandbox_resources(task_dir), + provider_options={"volumes": volumes}, + ), + ) + + def _verifier_sandbox( + self, + *, + image: str, + task_dir: Path, + paths: dict[str, Path], + ) -> AsyncSandbox: + return AsyncSandbox( + self._provider(), + SandboxSpec( + image=image, + ttl_s=self.config.sandbox_ttl_seconds, + workdir="/logs/agent/artifacts/lab-run/output", + resources=_sandbox_resources(task_dir), + provider_options={ + "volumes": [ + f"{paths['lab_run']}:/logs/agent/artifacts/lab-run:ro", + ] + }, + ), + ) + + async def _stage_and_run_verifier( + self, + sandbox: AsyncSandbox, + task_dir: Path, + paths: dict[str, Path], + ) -> tuple[dict[str, Any], bool, Optional[str]]: + with tempfile.NamedTemporaryFile(suffix=".tar.gz", delete=False) as temporary: + archive_path = Path(temporary.name) + try: + with tarfile.open(archive_path, "w:gz") as archive: + for child in sorted((task_dir / "tests").iterdir(), key=lambda item: item.name): + archive.add(child, arcname=child.name) + await sandbox.upload(archive_path, "/tmp/legal-agent-bench-tests.tar.gz") + finally: + archive_path.unlink(missing_ok=True) + + command = ( + "rm -rf /tests /logs/verifier && mkdir -p /tests /logs/verifier && " + "tar -xzf /tmp/legal-agent-bench-tests.tar.gz -C /tests && " + "bash /tests/test.sh" + ) + result = await sandbox.exec( + command, + cwd="/workspace/output", + env=_verifier_env(task_dir), + timeout_s=self.config.verifier_timeout_seconds, + user="root", + ) + + for filename in ("reward.json", "scores.json", "transcript.jsonl", "report.html", "error.json"): + exists = await sandbox.exec(f"test -f /logs/verifier/{filename}", timeout_s=30, user="root") + if exists.return_code == 0: + await sandbox.download(f"/logs/verifier/{filename}", paths["verifier"] / filename) + + timed_out = result.error_type == "timeout" + reward_path = paths["verifier"] / "reward.json" + if not reward_path.is_file(): + reason = result.stderr or result.stdout or "LAB verifier did not produce reward.json" + return {}, timed_out, reason[-2000:] + return json.loads(reward_path.read_text(encoding="utf-8")), timed_out, None + + @staticmethod + def _runner_status(paths: dict[str, Path]) -> dict[str, Any]: + status_path = paths["runtime"] / "runner_status.json" + if not status_path.is_file(): + return {} + try: + status = json.loads(status_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + return {"ok": False, "phase": "agent_execution", "error": f"Invalid runner status: {exc}"} + return ( + status + if isinstance(status, dict) + else { + "ok": False, + "phase": "agent_execution", + "error": "Invalid runner status: expected an object", + } + ) + + def _artifacts( + self, + paths: dict[str, Path], + *, + task_name: str, + model_name: str, + agent_elapsed: float, + response: NeMoGymResponse, + failure_reason: Optional[str], + ) -> None: + config = { + "agent_id": agent_key(self.config.agent_server_module), + "agent_config_id": self.config.name, + "model": model_name, + "task": task_name, + "run_id": paths["root"].name, + "tool_runtime": "gym-agent-in-docker", + "agent_server_module": self.config.agent_server_module, + "agent_server_class": self.config.agent_server_class, + "skills": list(REQUIRED_SKILLS), + } + metrics = { + "model": model_name, + "task": task_name, + "run_id": paths["root"].name, + "wall_clock_seconds": round(agent_elapsed, 3), + "agent_error": failure_reason, + } + (paths["lab_run"] / "config.json").write_text(json.dumps(config, indent=2)) + (paths["lab_run"] / "metrics.json").write_text(json.dumps(metrics, indent=2)) + (paths["agent"] / "trajectory.json").write_text(response.model_dump_json(indent=2)) + + def _write_run_summary(self, result: LegalAgentBenchAgentResponse) -> None: + if not result.artifact_dir: + return + summary_path = Path(result.artifact_dir) / "run_summary.json" + output_dir = Path(result.output_dir) if result.output_dir else None + output_files = ( + sorted(str(path.relative_to(output_dir)) for path in output_dir.rglob("*") if path.is_file()) + if output_dir and output_dir.is_dir() + else [] + ) + summary = { + "instance_id": result.instance_id, + "reward": result.reward, + "criteria_pass_rate": result.criteria_pass_rate, + "mask_sample": result.mask_sample, + "failure_reason": result.failure_reason, + "flags": { + "agent_failed": result.agent_failed, + "model_connection_failed": result.model_connection_failed, + "agent_timed_out": result.agent_timed_out, + "verifier_failed": result.verifier_failed, + "verifier_timed_out": result.verifier_timed_out, + "sandbox_failed": result.sandbox_failed, + "task_failed": result.task_failed, + "configuration_failed": result.configuration_failed, + "judge_error_count": result.judge_error_count, + "verifier_error": result.verifier_error, + }, + "paths": { + "artifact_dir": result.artifact_dir, + "agent_trace": result.agent_trace_path, + "agent_stdout": result.agent_stdout_path, + "agent_stderr": result.agent_stderr_path, + "verifier_report": result.verifier_report_path, + "output_dir": result.output_dir, + }, + "output_files": output_files, + } + summary_path.write_text(json.dumps(summary, indent=2)) + if result.mask_sample: + print( + f"LAB rollout failed: {result.failure_reason or 'unreliable result'}; artifacts={result.artifact_dir}", + flush=True, + ) + else: + print( + f"LAB rollout complete: reward={result.reward} " + f"criteria_pass_rate={result.criteria_pass_rate} artifacts={result.artifact_dir}", + flush=True, + ) + + def _response( + self, + *, + body: LegalAgentBenchRunRequest, + params: NeMoGymResponseCreateParamsNonStreaming, + response: NeMoGymResponse, + reward_data: dict[str, Any], + paths: Optional[dict[str, Path]], + agent_failed: bool = False, + model_connection_failed: bool = False, + agent_timed_out: bool = False, + verifier_failed: bool = False, + verifier_timed_out: bool = False, + sandbox_failed: bool = False, + task_failed: bool = False, + configuration_failed: bool = False, + failure_reason: Optional[str] = None, + ) -> LegalAgentBenchAgentResponse: + verifier_error = int(reward_data.get("verifier_error") or 0) + judge_errors = int(reward_data.get("judge_error_count") or 0) + unreliable = bool( + agent_failed + or model_connection_failed + or agent_timed_out + or verifier_failed + or verifier_timed_out + or sandbox_failed + or task_failed + or configuration_failed + or verifier_error + or judge_errors + ) + return LegalAgentBenchAgentResponse( + responses_create_params=params, + response=response, + reward=float(reward_data.get("reward") or 0.0), + instance_id=body.instance_id, + criteria_pass_rate=float(reward_data.get("criteria_pass_rate") or 0.0), + judge_error_count=judge_errors, + verifier_error=verifier_error, + mask_sample=unreliable, + agent_failed=agent_failed, + model_connection_failed=model_connection_failed, + agent_timed_out=agent_timed_out, + verifier_failed=verifier_failed, + verifier_timed_out=verifier_timed_out, + sandbox_failed=sandbox_failed, + task_failed=task_failed, + configuration_failed=configuration_failed, + failure_reason=failure_reason, + artifact_dir=str(paths["root"]) if paths else None, + run_summary_path=str(paths["root"] / "run_summary.json") if paths else None, + agent_trace_path=str(paths["agent"] / "trajectory.json") if paths else None, + agent_stdout_path=str(paths["agent"] / "stdout.log") if paths else None, + agent_stderr_path=str(paths["agent"] / "stderr.log") if paths else None, + verifier_report_path=( + str(paths["verifier"] / "report.html") + if paths and (paths["verifier"] / "report.html").is_file() + else None + ), + output_dir=str(paths["output"]) if paths else None, + ) + + async def run(self, request: Request, body: LegalAgentBenchRunRequest) -> LegalAgentBenchAgentResponse: + async with self._sem: + model_name = self.config.model_server.name + params = body.responses_create_params + response = _empty_response(model_name) + paths: Optional[dict[str, Path]] = None + sandbox: Optional[AsyncSandbox] = None + agent_failed = agent_timed_out = verifier_failed = verifier_timed_out = sandbox_failed = False + model_connection_failed = False + task_failed = configuration_failed = False + failure_reason: Optional[str] = None + reward_data: dict[str, Any] = {} + verifier_sandbox: Optional[AsyncSandbox] = None + + try: + model_name = self._model_name() + response = _empty_response(model_name) + task_dir = resolve_task_dir(self.config.runtime_tasks_dir, body.instance_id) + skills_dir = resolve_repo_path(self.config.skills_dir) + params = compose_agent_input(task_dir, skills_dir, params) + paths = self._run_dirs(task_dir.name) + image = await self._ensure_image(task_dir) + deps_dir = await self._ensure_runtime(image) + self._stage_agent_source(paths) + self._write_runner_config(paths, params, self._model_url(body)) + + sandbox = self._agent_sandbox( + image=image, + task_dir=task_dir, + skills_dir=skills_dir, + deps_dir=deps_dir, + paths=paths, + ) + await sandbox.start() + started = time.time() + agent_result = await sandbox.exec( + "/agent_deps_mount/bin/python /trajectories_mount/agent_runner.py", + cwd="/workspace/output", + timeout_s=self.config.agent_timeout_seconds, + ) + agent_elapsed = time.time() - started + (paths["agent"] / "stdout.log").write_text(agent_result.stdout or "") + (paths["agent"] / "stderr.log").write_text(agent_result.stderr or "") + agent_timed_out = agent_result.error_type == "timeout" + runner_status = self._runner_status(paths) + if runner_status.get("ok") is False: + agent_failed = True + model_connection_failed = runner_status.get("phase") == "model_connectivity" + failure_reason = str(runner_status.get("error") or "Agent runner failed")[-2000:] + if agent_result.return_code != 0: + agent_failed = True + failure_reason = ( + failure_reason or (agent_result.stderr or agent_result.stdout or "Agent runner failed")[-2000:] + ) + if agent_timed_out: + agent_failed = True + failure_reason = failure_reason or f"Agent timed out after {self.config.agent_timeout_seconds}s" + + response_path = paths["runtime"] / "response.json" + if response_path.is_file(): + try: + response = NeMoGymResponse.model_validate_json(response_path.read_text()) + except ValueError as exc: + agent_failed = True + failure_reason = failure_reason or f"Invalid agent response: {exc}" + else: + agent_failed = True + failure_reason = failure_reason or "Agent did not produce response.json" + if not agent_failed: + response_failure = agent_response_failure(response, self.config.agent_server_module) + if response_failure: + agent_failed = True + failure_reason = response_failure + self._artifacts( + paths, + task_name=task_dir.name, + model_name=model_name, + agent_elapsed=agent_elapsed, + response=response, + failure_reason=failure_reason, + ) + await sandbox.stop() + sandbox = None + + if not agent_failed: + verifier_sandbox = self._verifier_sandbox(image=image, task_dir=task_dir, paths=paths) + await verifier_sandbox.start() + reward_data, verifier_timed_out, verifier_failure = await self._stage_and_run_verifier( + verifier_sandbox, task_dir, paths + ) + verifier_failed = verifier_failure is not None + failure_reason = verifier_failure or failure_reason + except LegalAgentBenchTaskError as exc: + task_failed = True + failure_reason = f"{type(exc).__name__}: {exc}" + except LegalAgentBenchConfigurationError as exc: + configuration_failed = True + failure_reason = f"{type(exc).__name__}: {exc}" + except Exception as exc: + sandbox_failed = True + failure_reason = f"{type(exc).__name__}: {exc}" + finally: + if sandbox is not None: + try: + await sandbox.stop() + except Exception as exc: + sandbox_failed = True + failure_reason = failure_reason or f"Sandbox cleanup failed: {exc}" + if verifier_sandbox is not None: + try: + await verifier_sandbox.stop() + except Exception as exc: + sandbox_failed = True + failure_reason = failure_reason or f"Verifier sandbox cleanup failed: {exc}" + + result = self._response( + body=body, + params=params, + response=response, + reward_data=reward_data, + paths=paths, + agent_failed=agent_failed, + model_connection_failed=model_connection_failed, + agent_timed_out=agent_timed_out, + verifier_failed=verifier_failed, + verifier_timed_out=verifier_timed_out, + sandbox_failed=sandbox_failed, + task_failed=task_failed, + configuration_failed=configuration_failed, + failure_reason=failure_reason, + ) + self._write_run_summary(result) + return result + + +if __name__ == "__main__": + LegalAgentBenchAgent.run_webserver() diff --git a/responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_claude_code.yaml b/responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_claude_code.yaml new file mode 100644 index 0000000000..7d446f2968 --- /dev/null +++ b/responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_claude_code.yaml @@ -0,0 +1,39 @@ +config_paths: + - resources_servers/legal_agent_bench/configs/resources_only.yaml + +legal_agent_bench_claude_code_agent: + responses_api_agents: + legal_agent_bench_agent: + entrypoint: app.py + domain: agent + description: Legal Agent Bench with a config-selected Claude Code agent harness. + value: Evaluate legal-document agents independently of Harbor orchestration. + resources_server: + type: resources_servers + name: legal_agent_bench + model_server: + type: responses_api_models + name: policy_model + runtime_tasks_dir: ${legal_agent_bench.resources_servers.legal_agent_bench.harbor_tasks_dir} + skills_dir: ${legal_agent_bench.resources_servers.legal_agent_bench.harness_skills_dir} + concurrency: 1 + agent_timeout_seconds: 10800 + verifier_timeout_seconds: 3600 + agent_server_module: responses_api_agents.claude_code_agent.app + agent_server_class: ClaudeCodeAgent + agent_config_class: ClaudeCodeAgentConfig + agent_kwargs: + model: ${policy_model_name} + max_turns: null + timeout: 10800 + bare: true + claude_code_version: 2.1.211 + datasets: + - name: example + type: example + jsonl_fpath: resources_servers/legal_agent_bench/data/example.jsonl + license: MIT + - name: legal_agent_bench + type: validation + jsonl_fpath: resources_servers/legal_agent_bench/data/generated/all.jsonl + license: MIT diff --git a/responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_codex.yaml b/responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_codex.yaml new file mode 100644 index 0000000000..dfe39f106b --- /dev/null +++ b/responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_codex.yaml @@ -0,0 +1,39 @@ +config_paths: + - resources_servers/legal_agent_bench/configs/resources_only.yaml + +legal_agent_bench_codex_agent: + responses_api_agents: + legal_agent_bench_agent: + entrypoint: app.py + domain: agent + description: Legal Agent Bench with a config-selected Codex agent harness. + value: Evaluate legal-document agents independently of Harbor orchestration. + resources_server: + type: resources_servers + name: legal_agent_bench + model_server: + type: responses_api_models + name: policy_model + runtime_tasks_dir: ${legal_agent_bench.resources_servers.legal_agent_bench.harbor_tasks_dir} + skills_dir: ${legal_agent_bench.resources_servers.legal_agent_bench.harness_skills_dir} + concurrency: 1 + agent_timeout_seconds: 10800 + verifier_timeout_seconds: 3600 + agent_server_module: responses_api_agents.codex_agent.app + agent_server_class: CodexAgent + agent_config_class: CodexAgentConfig + agent_kwargs: + model: null + sandbox_mode: danger-full-access + timeout: 10800 + codex_version: 0.144.4 + cwd: /workspace/output + datasets: + - name: example + type: example + jsonl_fpath: resources_servers/legal_agent_bench/data/example.jsonl + license: MIT + - name: legal_agent_bench + type: validation + jsonl_fpath: resources_servers/legal_agent_bench/data/generated/all.jsonl + license: MIT diff --git a/responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_hermes.yaml b/responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_hermes.yaml new file mode 100644 index 0000000000..0fb6d8852e --- /dev/null +++ b/responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_hermes.yaml @@ -0,0 +1,39 @@ +config_paths: + - resources_servers/legal_agent_bench/configs/resources_only.yaml + +legal_agent_bench_hermes_agent: + responses_api_agents: + legal_agent_bench_agent: + entrypoint: app.py + domain: agent + description: Legal Agent Bench with a config-selected Hermes agent harness. + value: Evaluate legal-document agents independently of Harbor orchestration. + resources_server: + type: resources_servers + name: legal_agent_bench + model_server: + type: responses_api_models + name: policy_model + runtime_tasks_dir: ${legal_agent_bench.resources_servers.legal_agent_bench.harbor_tasks_dir} + skills_dir: ${legal_agent_bench.resources_servers.legal_agent_bench.harness_skills_dir} + concurrency: 1 + agent_timeout_seconds: 10800 + verifier_timeout_seconds: 3600 + agent_server_module: responses_api_agents.hermes_agent.app + agent_server_class: HermesAgent + agent_config_class: HermesAgentConfig + agent_kwargs: + max_turns: 90 + terminal_backend: local + terminal_timeout: 180 + compression_enabled: true + compression_threshold: 0.85 + datasets: + - name: example + type: example + jsonl_fpath: resources_servers/legal_agent_bench/data/example.jsonl + license: MIT + - name: legal_agent_bench + type: validation + jsonl_fpath: resources_servers/legal_agent_bench/data/generated/all.jsonl + license: MIT diff --git a/responses_api_agents/legal_agent_bench_agent/requirements.txt b/responses_api_agents/legal_agent_bench_agent/requirements.txt new file mode 100644 index 0000000000..eae22a6b0c --- /dev/null +++ b/responses_api_agents/legal_agent_bench_agent/requirements.txt @@ -0,0 +1 @@ +-e nemo-gym[dev,sandbox] @ ../../ diff --git a/responses_api_agents/legal_agent_bench_agent/setup_scripts/_portable_python.sh b/responses_api_agents/legal_agent_bench_agent/setup_scripts/_portable_python.sh new file mode 100755 index 0000000000..2319f9da62 --- /dev/null +++ b/responses_api_agents/legal_agent_bench_agent/setup_scripts/_portable_python.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# Shared helper for a relocatable CPython under $DEPS_DIR. +set -euo pipefail + +export PYTHONNOUSERSITE=1 + +PORTABLE_PYTHON_VERSION="${PORTABLE_PYTHON_VERSION:-3.12.8}" +PORTABLE_PYTHON_RELEASE="${PORTABLE_PYTHON_RELEASE:-20241219}" +if [ -z "${PORTABLE_PYTHON_ARCH:-}" ]; then + case "$(uname -m)" in + x86_64) PORTABLE_PYTHON_ARCH="x86_64-unknown-linux-gnu" ;; + aarch64|arm64) PORTABLE_PYTHON_ARCH="aarch64-unknown-linux-gnu" ;; + *) + echo "Unsupported portable Python architecture: $(uname -m)" >&2 + exit 1 + ;; + esac +fi + +install_portable_python() { + if [ -x "$DEPS_DIR/bin/python3" ]; then + return 0 + fi + local url="https://github.com/astral-sh/python-build-standalone/releases/download/${PORTABLE_PYTHON_RELEASE}/cpython-${PORTABLE_PYTHON_VERSION}+${PORTABLE_PYTHON_RELEASE}-${PORTABLE_PYTHON_ARCH}-install_only.tar.gz" + curl -fsSL "$url" | tar xz -C "$DEPS_DIR" --strip-components=1 + "$DEPS_DIR/bin/python3" -m pip install --upgrade pip +} + +install_nemo_gym_deps() { + local build_root + build_root="$(mktemp -d)" + mkdir -p "$build_root/cache" + cp "$NEMO_GYM_ROOT/pyproject.toml" "$NEMO_GYM_ROOT/README.md" "$build_root/" + cp -a "$NEMO_GYM_ROOT/nemo_gym" "$build_root/" + "$DEPS_DIR/bin/python3" -m pip install "$build_root" + rm -rf "$build_root" +} diff --git a/responses_api_agents/legal_agent_bench_agent/tests/__init__.py b/responses_api_agents/legal_agent_bench_agent/tests/__init__.py new file mode 100644 index 0000000000..52a7a9daf0 --- /dev/null +++ b/responses_api_agents/legal_agent_bench_agent/tests/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/responses_api_agents/legal_agent_bench_agent/tests/test_app.py b/responses_api_agents/legal_agent_bench_agent/tests/test_app.py new file mode 100644 index 0000000000..bc5a45ee2b --- /dev/null +++ b/responses_api_agents/legal_agent_bench_agent/tests/test_app.py @@ -0,0 +1,937 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Tests for the configurable Legal Agent Bench Gym-agent runner.""" + +from __future__ import annotations + +import json +from pathlib import Path +from types import SimpleNamespace + +import pytest +from omegaconf import OmegaConf + +from nemo_gym.config_types import ModelServerRef, ResourcesServerRef +from nemo_gym.openai_utils import NeMoGymResponseCreateParamsNonStreaming +from responses_api_agents.legal_agent_bench_agent import app + + +def _config(**overrides) -> app.LegalAgentBenchAgentConfig: + values = { + "host": "0.0.0.0", + "port": 10000, + "name": "lab_test_agent", + "entrypoint": "app.py", + "resources_server": ResourcesServerRef(name="lab", type="resources_servers"), + "model_server": ModelServerRef(name="policy_model", type="responses_api_models"), + "agent_server_module": "responses_api_agents.hermes_agent.app", + "agent_server_class": "HermesAgent", + "agent_config_class": "HermesAgentConfig", + } + values.update(overrides) + return app.LegalAgentBenchAgentConfig(**values) + + +def _task_tree(tmp_path: Path, task_name: str = "area__task") -> tuple[Path, Path]: + root = tmp_path / "tasks" + task = root / task_name + for directory in ("documents", "environment/harness", "tests"): + (task / directory).mkdir(parents=True, exist_ok=True) + (task / "instruction.md").write_text("Do the task") + (task / "task.json").write_text( + json.dumps({"title": "Legal task", "instructions": "Write a memo.", "criteria": [{"id": "1"}]}) + ) + (task / "task.toml").write_text( + """ +version = "1.0" +[agent] +timeout_sec = 30 +[verifier] +timeout_sec = 60 +[verifier.env] +LAB_JUDGE_API_KEY = "secret" # pragma: allowlist secret +LAB_JUDGE_MODEL = "judge" +[environment] +cpus = 2 +memory_mb = 4096 +storage_mb = 10240 +""" + ) + (task / "environment" / "Dockerfile").write_text("FROM python:3.12-slim\nWORKDIR /workspace/output\n") + (task / "environment" / "harness" / "runner.py").write_text("print('ok')\n") + (task / "tests" / "test.sh").write_text("#!/bin/bash\n") + return root, task + + +def _skills(tmp_path: Path) -> Path: + root = tmp_path / "skills" + for name in app.REQUIRED_SKILLS: + (root / name).mkdir(parents=True) + (root / name / "SKILL.md").write_text(f"# {name}\nUse {name}.") + return root + + +def _successful_response(text: str = "Done") -> app.NeMoGymResponse: + return app.NeMoGymResponse.model_validate( + { + "id": "resp-success", + "created_at": 1, + "model": "policy", + "object": "response", + "output": [ + { + "id": "msg-success", + "content": [{"annotations": [], "text": text, "type": "output_text"}], + "role": "assistant", + "status": "completed", + "type": "message", + } + ], + "parallel_tool_calls": False, + "tool_choice": "auto", + "tools": [], + "usage": { + "input_tokens": 1, + "input_tokens_details": {"cached_tokens": 0}, + "output_tokens": 1, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 2, + }, + } + ) + + +@pytest.mark.parametrize( + ("module", "expected"), + [ + ("responses_api_agents.hermes_agent.app", "hermes_agent"), + ("responses_api_agents.claude_code_agent.app", "claude_code_agent"), + ("responses_api_agents.codex_agent.app", "codex_agent"), + ], +) +def test_agent_key(module, expected) -> None: + assert app.agent_key(module) == expected + + +@pytest.mark.parametrize("module", ["hermes_agent", "responses_api_agents.bad-name.app", "x.y.z"]) +def test_agent_key_rejects_invalid_modules(module) -> None: + with pytest.raises(ValueError): + app.agent_key(module) + + +def test_all_supported_agents_have_dependency_scripts() -> None: + for module in ( + "responses_api_agents.hermes_agent.app", + "responses_api_agents.claude_code_agent.app", + "responses_api_agents.codex_agent.app", + ): + assert app.resolve_agent_setup_script(module).is_file() + + +def test_dependency_runtime_cache_is_harness_and_recipe_specific(monkeypatch, tmp_path) -> None: + package_dir = tmp_path / "legal_agent_bench_agent" + package_dir.mkdir() + portable = package_dir / "_portable_python.sh" + portable.write_text("portable-v1") + script = tmp_path / "hermes_agent_deps.sh" + script.write_text("hermes-v1") + (tmp_path / "pyproject.toml").write_text("[project]\nname = 'nemo-gym'\n") + (tmp_path / "nemo_gym").mkdir() + (tmp_path / "nemo_gym" / "runtime.py").write_text("VERSION = 1\n") + calls = [] + + monkeypatch.setattr(app, "PACKAGE_DIR", package_dir) + monkeypatch.setattr(app, "PARENT_DIR", tmp_path) + monkeypatch.setattr(app, "PORTABLE_PYTHON_SH", portable) + monkeypatch.setattr(app, "resolve_agent_setup_script", lambda module: script) + + def fake_run(args, **kwargs): + calls.append((args, kwargs)) + return SimpleNamespace(stdout="sha256:image:linux/arm64\n") + + monkeypatch.setattr(app.shutil, "which", lambda name: "/usr/bin/docker") + monkeypatch.setattr(app.subprocess, "run", fake_run) + + first = app.ensure_agent_runtime( + "responses_api_agents.hermes_agent.app", + agent_kwargs={}, + image="lab:image", + docker_network="host", + timeout_seconds=60, + ) + second = app.ensure_agent_runtime( + "responses_api_agents.hermes_agent.app", + agent_kwargs={}, + image="lab:image", + docker_network="host", + timeout_seconds=60, + ) + script.write_text("hermes-v2") + third = app.ensure_agent_runtime( + "responses_api_agents.hermes_agent.app", + agent_kwargs={}, + image="lab:image", + docker_network="host", + timeout_seconds=60, + ) + (tmp_path / "pyproject.toml").write_text("[project]\nname = 'nemo-gym-updated'\n") + fourth = app.ensure_agent_runtime( + "responses_api_agents.hermes_agent.app", + agent_kwargs={}, + image="lab:image", + docker_network="host", + timeout_seconds=60, + ) + (tmp_path / "nemo_gym" / "runtime.py").write_text("VERSION = 2\n") + fifth = app.ensure_agent_runtime( + "responses_api_agents.hermes_agent.app", + agent_kwargs={}, + image="lab:image", + docker_network="host", + timeout_seconds=60, + ) + + assert first == second == third == fourth == fifth == package_dir / ".deps" / "hermes_agent" + docker_runs = [call for call in calls if call[0][1] == "run"] + assert len(docker_runs) == 4 + assert f"{first}:/agent_deps" in docker_runs[0][0] + assert "--network" in docker_runs[0][0] + + +@pytest.mark.parametrize( + ("module", "kwargs", "expected"), + [ + ( + "responses_api_agents.claude_code_agent.app", + {"claude_code_version": "2.1.211"}, + {"CLAUDE_SPEC": "@anthropic-ai/claude-code@2.1.211"}, + ), + ( + "responses_api_agents.codex_agent.app", + {"codex_version": "0.144.4"}, + {"CODEX_SPEC": "@openai/codex@0.144.4"}, + ), + ("responses_api_agents.hermes_agent.app", {}, {}), + ], +) +def test_agent_runtime_env_uses_configured_harness_pin(module, kwargs, expected) -> None: + assert app.agent_runtime_env(module, kwargs) == expected + + +@pytest.mark.parametrize( + ("module", "field"), + [ + ("responses_api_agents.claude_code_agent.app", "claude_code_version"), + ("responses_api_agents.codex_agent.app", "codex_version"), + ], +) +def test_agent_runtime_env_rejects_missing_harness_pin(module, field) -> None: + with pytest.raises(app.LegalAgentBenchConfigurationError, match=field): + app.agent_runtime_env(module, {}) + + +def test_dependency_provisioning_uses_pin_and_invalidates_when_it_changes(monkeypatch, tmp_path) -> None: + package_dir = tmp_path / "legal_agent_bench_agent" + package_dir.mkdir() + portable = package_dir / "_portable_python.sh" + portable.write_text("portable") + script = tmp_path / "claude_code_agent_deps.sh" + script.write_text("claude") + agent_dir = tmp_path / "responses_api_agents" / "claude_code_agent" + agent_dir.mkdir(parents=True) + (agent_dir / "requirements.txt").write_text("nemo-gym\n") + (tmp_path / "pyproject.toml").write_text("[project]\nname = 'nemo-gym'\n") + (tmp_path / "nemo_gym").mkdir() + (tmp_path / "nemo_gym" / "runtime.py").write_text("VERSION = 1\n") + calls = [] + + monkeypatch.setattr(app, "PACKAGE_DIR", package_dir) + monkeypatch.setattr(app, "PARENT_DIR", tmp_path) + monkeypatch.setattr(app, "PORTABLE_PYTHON_SH", portable) + monkeypatch.setattr(app, "resolve_agent_setup_script", lambda module: script) + + def fake_run(args, **kwargs): + calls.append((args, kwargs)) + return SimpleNamespace(stdout="sha256:image:linux/arm64\n") + + monkeypatch.setattr(app.shutil, "which", lambda name: "/usr/bin/docker") + monkeypatch.setattr(app.subprocess, "run", fake_run) + + for version in ("2.1.211", "2.1.212"): + app.ensure_agent_runtime( + "responses_api_agents.claude_code_agent.app", + agent_kwargs={"claude_code_version": version}, + image="lab:image", + docker_network="host", + timeout_seconds=60, + ) + + docker_runs = [call[0] for call in calls if call[0][1] == "run"] + assert len(docker_runs) == 2 + assert "CLAUDE_SPEC=@anthropic-ai/claude-code@2.1.211" in docker_runs[0] + assert "CLAUDE_SPEC=@anthropic-ai/claude-code@2.1.212" in docker_runs[1] + + +def test_config_defaults_are_docker_and_single_concurrency() -> None: + config = _config() + assert config.concurrency == 1 + assert config.docker_network == "host" + assert config.results_dir == "results/legal_agent_bench" + assert config.agent_timeout_seconds == 10800 + assert config.model_connect_timeout_seconds == 10 + assert config.verifier_timeout_seconds == 3600 + assert config.runtime_build_timeout_seconds == 3600 + + +@pytest.mark.parametrize( + ("module", "job_dir"), + [ + ("responses_api_agents.hermes_agent.app", "hermes_jobs"), + ("responses_api_agents.claude_code_agent.app", "claude_code_jobs"), + ("responses_api_agents.codex_agent.app", "codex_jobs"), + ], +) +def test_results_session_dir_is_harness_date_and_model_browsable(tmp_path, module, job_dir) -> None: + session = app._results_session_dir( + tmp_path, + agent_server_module=module, + model_name="nvidia/model name", + timestamp=1785260796, + session_id="0b6511a6", + ) + + timestamp = app.time.strftime("%Y%m%d-%H%M%S", app.time.localtime(1785260796)) + assert session == tmp_path / job_dir / "nvidia-model-name" / f"{timestamp}_0b6511a6" + + +def test_results_segment_rejects_path_syntax() -> None: + assert app._results_segment("../../model/name", fallback="unknown") == "model-name" + assert app._results_segment("...", fallback="unknown") == "unknown" + + +def test_agent_construction_does_not_create_empty_results_session(tmp_path) -> None: + runner = app.LegalAgentBenchAgent.model_construct( + config=_config(results_dir=str(tmp_path)), + server_client=SimpleNamespace(global_config_dict=OmegaConf.create({"policy_model_name": "org/model"})), + ) + + assert runner._session_results_dir.parent == tmp_path / "hermes_jobs" / "org-model" + assert not runner._session_results_dir.exists() + + paths = runner._run_dirs("area__task") + + assert paths["root"].is_dir() + assert runner._session_results_dir.is_dir() + + +def test_model_name_reads_omegaconf_global_config() -> None: + runner = app.LegalAgentBenchAgent.model_construct(config=_config()) + runner.server_client = SimpleNamespace( + global_config_dict=OmegaConf.create({"policy_model_name": "nvidia/nemotron-3-ultra"}) + ) + + assert runner._model_name() == "nvidia/nemotron-3-ultra" + + +def test_resolve_task_dir_accepts_known_task(tmp_path) -> None: + root, task = _task_tree(tmp_path) + assert app.resolve_task_dir(root, "legal_agent_bench::area__task") == task.resolve() + + +@pytest.mark.parametrize( + "instance_id", + [ + "wrong::area__task", + "legal_agent_bench::../task", + "legal_agent_bench::nested/task", + "legal_agent_bench::", + "area__task", + ], +) +def test_resolve_task_dir_rejects_unsafe_or_invalid_ids(tmp_path, instance_id) -> None: + root, _task = _task_tree(tmp_path) + with pytest.raises((ValueError, FileNotFoundError)): + app.resolve_task_dir(root, instance_id) + + +def test_resolve_task_dir_rejects_incomplete_task(tmp_path) -> None: + root = tmp_path / "tasks" + (root / "broken").mkdir(parents=True) + with pytest.raises(app.LegalAgentBenchTaskError, match="incomplete"): + app.resolve_task_dir(root, "legal_agent_bench::broken") + + +def test_compose_agent_input_uses_task_and_skills_without_rubric(monkeypatch, tmp_path) -> None: + _root, task = _task_tree(tmp_path) + skills = _skills(tmp_path) + monkeypatch.setattr(app, "validate_harness_skills", lambda path: Path(path)) + params = NeMoGymResponseCreateParamsNonStreaming(input=[], temperature=0.5) + + result = app.compose_agent_input(task, skills, params) + + assert result.temperature == 0.5 + assert len(result.input) == 2 + system = result.input[0]["content"] + assert "Write a memo." in system + assert "Skill: docx" in system + assert "/workspace/vdr" in system + assert "criteria" not in system + assert result.input[1]["content"] == app.INITIAL_USER_PROMPT + + +def test_verifier_credentials_are_read_separately_from_sandbox_resources(tmp_path) -> None: + _root, task = _task_tree(tmp_path) + assert app._verifier_env(task) == { + "LAB_JUDGE_API_KEY": "secret", # pragma: allowlist secret + "LAB_JUDGE_MODEL": "judge", + } + resources = app._sandbox_resources(task) + assert resources.cpu == 2 + assert resources.memory_mib == 4096 + assert resources.disk_gib == 10 + + +def test_environment_hash_is_deterministic_and_content_sensitive(tmp_path) -> None: + _root, task = _task_tree(tmp_path) + environment = task / "environment" + first = app._environment_hash(environment) + assert first == app._environment_hash(environment) + (environment / "Dockerfile").write_text("FROM python:3.12-slim\nRUN true\n") + assert app._environment_hash(environment) != first + + +def test_empty_response_is_schema_valid() -> None: + response = app._empty_response("policy") + assert response.model == "policy" + assert response.output == [] + + +def test_runner_config_preserves_dynamic_agent_configuration(tmp_path) -> None: + runner = app.LegalAgentBenchAgent.model_construct( + config=_config(agent_kwargs={"max_turns": 17, "terminal_backend": "local"}) + ) + paths = {"runtime": tmp_path} + params = NeMoGymResponseCreateParamsNonStreaming(input="Do the work") + + runner._write_runner_config(paths, params, "http://model.internal:8000") + + payload = json.loads((tmp_path / "runner.json").read_text()) + assert payload["agent_server_module"] == "responses_api_agents.hermes_agent.app" + assert payload["agent_server_class"] == "HermesAgent" + assert payload["agent_config_class"] == "HermesAgentConfig" + assert payload["agent_kwargs"]["max_turns"] == 17 + assert payload["model_url"] == "http://model.internal:8000" + assert payload["model_connect_timeout_seconds"] == 10 + assert payload["disable_endpoint_metadata_probe"] is True + assert "inspect.signature(agent.responses)" in (tmp_path / "agent_runner.py").read_text() + assert "socket.create_connection" in (tmp_path / "agent_runner.py").read_text() + assert "runner_status.json" in (tmp_path / "agent_runner.py").read_text() + assert "hermes_model_metadata.fetch_endpoint_model_metadata" in (tmp_path / "agent_runner.py").read_text() + assert "hermes_usage_pricing.fetch_endpoint_model_metadata" in (tmp_path / "agent_runner.py").read_text() + + +def test_runner_config_keeps_endpoint_metadata_for_non_hermes_harnesses(tmp_path) -> None: + runner = app.LegalAgentBenchAgent.model_construct( + config=_config( + agent_server_module="responses_api_agents.codex_agent.app", + agent_server_class="CodexAgent", + agent_config_class="CodexAgentConfig", + ) + ) + paths = {"runtime": tmp_path} + params = NeMoGymResponseCreateParamsNonStreaming(input="Do the work") + + runner._write_runner_config(paths, params, "http://model.internal:8000") + + payload = json.loads((tmp_path / "runner.json").read_text()) + assert payload["disable_endpoint_metadata_probe"] is False + + +def test_stage_agent_source_copies_only_selected_runtime_package(monkeypatch, tmp_path) -> None: + repository = tmp_path / "repository" + package = repository / "responses_api_agents" / "hermes_agent" + (package / "tests").mkdir(parents=True) + (package / "data").mkdir() + (package / "runtime_helpers").mkdir() + (package / "app.py").write_text("VALUE = 1\n") + (package / "runtime_helpers" / "tool.py").write_text("VALUE = 2\n") + (package / "tests" / "rubric.py").write_text("SECRET = True\n") + (package / "data" / "example.jsonl").write_text("{}\n") + (repository / "resources_servers" / "legal_agent_bench" / "data" / "runtime").mkdir(parents=True) + (repository / "resources_servers" / "legal_agent_bench" / "data" / "runtime" / "rubric.json").write_text("{}") + paths = {"agent_source": tmp_path / "staged"} + paths["agent_source"].mkdir() + runner = app.LegalAgentBenchAgent.model_construct(config=_config()) + monkeypatch.setattr(app, "PARENT_DIR", repository) + + runner._stage_agent_source(paths) + + staged = paths["agent_source"] / "responses_api_agents" / "hermes_agent" + assert (staged / "app.py").is_file() + assert (staged / "runtime_helpers" / "tool.py").is_file() + assert not (staged / "tests").exists() + assert not (staged / "data").exists() + assert not (paths["agent_source"] / "resources_servers").exists() + + +def test_model_url_uses_override_without_server_lookup() -> None: + runner = app.LegalAgentBenchAgent.model_construct( + config=_config(sandbox_model_base_url="http://host.docker.internal:9000") + ) + runner.server_client = SimpleNamespace(global_config_dict={}) + body = app.LegalAgentBenchRunRequest( + instance_id="legal_agent_bench::area__task", + responses_create_params=NeMoGymResponseCreateParamsNonStreaming(input=[]), + ) + + assert runner._model_url(body) == "http://host.docker.internal:9000" + + +@pytest.mark.parametrize( + ("url", "network", "platform_name", "expected"), + [ + ( + "http://127.0.0.1:16300/rollout/run-1", + "host", + "darwin", + "http://host.docker.internal:16300/rollout/run-1", + ), + ("http://localhost:16300", "host", "win32", "http://host.docker.internal:16300"), + ("http://0.0.0.0:16300", None, "linux", "http://host.docker.internal:16300"), + ("http://127.0.0.1:16300", "host", "linux", "http://127.0.0.1:16300"), + ("http://model.internal:16300", "host", "darwin", "http://model.internal:16300"), + ], +) +def test_sandbox_model_url_routes_loopback_for_docker(url, network, platform_name, expected) -> None: + assert app.sandbox_model_url(url, docker_network=network, platform_name=platform_name) == expected + + +def test_linux_bridge_provider_registers_host_gateway(monkeypatch) -> None: + runner = app.LegalAgentBenchAgent.model_construct(config=_config(docker_network=None)) + monkeypatch.setattr(app.sys, "platform", "linux") + + provider = runner._provider() + + assert provider._create_config.extra_run_args == [ + "--add-host", + "host.docker.internal:host-gateway", + ] + + +def test_hermes_synthetic_connection_error_is_an_agent_failure() -> None: + response = app.NeMoGymResponse.model_validate( + { + **_successful_response().model_dump(mode="json"), + "usage": { + "input_tokens": 0, + "input_tokens_details": {"cached_tokens": 0}, + "output_tokens": 0, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 0, + }, + "output": [ + { + "id": "msg-error", + "content": [{"annotations": [], "text": "Connection error.", "type": "output_text"}], + "role": "assistant", + "status": "completed", + "type": "message", + "prompt_token_ids": [0], + "generation_token_ids": [0], + "generation_log_probs": [0.0], + } + ], + } + ) + + failure = app.agent_response_failure(response, "responses_api_agents.hermes_agent.app") + + assert failure == "Hermes produced no model trajectory: Connection error." + + +def test_real_agent_activity_is_not_treated_as_an_infrastructure_failure() -> None: + assert ( + app.agent_response_failure( + _successful_response(), + "responses_api_agents.hermes_agent.app", + ) + is None + ) + + +def test_response_masks_harness_and_verifier_failures() -> None: + runner = app.LegalAgentBenchAgent.model_construct(config=_config()) + params = NeMoGymResponseCreateParamsNonStreaming(input=[]) + body = app.LegalAgentBenchRunRequest( + instance_id="legal_agent_bench::area__task", + responses_create_params=params, + ) + + response = runner._response( + body=body, + params=params, + response=app._empty_response("policy"), + reward_data={}, + paths=None, + agent_failed=True, + model_connection_failed=True, + verifier_failed=True, + ) + + assert response.agent_failed is True + assert response.model_connection_failed is True + assert response.verifier_failed is True + assert response.mask_sample is True + + +@pytest.mark.asyncio +async def test_run_classifies_invalid_task_without_sandbox_failure(monkeypatch) -> None: + runner = app.LegalAgentBenchAgent.model_construct(config=_config()) + runner._sem = app.asyncio.Semaphore(1) + monkeypatch.setattr(runner, "_model_name", lambda: "policy") + body = app.LegalAgentBenchRunRequest( + instance_id="legal_agent_bench::../unsafe", + responses_create_params=NeMoGymResponseCreateParamsNonStreaming(input=[]), + ) + + response = await runner.run(None, body) + + assert response.task_failed is True + assert response.configuration_failed is False + assert response.sandbox_failed is False + assert response.mask_sample is True + assert "Unsafe Legal Agent Bench task name" in response.failure_reason + + +@pytest.mark.asyncio +async def test_run_classifies_bad_agent_configuration_without_sandbox_failure(monkeypatch, tmp_path) -> None: + _root, task = _task_tree(tmp_path) + skills = _skills(tmp_path) + runner = app.LegalAgentBenchAgent.model_construct(config=_config()) + runner._sem = app.asyncio.Semaphore(1) + runner._session_results_dir = tmp_path / "results" + runner._session_results_dir.mkdir() + + async def ensure_image(_task): + return "lab:image" + + async def reject_runtime(_image): + raise app.LegalAgentBenchConfigurationError("missing configured CLI pin") + + monkeypatch.setattr(app, "resolve_task_dir", lambda runtime, instance: task) + monkeypatch.setattr(app, "resolve_repo_path", lambda path: skills) + monkeypatch.setattr(app, "compose_agent_input", lambda task_dir, skills_dir, params: params) + monkeypatch.setattr(runner, "_model_name", lambda: "policy") + monkeypatch.setattr(runner, "_ensure_image", ensure_image) + monkeypatch.setattr(runner, "_ensure_runtime", reject_runtime) + params = NeMoGymResponseCreateParamsNonStreaming(input=[]) + body = app.LegalAgentBenchRunRequest( + instance_id="legal_agent_bench::area__task", + responses_create_params=params, + ) + + response = await runner.run(None, body) + + assert response.configuration_failed is True + assert response.task_failed is False + assert response.sandbox_failed is False + assert response.mask_sample is True + assert "missing configured CLI pin" in response.failure_reason + + +@pytest.mark.asyncio +async def test_concurrent_image_requests_build_once(monkeypatch, tmp_path) -> None: + _root, task = _task_tree(tmp_path) + runner = app.LegalAgentBenchAgent.model_construct(config=_config()) + runner._image_lock = app.asyncio.Lock() + calls = [] + + async def fake_run(args, *, cwd, timeout): + calls.append(args) + if args[1:3] == ["image", "inspect"]: + inspect_count = sum(call[1:3] == ["image", "inspect"] for call in calls) + return (1 if inspect_count == 1 else 0), "", "" + return 0, "", "" + + monkeypatch.setattr(app.shutil, "which", lambda name: "/usr/bin/docker") + monkeypatch.setattr(app, "_run_process", fake_run) + + first, second = await app.asyncio.gather(runner._ensure_image(task), runner._ensure_image(task)) + + assert first == second + assert sum(call[1] == "build" for call in calls) == 1 + + +def test_sandbox_mounts_inputs_read_only_and_output_writable(monkeypatch, tmp_path) -> None: + _root, task = _task_tree(tmp_path) + skills = _skills(tmp_path) + deps = tmp_path / "deps" + deps.mkdir() + paths = {name: tmp_path / name for name in ("runtime", "agent_source", "agent", "output", "workspace")} + for path in paths.values(): + path.mkdir() + captured = {} + + def fake_sandbox(provider, spec): + captured["provider"] = provider + captured["spec"] = spec + return SimpleNamespace() + + runner = app.LegalAgentBenchAgent.model_construct(config=_config()) + monkeypatch.setattr(runner, "_provider", lambda: "docker-provider") + monkeypatch.setattr(app, "AsyncSandbox", fake_sandbox) + + runner._agent_sandbox( + image="lab:image", + task_dir=task, + skills_dir=skills, + deps_dir=deps, + paths=paths, + ) + + volumes = captured["spec"].provider_options["volumes"] + assert f"{paths['agent_source']}:/agent_source_mount:ro" in volumes + assert f"{task / 'documents'}:/workspace/vdr:ro" in volumes + assert f"{skills}:/workspace/skills:ro" in volumes + assert f"{deps}:/agent_deps_mount:ro" in volumes + assert f"{paths['output']}:/workspace/output" in volumes + assert str(app.PARENT_DIR) not in {volume.split(":", 1)[0] for volume in volumes} + assert all("resources_servers" not in volume for volume in volumes) + assert all("tests" not in volume for volume in volumes) + assert all("LAB_JUDGE" not in volume for volume in volumes) + + +def test_verifier_sandbox_mounts_only_completed_lab_run(monkeypatch, tmp_path) -> None: + _root, task = _task_tree(tmp_path) + paths = {"lab_run": tmp_path / "lab-run"} + paths["lab_run"].mkdir() + captured = {} + + def fake_sandbox(provider, spec): + captured["provider"] = provider + captured["spec"] = spec + return SimpleNamespace() + + runner = app.LegalAgentBenchAgent.model_construct(config=_config()) + monkeypatch.setattr(runner, "_provider", lambda: "docker-provider") + monkeypatch.setattr(app, "AsyncSandbox", fake_sandbox) + + runner._verifier_sandbox(image="lab:image", task_dir=task, paths=paths) + + assert captured["spec"].workdir == "/logs/agent/artifacts/lab-run/output" + assert captured["spec"].provider_options["volumes"] == [f"{paths['lab_run']}:/logs/agent/artifacts/lab-run:ro"] + + +@pytest.mark.asyncio +async def test_run_stops_agent_sandbox_before_starting_verifier_sandbox(monkeypatch, tmp_path, capsys) -> None: + _root, task = _task_tree(tmp_path) + skills = _skills(tmp_path) + deps = tmp_path / "deps" + deps.mkdir() + events = [] + + class PhaseSandbox: + def __init__(self, phase): + self.phase = phase + + async def start(self): + events.append(f"{self.phase}:start") + + async def exec(self, *args, **kwargs): + events.append(f"{self.phase}:exec") + return SimpleNamespace(return_code=0, error_type=None, stdout="", stderr="") + + async def stop(self): + events.append(f"{self.phase}:stop") + + agent_sandbox = PhaseSandbox("agent") + verifier_sandbox = PhaseSandbox("verifier") + runner = app.LegalAgentBenchAgent.model_construct(config=_config()) + runner._sem = app.asyncio.Semaphore(1) + runner._session_results_dir = tmp_path / "results" + runner._session_results_dir.mkdir() + + async def ensure_image(_task): + return "lab:image" + + async def ensure_runtime(_image): + return deps + + def write_runner(paths, params, model_url): + (paths["runtime"] / "response.json").write_text(_successful_response().model_dump_json()) + + async def run_verifier(sandbox, task_dir, paths): + assert sandbox is verifier_sandbox + assert events == ["agent:start", "agent:exec", "agent:stop", "verifier:start"] + return {"reward": 1.0, "criteria_pass_rate": 1.0}, False, None + + monkeypatch.setattr(app, "resolve_task_dir", lambda runtime, instance: task) + monkeypatch.setattr(app, "resolve_repo_path", lambda path: skills) + monkeypatch.setattr(app, "compose_agent_input", lambda task_dir, skills_dir, params: params) + monkeypatch.setattr(runner, "_ensure_image", ensure_image) + monkeypatch.setattr(runner, "_ensure_runtime", ensure_runtime) + monkeypatch.setattr(runner, "_stage_agent_source", lambda paths: None) + monkeypatch.setattr(runner, "_write_runner_config", write_runner) + monkeypatch.setattr(runner, "_model_url", lambda body: "http://model") + monkeypatch.setattr(runner, "_model_name", lambda: "policy") + monkeypatch.setattr(runner, "_agent_sandbox", lambda **kwargs: agent_sandbox) + monkeypatch.setattr(runner, "_verifier_sandbox", lambda **kwargs: verifier_sandbox) + monkeypatch.setattr(runner, "_artifacts", lambda *args, **kwargs: None) + monkeypatch.setattr(runner, "_stage_and_run_verifier", run_verifier) + body = app.LegalAgentBenchRunRequest( + instance_id="legal_agent_bench::area__task", + responses_create_params=NeMoGymResponseCreateParamsNonStreaming(input=[]), + ) + + response = await runner.run(None, body) + + assert events == ["agent:start", "agent:exec", "agent:stop", "verifier:start", "verifier:stop"] + assert response.reward == 1.0 + assert response.sandbox_failed is False + terminal_output = capsys.readouterr().out + assert "LAB rollout artifacts:" in terminal_output + assert "LAB rollout complete:" in terminal_output + + +@pytest.mark.asyncio +async def test_model_connectivity_failure_is_masked_and_skips_verifier(monkeypatch, tmp_path, capsys) -> None: + _root, task = _task_tree(tmp_path) + skills = _skills(tmp_path) + deps = tmp_path / "deps" + deps.mkdir() + events = [] + + class AgentSandbox: + async def start(self): + events.append("agent:start") + + async def exec(self, *args, **kwargs): + events.append("agent:exec") + return SimpleNamespace(return_code=1, error_type=None, stdout="", stderr="connection failed") + + async def stop(self): + events.append("agent:stop") + + runner = app.LegalAgentBenchAgent.model_construct(config=_config()) + runner._sem = app.asyncio.Semaphore(1) + runner._session_results_dir = tmp_path / "results" + runner._session_results_dir.mkdir() + + async def ensure_image(_task): + return "lab:image" + + async def ensure_runtime(_image): + return deps + + def write_runner(paths, params, model_url): + (paths["runtime"] / "runner_status.json").write_text( + json.dumps( + { + "ok": False, + "phase": "model_connectivity", + "error": "Policy model is unreachable from the LAB sandbox: http://model", + } + ) + ) + + monkeypatch.setattr(app, "resolve_task_dir", lambda runtime, instance: task) + monkeypatch.setattr(app, "resolve_repo_path", lambda path: skills) + monkeypatch.setattr(app, "compose_agent_input", lambda task_dir, skills_dir, params: params) + monkeypatch.setattr(runner, "_ensure_image", ensure_image) + monkeypatch.setattr(runner, "_ensure_runtime", ensure_runtime) + monkeypatch.setattr(runner, "_stage_agent_source", lambda paths: None) + monkeypatch.setattr(runner, "_write_runner_config", write_runner) + monkeypatch.setattr(runner, "_model_url", lambda body: "http://model") + monkeypatch.setattr(runner, "_model_name", lambda: "policy") + monkeypatch.setattr(runner, "_agent_sandbox", lambda **kwargs: AgentSandbox()) + monkeypatch.setattr( + runner, + "_verifier_sandbox", + lambda **kwargs: pytest.fail("verifier must not run after model connectivity failure"), + ) + body = app.LegalAgentBenchRunRequest( + instance_id="legal_agent_bench::area__task", + responses_create_params=NeMoGymResponseCreateParamsNonStreaming(input=[]), + ) + + response = await runner.run(None, body) + + assert events == ["agent:start", "agent:exec", "agent:stop"] + assert response.agent_failed is True + assert response.model_connection_failed is True + assert response.mask_sample is True + assert response.verifier_failed is False + assert "unreachable" in response.failure_reason + summary = json.loads(Path(response.run_summary_path).read_text()) + assert summary["flags"]["model_connection_failed"] is True + assert summary["paths"]["agent_trace"] == response.agent_trace_path + terminal_output = capsys.readouterr().out + assert "LAB rollout artifacts:" in terminal_output + assert "LAB rollout failed:" in terminal_output + + +class _FakeSandbox: + def __init__(self, reward: dict | None, *, timed_out: bool = False): + self.reward = reward + self.timed_out = timed_out + self.uploaded = [] + self.exec_calls = [] + + async def upload(self, source, destination): + self.uploaded.append(destination) + + async def exec(self, command, **kwargs): + self.exec_calls.append((command, kwargs)) + if command.startswith("test -f"): + filename = command.rsplit("/", 1)[-1] + exists = self.reward is not None and filename in {"reward.json", "scores.json"} + return type( + "Result", (), {"return_code": 0 if exists else 1, "error_type": None, "stdout": "", "stderr": ""} + )() + return type( + "Result", + (), + { + "return_code": 124 if self.timed_out else 0, + "error_type": "timeout" if self.timed_out else None, + "stdout": "", + "stderr": "", + }, + )() + + async def download(self, source, destination): + payload = self.reward if source.endswith("reward.json") else {"summary": "ok"} + Path(destination).write_text(json.dumps(payload)) + + +@pytest.mark.asyncio +async def test_verifier_is_staged_after_agent_and_receives_secrets_only_on_exec(tmp_path) -> None: + _root, task = _task_tree(tmp_path) + paths = {"verifier": tmp_path / "verifier"} + paths["verifier"].mkdir() + sandbox = _FakeSandbox({"reward": 1.0, "criteria_pass_rate": 1.0, "judge_error_count": 0}) + runner = app.LegalAgentBenchAgent.model_construct(config=_config()) + + reward, timed_out, failure = await runner._stage_and_run_verifier(sandbox, task, paths) + + assert sandbox.uploaded == ["/tmp/legal-agent-bench-tests.tar.gz"] + verifier_call = next(call for call in sandbox.exec_calls if "bash /tests/test.sh" in call[0]) + assert verifier_call[1]["env"]["LAB_JUDGE_API_KEY"] == "secret" # pragma: allowlist secret + assert reward["reward"] == 1.0 + assert timed_out is False + assert failure is None + + +@pytest.mark.asyncio +async def test_missing_verifier_reward_returns_failure(tmp_path) -> None: + _root, task = _task_tree(tmp_path) + paths = {"verifier": tmp_path / "verifier"} + paths["verifier"].mkdir() + sandbox = _FakeSandbox(None, timed_out=True) + runner = app.LegalAgentBenchAgent.model_construct(config=_config(verifier_timeout_seconds=1)) + + reward, timed_out, failure = await runner._stage_and_run_verifier(sandbox, task, paths) + + assert reward == {} + assert timed_out is True + assert "reward.json" in failure From aff95beb8d5c835b9ecb77bd6f44bcccab10d360 Mon Sep 17 00:00:00 2001 From: Ryan Stewart Date: Thu, 30 Jul 2026 11:33:15 -0400 Subject: [PATCH 02/26] pass an empty request context from the inner LAB runner instead of None to address changes introduced in main Signed-off-by: Ryan Stewart --- responses_api_agents/legal_agent_bench_agent/app.py | 3 ++- responses_api_agents/legal_agent_bench_agent/tests/test_app.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/responses_api_agents/legal_agent_bench_agent/app.py b/responses_api_agents/legal_agent_bench_agent/app.py index f2fd5621b3..e650433a04 100644 --- a/responses_api_agents/legal_agent_bench_agent/app.py +++ b/responses_api_agents/legal_agent_bench_agent/app.py @@ -75,6 +75,7 @@ import socket import sys from pathlib import Path +from types import SimpleNamespace from urllib.parse import urlsplit sys.path.insert(0, "/agent_source_mount") @@ -162,7 +163,7 @@ def empty_endpoint_model_metadata(*args, **kwargs): body = NeMoGymResponseCreateParamsNonStreaming.model_validate(runner["responses_create_params"]) response_kwargs = {"body": body} if "request" in inspect.signature(agent.responses).parameters: - response_kwargs["request"] = None + response_kwargs["request"] = SimpleNamespace(path_params={}) response = asyncio.run(agent.responses(**response_kwargs)) Path("/trajectories_mount/response.json").write_text(response.model_dump_json()) except Exception as exc: diff --git a/responses_api_agents/legal_agent_bench_agent/tests/test_app.py b/responses_api_agents/legal_agent_bench_agent/tests/test_app.py index bc5a45ee2b..ba624d7862 100644 --- a/responses_api_agents/legal_agent_bench_agent/tests/test_app.py +++ b/responses_api_agents/legal_agent_bench_agent/tests/test_app.py @@ -424,6 +424,7 @@ def test_runner_config_preserves_dynamic_agent_configuration(tmp_path) -> None: assert payload["model_connect_timeout_seconds"] == 10 assert payload["disable_endpoint_metadata_probe"] is True assert "inspect.signature(agent.responses)" in (tmp_path / "agent_runner.py").read_text() + assert "SimpleNamespace(path_params={})" in (tmp_path / "agent_runner.py").read_text() assert "socket.create_connection" in (tmp_path / "agent_runner.py").read_text() assert "runner_status.json" in (tmp_path / "agent_runner.py").read_text() assert "hermes_model_metadata.fetch_endpoint_model_metadata" in (tmp_path / "agent_runner.py").read_text() From 0658a0efe62a3fbd074e4d83e3cad944bcb8001c Mon Sep 17 00:00:00 2001 From: Ryan Stewart Date: Mon, 3 Aug 2026 14:16:53 -0400 Subject: [PATCH 03/26] propagate partial agent harness failures Signed-off-by: Ryan Stewart --- responses_api_agents/claude_code_agent/app.py | 14 +++++- .../claude_code_agent/tests/test_app.py | 24 ++++++++++ responses_api_agents/codex_agent/app.py | 41 +++++++++++++--- .../codex_agent/tests/test_app.py | 47 +++++++++++++++---- responses_api_agents/hermes_agent/app.py | 3 ++ .../hermes_agent/tests/test_app.py | 33 +++++++++++++ .../legal_agent_bench_agent/tests/test_app.py | 15 ++++++ 7 files changed, 161 insertions(+), 16 deletions(-) diff --git a/responses_api_agents/claude_code_agent/app.py b/responses_api_agents/claude_code_agent/app.py index dd85306962..3ce3de4be9 100644 --- a/responses_api_agents/claude_code_agent/app.py +++ b/responses_api_agents/claude_code_agent/app.py @@ -489,13 +489,13 @@ async def _run_claude_code( proc.kill() stdout, _ = await communication LOG.warning("claude-code timed out after %ds", self.config.timeout) - _, run_metadata = parse_stream_json(stdout.decode(errors="replace")) + output_items, run_metadata = parse_stream_json(stdout.decode(errors="replace")) run_metadata.update( status="incomplete", error_type="timeout", duration_ms=(monotonic() - process_started_at) * 1000, ) - return [], model, run_metadata + return output_items, model, run_metadata except asyncio.CancelledError: if proc.returncode is None: with suppress(ProcessLookupError): @@ -627,10 +627,20 @@ async def _create_response( input_tokens = run_metadata.get("input_tokens", 0) output_tokens = run_metadata.get("output_tokens", 0) + run_status = str(run_metadata.get("status") or "incomplete") + failure_message = None + if run_status != "completed": + failure_message = str(run_metadata.get("error_type") or run_status) return NeMoGymResponse( id=f"resp_{uuid4().hex}", created_at=int(time()), + status="failed" if failure_message else "completed", + error=( + {"code": "server_error", "message": f"Claude Code failed: {failure_message}"} + if failure_message + else None + ), model=model_name, object="response", output=output_items, diff --git a/responses_api_agents/claude_code_agent/tests/test_app.py b/responses_api_agents/claude_code_agent/tests/test_app.py index 1625d753ee..972a7f9a07 100644 --- a/responses_api_agents/claude_code_agent/tests/test_app.py +++ b/responses_api_agents/claude_code_agent/tests/test_app.py @@ -1032,3 +1032,27 @@ def test_config_yaml_parses(self) -> None: assert inner["entrypoint"] == "app.py" assert inner["concurrency"] == 32 assert inner["max_turns"] == 30 + + +def test_partial_claude_output_preserves_trajectory_and_marks_response_failed() -> None: + agent = _make_agent() + partial = NeMoGymResponseOutputMessage.model_validate( + { + "id": "msg-1", + "content": [{"type": "output_text", "text": "partial answer", "annotations": []}], + "role": "assistant", + "status": "completed", + "type": "message", + } + ) + + async def fake_run_claude_code(*args, **kwargs): + return [partial], "claude-test", {"status": "failed", "error_type": "process_exit_1"} + + object.__setattr__(agent, "_run_claude_code", fake_run_claude_code) + response = asyncio.run(agent._create_response(NeMoGymResponseCreateParamsNonStreaming(input="perform the task"))) + + assert response.output + assert response.status == "failed" + assert response.error is not None + assert "process_exit_1" in response.error.message diff --git a/responses_api_agents/codex_agent/app.py b/responses_api_agents/codex_agent/app.py index e5eaec65bd..04f8032e6b 100644 --- a/responses_api_agents/codex_agent/app.py +++ b/responses_api_agents/codex_agent/app.py @@ -463,8 +463,8 @@ async def _run_codex( mcp_servers: Optional[dict[str, Any]] = None, skills_path: Optional[str] = None, rollout_id: Optional[str] = None, - ) -> tuple[str, str]: - """Run ``codex exec --json`` and return (stdout, model_name). + ) -> tuple[str, str, dict[str, Any]]: + """Run ``codex exec --json`` and return stdout, model name, and process outcome. When ``rollout_id`` is set and a model server is configured, the per-rollout capture prefix is applied to the provider base_url so the CLI's streaming /v1/responses calls correlate to @@ -509,15 +509,29 @@ async def _run_codex( stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=self.config.timeout) except asyncio.TimeoutError: _kill_process_group(proc) - await proc.communicate() + stdout, stderr = await proc.communicate() LOG.warning("codex timed out after %ds", self.config.timeout) - return "", model + return ( + stdout.decode(errors="replace"), + model, + { + "status": "failed", + "error_type": "timeout", + "stderr": stderr.decode(errors="replace"), + }, + ) + outcome = {"status": "completed", "return_code": proc.returncode} if proc.returncode not in (0, None): LOG.warning("codex exited %d: %s", proc.returncode, stderr.decode(errors="replace")[:500]) + outcome.update( + status="failed", + error_type=f"process_exit_{proc.returncode}", + stderr=stderr.decode(errors="replace"), + ) LOG.debug("codex stdout (%d chars): %s", len(stdout), stdout[:2000].decode(errors="replace")) - return stdout.decode(errors="replace"), model + return stdout.decode(errors="replace"), model, outcome finally: if codex_home is not None: shutil.rmtree(codex_home, ignore_errors=True) @@ -572,7 +586,7 @@ async def _create_response( system_parts = [p for p in [self.config.system_prompt, input_system] if p] system_prompt = "\n\n".join(system_parts) if system_parts else None - stdout, model_name = await self._run_codex( + stdout, model_name, run_metadata = await self._run_codex( user_message, system_prompt=system_prompt, mcp_servers=mcp_servers, @@ -583,6 +597,11 @@ async def _create_response( if usage.get("errors"): LOG.warning("codex reported errors: %s", usage["errors"]) + run_metadata.update( + status="failed", + error_type="stream_error", + stderr="; ".join(str(error) for error in usage["errors"]), + ) if not any( getattr(item, "type", None) == "message" and getattr(item, "role", None) == "assistant" @@ -601,10 +620,20 @@ async def _create_response( input_tokens = usage.get("input_tokens", 0) output_tokens = usage.get("output_tokens", 0) + failure_message = None + if run_metadata.get("status") != "completed": + failure_message = str(run_metadata.get("error_type") or "codex_failed") + stderr = str(run_metadata.get("stderr") or "").strip() + if stderr: + failure_message = f"{failure_message}: {stderr[-1000:]}" return NeMoGymResponse( id=f"resp_{uuid4().hex}", created_at=int(time()), + status="failed" if failure_message else "completed", + error=( + {"code": "server_error", "message": f"Codex failed: {failure_message}"} if failure_message else None + ), model=model_name, object="response", output=output_items, diff --git a/responses_api_agents/codex_agent/tests/test_app.py b/responses_api_agents/codex_agent/tests/test_app.py index 78bdf18d99..4dd596d3d7 100644 --- a/responses_api_agents/codex_agent/tests/test_app.py +++ b/responses_api_agents/codex_agent/tests/test_app.py @@ -311,7 +311,7 @@ def _run(self, agent: CodexAgent, body: CodexAgentRunRequest, run_codex: AsyncMo def test_skills_ref_path_forwarded(self) -> None: agent = _make_agent() - run_codex = AsyncMock(return_value=("", "codex-default")) + run_codex = AsyncMock(return_value=("", "codex-default", {"status": "completed"})) body = CodexAgentRunRequest.model_validate( { "responses_create_params": {"input": []}, @@ -325,7 +325,7 @@ def test_skills_ref_path_forwarded(self) -> None: def test_no_skills_ref_forwards_none(self) -> None: agent = _make_agent() - run_codex = AsyncMock(return_value=("", "codex-default")) + run_codex = AsyncMock(return_value=("", "codex-default", {"status": "completed"})) body = CodexAgentRunRequest.model_validate({"responses_create_params": {"input": []}}) self._run(agent, body, run_codex) @@ -365,7 +365,7 @@ async def fake_exec(*cmd, **kwargs): patch("responses_api_agents.codex_agent.app.Path.home", return_value=tmp_path), patch("responses_api_agents.codex_agent.app.asyncio.create_subprocess_exec", fake_exec), ): - stdout, model = asyncio.run(agent._run_codex("hello", system_prompt="be terse")) + stdout, model, outcome = asyncio.run(agent._run_codex("hello", system_prompt="be terse")) assert captured["cmd"][0] == "codex" assert captured["cmd"][-1] == "hello" @@ -375,6 +375,7 @@ async def fake_exec(*cmd, **kwargs): assert captured["start_new_session"] is True assert captured["config_during_run"]["developer_instructions"] == "be terse" assert captured["cwd_exists_during_run"] is True + assert outcome["status"] == "completed" # per-run home and scratch cwd are removed after the run (no leakage between rollouts) assert not Path(captured["codex_home"]).exists() assert not Path(captured["scratch_cwd"]).exists() @@ -446,11 +447,12 @@ async def fake_wait_for(coro, timeout): patch("responses_api_agents.codex_agent.app.asyncio.create_subprocess_exec", fake_exec), patch("responses_api_agents.codex_agent.app.asyncio.wait_for", fake_wait_for), ): - stdout, model = asyncio.run(agent._run_codex("hello")) + stdout, model, outcome = asyncio.run(agent._run_codex("hello")) assert stdout == "" assert killed["called"] is True assert model == "codex-default" + assert outcome["error_type"] == "timeout" class TestRolloutMCPServers: @@ -518,9 +520,13 @@ async def fake_run_codex(instruction, system_prompt=None, mcp_servers=None, **kw captured["instruction"] = instruction captured["mcp_servers"] = mcp_servers captured["config"] = agent._build_config("http://x/v1", mcp_servers=mcp_servers) - return _item_completed( - {"id": "item_1", "type": "agent_message", "text": "The weather in Paris is sunny and 72 F."} - ), "codex-default" + return ( + _item_completed( + {"id": "item_1", "type": "agent_message", "text": "The weather in Paris is sunny and 72 F."} + ), + "codex-default", + {"status": "completed"}, + ) agent.server_client.post.side_effect = fake_post object.__setattr__(agent, "_run_codex", fake_run_codex) @@ -563,7 +569,11 @@ async def fake_post(server_name, url_path, json=None, cookies=None): async def fake_run_codex(instruction, system_prompt=None, mcp_servers=None, **kwargs): captured["mcp_servers"] = mcp_servers - return _item_completed({"id": "item_1", "type": "agent_message", "text": "ok"}), "codex-default" + return ( + _item_completed({"id": "item_1", "type": "agent_message", "text": "ok"}), + "codex-default", + {"status": "completed"}, + ) agent.server_client.post.side_effect = fake_post object.__setattr__(agent, "_run_codex", fake_run_codex) @@ -855,3 +865,24 @@ def test_config_yaml_parses(self) -> None: assert inner["entrypoint"] == "app.py" assert inner["concurrency"] == 32 assert inner["sandbox_mode"] == "danger-full-access" + + +def test_partial_codex_output_preserves_trajectory_and_marks_response_failed() -> None: + agent = _make_agent() + stdout = "\n".join( + [ + _item_completed({"id": "msg-1", "type": "agent_message", "text": "partial answer"}), + json.dumps({"type": "turn.failed", "error": {"message": "stream disconnected"}}), + ] + ) + + async def fake_run_codex(*args, **kwargs): + return stdout, "codex-default", {"status": "completed"} + + object.__setattr__(agent, "_run_codex", fake_run_codex) + response = asyncio.run(agent._create_response(NeMoGymResponseCreateParamsNonStreaming(input="perform the task"))) + + assert response.output + assert response.status == "failed" + assert response.error is not None + assert "stream_error" in response.error.message diff --git a/responses_api_agents/hermes_agent/app.py b/responses_api_agents/hermes_agent/app.py index b935bec52d..bf4994a309 100644 --- a/responses_api_agents/hermes_agent/app.py +++ b/responses_api_agents/hermes_agent/app.py @@ -427,9 +427,12 @@ def _patched_build_api_kwargs(api_messages): ) ) + agent_error = result.get("error") return NeMoGymResponse( id=f"resp_{uuid4().hex}", created_at=int(time()), + status="failed" if agent_error else "completed", + error=({"code": "server_error", "message": str(agent_error)} if agent_error else None), model=model_name, object="response", output=output_items, diff --git a/responses_api_agents/hermes_agent/tests/test_app.py b/responses_api_agents/hermes_agent/tests/test_app.py index 37ebb3bf18..f0fcab673e 100644 --- a/responses_api_agents/hermes_agent/tests/test_app.py +++ b/responses_api_agents/hermes_agent/tests/test_app.py @@ -364,6 +364,39 @@ def run_conversation(self, *args, **kwargs) -> dict: assert episode.observations.source == "hermes" assert episode.observations.records[0].invocation_id == "root" + def test_partial_output_preserves_trajectory_and_marks_response_failed(self, monkeypatch) -> None: + import nemo_gym.base_responses_api_agent as base_agent + from nemo_gym.openai_utils import NeMoGymResponseCreateParamsNonStreaming + + monkeypatch.setattr(base_agent, "get_first_server_config_dict", lambda _gc, _name: {"host": "h", "port": 1}) + server_client = MagicMock(spec=ServerClient) + server_client.global_config_dict = {} + server_client._build_server_base_url = lambda _cfg: "http://h:1" + agent = HermesAgent(config=_config(), server_client=server_client) + monkeypatch.setattr(agent, "_ensure_sigterm_handler", lambda: None) + + class _StubAIAgent: + def __init__(self, **kwargs) -> None: + self._build_api_kwargs = lambda _messages: {} + self.compression_enabled = True + + def run_conversation(self, *args, **kwargs) -> dict: + return { + "messages": [{"role": "assistant", "content": "partial answer"}], + "error": "model stream disconnected", + "prompt_tokens": 12, + "output_tokens": 3, + "total_tokens": 15, + } + + monkeypatch.setattr("run_agent.AIAgent", _StubAIAgent) + response = asyncio.run(agent.responses(request=None, body=NeMoGymResponseCreateParamsNonStreaming(input="hi"))) + + assert response.output + assert response.status == "failed" + assert response.error is not None + assert "stream disconnected" in response.error.message + class TestObservability: @pytest.mark.parametrize( diff --git a/responses_api_agents/legal_agent_bench_agent/tests/test_app.py b/responses_api_agents/legal_agent_bench_agent/tests/test_app.py index ba624d7862..e444438e85 100644 --- a/responses_api_agents/legal_agent_bench_agent/tests/test_app.py +++ b/responses_api_agents/legal_agent_bench_agent/tests/test_app.py @@ -560,6 +560,21 @@ def test_real_agent_activity_is_not_treated_as_an_infrastructure_failure() -> No ) +def test_partial_response_with_harness_error_is_an_agent_failure() -> None: + response = app.NeMoGymResponse.model_validate( + { + **_successful_response().model_dump(mode="json"), + "status": "failed", + "error": {"code": "server_error", "message": "adapter failed after partial output"}, + } + ) + + failure = app.agent_response_failure(response, "responses_api_agents.codex_agent.app") + + assert failure is not None + assert "adapter failed after partial output" in failure + + def test_response_masks_harness_and_verifier_failures() -> None: runner = app.LegalAgentBenchAgent.model_construct(config=_config()) params = NeMoGymResponseCreateParamsNonStreaming(input=[]) From 14ad16d7f19c3cb045d2dc59e09d54bb52b45a7c Mon Sep 17 00:00:00 2001 From: Ryan Stewart Date: Mon, 3 Aug 2026 14:18:41 -0400 Subject: [PATCH 04/26] isolate LAB runtime dependency provisioning Signed-off-by: Ryan Stewart --- .../legal_agent_bench_agent/README.md | 22 ++-- .../legal_agent_bench_agent/app.py | 108 +++++++++++++----- .../legal_agent_bench_agent/tests/test_app.py | 15 ++- 3 files changed, 102 insertions(+), 43 deletions(-) diff --git a/responses_api_agents/legal_agent_bench_agent/README.md b/responses_api_agents/legal_agent_bench_agent/README.md index 3aaefac3fe..77ee3eb6f9 100644 --- a/responses_api_agents/legal_agent_bench_agent/README.md +++ b/responses_api_agents/legal_agent_bench_agent/README.md @@ -22,15 +22,17 @@ compatibility config is reserved for Harbor. ## Execution boundary For each rollout, the runner validates `instance_id`, builds or reuses the -content-addressed LAB image, provisions a cached portable Linux runtime for the -selected harness inside that image, and starts an agent-only Docker sandbox. -Provisioning selects ARM64 or x86-64 dependencies from the container -architecture, so the host does not need to execute container binaries. The -runtime cache is invalidated by changes to its setup script, requirements, -configured CLI pin, LAB image, Gym packaging metadata, or installed `nemo_gym` -source. Only the selected Gym agent package, source documents, skills, and -portable dependencies are mounted read-only; the repository and LAB task cache -are not mounted. Output and scratch directories are writable. +content-addressed LAB image, provisions a portable Linux runtime for the +selected harness, and starts an agent-only Docker sandbox. Runtimes are +immutable and content-addressed by their setup script, requirements, configured +CLI pin, LAB image identity and architecture, Gym packaging metadata, and +installed `nemo_gym` source. An interprocess lock and atomic publication +prevent concurrent evaluations from rewriting a runtime that is being built or +used. + +Dependency provisioning exposes an explicit allowlist containing only the +installer and required Gym package files. It never mounts the repository, +`env.yaml`, task tests, judge credentials, or unrelated files. Only the task instructions and public skill manuals are supplied to the configured agent. After it exits, its container is destroyed before a fresh @@ -82,4 +84,4 @@ that inherits it. CLI versions belong in the harness configuration's `agent_kwargs`; provisioning derives its exact package specification from that pin and rejects unpinned Claude Code or Codex configurations. -The runner is compatible with Docker only. \ No newline at end of file +The runner is compatible with Docker only. diff --git a/responses_api_agents/legal_agent_bench_agent/app.py b/responses_api_agents/legal_agent_bench_agent/app.py index e650433a04..c325b3249c 100644 --- a/responses_api_agents/legal_agent_bench_agent/app.py +++ b/responses_api_agents/legal_agent_bench_agent/app.py @@ -5,6 +5,7 @@ from __future__ import annotations import asyncio +import fcntl import hashlib import json import os @@ -317,6 +318,7 @@ def ensure_agent_runtime( image: str, docker_network: Optional[str], timeout_seconds: int, + docker_platform: Optional[str] = None, ) -> Path: key = agent_key(agent_server_module) script = resolve_agent_setup_script(agent_server_module) @@ -325,6 +327,12 @@ def ensure_agent_runtime( docker = shutil.which("docker") if not docker: raise FileNotFoundError("Docker CLI is required to provision Legal Agent Bench agent dependencies") + if docker_platform: + subprocess.run( + [docker, "pull", "--platform", docker_platform, image], + check=True, + timeout=timeout_seconds, + ) image_info = subprocess.run( [docker, "image", "inspect", "--format", "{{.Id}}:{{.Os}}/{{.Architecture}}", image], check=True, @@ -338,45 +346,83 @@ def ensure_agent_runtime( script, requirements, PARENT_DIR / "pyproject.toml", + PARENT_DIR / "README.md", PARENT_DIR / "nemo_gym", ], values=[image_info, json.dumps(runtime_env, sort_keys=True)], ) - deps_dir = PACKAGE_DIR / ".deps" / key + cache_root = PACKAGE_DIR / ".deps" / key + deps_dir = cache_root / recipe sentinel = deps_dir / ".installed" if sentinel.is_file() and sentinel.read_text().strip() == recipe: return deps_dir - if deps_dir.exists(): - shutil.rmtree(deps_dir) - deps_dir.mkdir(parents=True) - env = { - "PORTABLE_PYTHON_SH": ( - "/nemo_gym_mount/responses_api_agents/legal_agent_bench_agent/setup_scripts/_portable_python.sh" - ), - "DEPS_DIR": "/agent_deps", - "NEMO_GYM_ROOT": "/nemo_gym_mount", - "HOME": "/tmp", - **runtime_env, - } - command = [ - docker, - "run", - "--rm", - "--user", - f"{os.getuid()}:{os.getgid()}", - "--volume", - f"{PARENT_DIR}:/nemo_gym_mount:ro", - "--volume", - f"{deps_dir}:/agent_deps", - ] - if docker_network: - command.extend(["--network", docker_network]) - for name, value in env.items(): - command.extend(["--env", f"{name}={value}"]) - command.extend([image, "bash", f"/nemo_gym_mount/{script.relative_to(PARENT_DIR).as_posix()}"]) - subprocess.run(command, check=True, timeout=timeout_seconds) - sentinel.write_text(recipe) + cache_root.mkdir(parents=True, exist_ok=True) + lock_root = PACKAGE_DIR / ".deps" / ".locks" + lock_root.mkdir(parents=True, exist_ok=True) + lock_path = lock_root / f"{key}-{recipe}.lock" + with lock_path.open("a+") as lock_file: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX) + try: + if sentinel.is_file() and sentinel.read_text().strip() == recipe: + return deps_dir + + with tempfile.TemporaryDirectory(prefix=f".{recipe}-bundle-", dir=cache_root) as bundle_raw: + bundle = Path(bundle_raw) + shutil.copy2(PARENT_DIR / "pyproject.toml", bundle / "pyproject.toml") + shutil.copy2(PARENT_DIR / "README.md", bundle / "README.md") + shutil.copytree(PARENT_DIR / "nemo_gym", bundle / "nemo_gym") + staged_agent = bundle / "responses_api_agents" / key + (staged_agent / "scripts").mkdir(parents=True) + shutil.copy2(requirements, staged_agent / "requirements.txt") + shutil.copy2(script, staged_agent / "scripts" / script.name) + staged_setup = bundle / "responses_api_agents" / "legal_agent_bench_agent" / "setup_scripts" + staged_setup.mkdir(parents=True) + shutil.copy2(PORTABLE_PYTHON_SH, staged_setup / PORTABLE_PYTHON_SH.name) + + build_dir = Path(tempfile.mkdtemp(prefix=f".{recipe}-build-", dir=cache_root)) + try: + env = { + "PORTABLE_PYTHON_SH": ( + "/nemo_gym_mount/responses_api_agents/legal_agent_bench_agent/" + "setup_scripts/_portable_python.sh" + ), + "DEPS_DIR": "/agent_deps", + "NEMO_GYM_ROOT": "/nemo_gym_mount", + "HOME": "/tmp", + **runtime_env, + } + command = [ + docker, + "run", + "--rm", + "--user", + f"{os.getuid()}:{os.getgid()}", + "--volume", + f"{bundle}:/nemo_gym_mount:ro", + "--volume", + f"{build_dir}:/agent_deps", + ] + if docker_platform: + command.extend(["--platform", docker_platform]) + if docker_network: + command.extend(["--network", docker_network]) + for name, value in env.items(): + command.extend(["--env", f"{name}={value}"]) + command.extend( + [image, "bash", f"/nemo_gym_mount/responses_api_agents/{key}/scripts/{script.name}"] + ) + subprocess.run(command, check=True, timeout=timeout_seconds) + (build_dir / ".installed").write_text(recipe) + if deps_dir.exists(): + shutil.rmtree(build_dir) + else: + build_dir.replace(deps_dir) + finally: + if build_dir.exists(): + shutil.rmtree(build_dir) + finally: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN) return deps_dir diff --git a/responses_api_agents/legal_agent_bench_agent/tests/test_app.py b/responses_api_agents/legal_agent_bench_agent/tests/test_app.py index e444438e85..7100d7026b 100644 --- a/responses_api_agents/legal_agent_bench_agent/tests/test_app.py +++ b/responses_api_agents/legal_agent_bench_agent/tests/test_app.py @@ -135,7 +135,12 @@ def test_dependency_runtime_cache_is_harness_and_recipe_specific(monkeypatch, tm portable.write_text("portable-v1") script = tmp_path / "hermes_agent_deps.sh" script.write_text("hermes-v1") + agent_dir = tmp_path / "responses_api_agents" / "hermes_agent" + agent_dir.mkdir(parents=True) + (agent_dir / "requirements.txt").write_text("nemo-gym\n") (tmp_path / "pyproject.toml").write_text("[project]\nname = 'nemo-gym'\n") + (tmp_path / "README.md").write_text("# Gym\n") + (tmp_path / "env.yaml").write_text("judge_key: must-not-be-mounted\n") (tmp_path / "nemo_gym").mkdir() (tmp_path / "nemo_gym" / "runtime.py").write_text("VERSION = 1\n") calls = [] @@ -191,10 +196,15 @@ def fake_run(args, **kwargs): timeout_seconds=60, ) - assert first == second == third == fourth == fifth == package_dir / ".deps" / "hermes_agent" + assert first == second + assert len({first, third, fourth, fifth}) == 4 + assert all(path.parent == package_dir / ".deps" / "hermes_agent" for path in (first, third, fourth, fifth)) + assert all(path.is_dir() for path in (first, third, fourth, fifth)) docker_runs = [call for call in calls if call[0][1] == "run"] assert len(docker_runs) == 4 - assert f"{first}:/agent_deps" in docker_runs[0][0] + mounted_sources = [arg.split(":", 1)[0] for arg in docker_runs[0][0] if ":/nemo_gym_mount:ro" in arg] + assert mounted_sources and mounted_sources[0] != str(tmp_path) + assert "env.yaml" not in " ".join(docker_runs[0][0]) assert "--network" in docker_runs[0][0] @@ -241,6 +251,7 @@ def test_dependency_provisioning_uses_pin_and_invalidates_when_it_changes(monkey agent_dir.mkdir(parents=True) (agent_dir / "requirements.txt").write_text("nemo-gym\n") (tmp_path / "pyproject.toml").write_text("[project]\nname = 'nemo-gym'\n") + (tmp_path / "README.md").write_text("# Gym\n") (tmp_path / "nemo_gym").mkdir() (tmp_path / "nemo_gym" / "runtime.py").write_text("VERSION = 1\n") calls = [] From 82060c3096483dcc2f044977696715158290f529 Mon Sep 17 00:00:00 2001 From: Ryan Stewart Date: Mon, 3 Aug 2026 14:21:06 -0400 Subject: [PATCH 05/26] remove writable host mounts from LAB sandboxes Signed-off-by: Ryan Stewart --- benchmarks/legal_agent_bench/README.md | 5 +- .../legal_agent_bench_agent/README.md | 55 +-- .../legal_agent_bench_agent/app.py | 383 +++++++++++++++--- .../legal_agent_bench_agent/tests/test_app.py | 239 +++++++++-- 4 files changed, 572 insertions(+), 110 deletions(-) diff --git a/benchmarks/legal_agent_bench/README.md b/benchmarks/legal_agent_bench/README.md index ae1907ee51..a34c60e6dd 100644 --- a/benchmarks/legal_agent_bench/README.md +++ b/benchmarks/legal_agent_bench/README.md @@ -261,8 +261,9 @@ Each configurable output row includes `artifact_dir`, `run_summary_path`, `verifier_report_path`, and `output_dir`. The inspection commands above use these paths. -The agent-server log also prints the artifact directory when the rollout is -created and again when it completes or fails. +The agent-server log prints the artifact directory after the agent sandbox has +stopped and its downloaded files pass validation, then again when verification +completes or the rollout fails. ## Test diff --git a/responses_api_agents/legal_agent_bench_agent/README.md b/responses_api_agents/legal_agent_bench_agent/README.md index 77ee3eb6f9..00e8dff792 100644 --- a/responses_api_agents/legal_agent_bench_agent/README.md +++ b/responses_api_agents/legal_agent_bench_agent/README.md @@ -1,8 +1,8 @@ # Legal Agent Bench Configurable Agent -This task-driven Gym agent runs a configured Gym Responses API agent inside the -Legal Agent Bench (LAB) Docker environment. It keeps the benchmark task set, -skills, and verifier fixed while allowing the agent harness to vary. +This task-driven Gym agent runs a configured Gym Responses API agent inside a +Legal Agent Bench (LAB) sandbox. It keeps the benchmark task set, skills, and +verifier fixed while allowing the agent harness and sandbox provider to vary. The committed configurations select Gym's Hermes, Claude Code, and Codex agents through: @@ -13,8 +13,9 @@ through: - `agent_kwargs` The runner also accepts the policy `model_server`, LAB runtime task and skill -paths, concurrency and timeout settings, Docker network and model URL -overrides, and a result directory. See `configs/` for complete examples. +paths, concurrency and timeout settings, `sandbox_provider`, an optional +`sandbox_image`, Docker platform, network and model URL overrides, and a result +directory. See `configs/` for complete examples. Those configurations include the LAB `resources_only.yaml` definition so Gym starts one resource server and only the selected agent; the combined compatibility config is reserved for Harbor. @@ -23,28 +24,35 @@ compatibility config is reserved for Harbor. For each rollout, the runner validates `instance_id`, builds or reuses the content-addressed LAB image, provisions a portable Linux runtime for the -selected harness, and starts an agent-only Docker sandbox. Runtimes are -immutable and content-addressed by their setup script, requirements, configured -CLI pin, LAB image identity and architecture, Gym packaging metadata, and -installed `nemo_gym` source. An interprocess lock and atomic publication -prevent concurrent evaluations from rewriting a runtime that is being built or -used. +selected harness, and starts an agent-only sandbox. Runtimes are immutable and +content-addressed by their setup script, requirements, configured CLI pin, LAB +image identity and architecture, Gym packaging metadata, and installed +`nemo_gym` source. An interprocess lock and atomic publication prevent +concurrent evaluations from rewriting a runtime that is being built or used. Dependency provisioning exposes an explicit allowlist containing only the installer and required Gym package files. It never mounts the repository, `env.yaml`, task tests, judge credentials, or unrelated files. +Agent source, documents, skills, runtime, and runner inputs are transferred +through Gym's sandbox file API. The sandbox has no writable host mounts. + Only the task instructions and public skill manuals are supplied to the -configured agent. After it exits, its container is destroyed before a fresh -verifier-only sandbox starts. That sandbox receives the completed `lab-run` -directory read-only, then receives the rubric tests and judge credentials. -The verifier sees deliverables at `lab-run/output`, preserving LAB's contract -without sharing a process namespace with the agent. - -The runner translates derived host-loopback model URLs to -`host.docker.internal` on Docker Desktop and registers the equivalent host -gateway for Linux bridge networking. An explicit `sandbox_model_base_url` is -used unchanged. Before importing the selected harness, the in-container runner +configured agent. Its results are downloaded to a private temporary directory, +the sandbox is destroyed, and links, devices, traversal paths, and other unsafe +archive entries are rejected before Gym creates host artifacts. A fresh +verifier-only sandbox then receives a separately staged, sanitized `lab-run` +tree, rubric tests, and verifier-only judge credentials. The verifier sees +deliverables at `lab-run/output`, preserving LAB's contract without sharing a +process namespace or writable filesystem with the agent. + +The default provider is Docker. The runner translates derived host-loopback +model URLs to `host.docker.internal` on Docker Desktop and registers the +equivalent host gateway for Linux bridge networking. A named Gym provider can +be selected through `sandbox_provider`; non-Docker providers require an +immutable registry `sandbox_image` and an explicit reachable +`sandbox_model_base_url`. Before importing the selected harness, the +in-container runner performs a bounded TCP connectivity check and writes `runtime/runner_status.json`. Connectivity or harness failures set the `mask_sample` flag on the row, skip the judge, and are distinct from an ordinary zero-reward @@ -65,7 +73,7 @@ for the summary, trajectory, stdout, stderr, output directory, and verifier report. For setup, four copy-paste smoke commands, and result inspection, see the -[benchmark README](../../benchmarks/legal_agent_bench/README.md#smoke-test-every-harness). +[benchmark README](../../benchmarks/legal_agent_bench/README.md#test-the-various-harnesses). Run `gym` from an activated repository environment rather than prefixing server-starting commands with `uv run`. @@ -84,4 +92,5 @@ that inherits it. CLI versions belong in the harness configuration's `agent_kwargs`; provisioning derives its exact package specification from that pin and rejects unpinned Claude Code or Codex configurations. -The runner is compatible with Docker only. +Docker remains the zero-configuration local backend. Non-Docker providers +require an immutable registry image and a model URL reachable from the sandbox. diff --git a/responses_api_agents/legal_agent_bench_agent/app.py b/responses_api_agents/legal_agent_bench_agent/app.py index c325b3249c..f0102a91f6 100644 --- a/responses_api_agents/legal_agent_bench_agent/app.py +++ b/responses_api_agents/legal_agent_bench_agent/app.py @@ -10,6 +10,7 @@ import json import os import shutil +import stat import subprocess import sys import tarfile @@ -30,8 +31,13 @@ from nemo_gym.config_types import ModelServerRef, ResourcesServerRef from nemo_gym.global_config import get_first_server_config_dict from nemo_gym.openai_utils import NeMoGymResponse, NeMoGymResponseCreateParamsNonStreaming -from nemo_gym.sandbox import AsyncSandbox, SandboxResources, SandboxSpec -from nemo_gym.sandbox.providers.docker import DockerProvider +from nemo_gym.sandbox import ( + AsyncSandbox, + SandboxResources, + SandboxSpec, + resolve_provider_config, + resolve_provider_metadata, +) from nemo_gym.server_utils import apply_rollout_prefix from resources_servers.legal_agent_bench.prepare import ( DEFAULT_RUNTIME_TASKS_DIR, @@ -192,6 +198,9 @@ class LegalAgentBenchAgentConfig(BaseResponsesAPIAgentConfig): runtime_build_timeout_seconds: int = Field(default=3600, ge=1) image_build_timeout_seconds: int = Field(default=3600, ge=1) sandbox_ttl_seconds: int = Field(default=14400, ge=1) + sandbox_provider: str | dict[str, Any] = Field(default_factory=lambda: {"docker": {}}) + sandbox_image: Optional[str] = None + runtime_docker_platform: Optional[str] = None docker_network: Optional[str] = "host" sandbox_model_base_url: Optional[str] = None image_repository: str = "nemo-gym-legal-agent-bench" @@ -236,6 +245,88 @@ class LegalAgentBenchConfigurationError(ValueError): """The selected Gym agent or its LAB configuration is invalid.""" +class LegalAgentBenchArtifactError(RuntimeError): + """The sandbox returned an unsafe or malformed artifact tree.""" + + +def _provider_name(provider: Mapping[str, Any]) -> str: + names = list(provider) + if len(names) != 1: + raise LegalAgentBenchConfigurationError( + f"sandbox_provider must resolve to exactly one provider, got {names!r}" + ) + return str(names[0]) + + +def _create_archive(destination: Path, entries: list[tuple[Path, str]]) -> None: + """Archive trusted host inputs under explicit sandbox-relative names.""" + with tarfile.open(destination, "w:gz", dereference=False) as archive: + for source, arcname in entries: + if not source.exists(): + raise FileNotFoundError(source) + archive.add(source, arcname=arcname, recursive=True) + + +def _validate_archive_member(member: tarfile.TarInfo) -> None: + if member.name in {".", "./"} and member.isdir(): + return + path = PurePosixPath(member.name) + if path.is_absolute() or not path.parts or any(part in {"", ".", ".."} for part in path.parts): + raise LegalAgentBenchArtifactError(f"Unsafe sandbox artifact path: {member.name!r}") + if member.issym() or member.islnk(): + raise LegalAgentBenchArtifactError(f"Sandbox artifacts may not contain links: {member.name!r}") + if not (member.isdir() or member.isreg()): + raise LegalAgentBenchArtifactError( + f"Sandbox artifacts may contain only directories and regular files: {member.name!r}" + ) + + +def _extract_untrusted_archive(archive_path: Path, destination: Path) -> None: + """Materialize sandbox output without following links or restoring ownership.""" + destination.mkdir(parents=True, exist_ok=True) + with tarfile.open(archive_path, "r:gz") as archive: + members = archive.getmembers() + for member in members: + _validate_archive_member(member) + archive.extractall(destination, members=members, filter="data") + + +def _copy_downloaded_file(source: Path, destination: Path) -> None: + mode = source.lstat().st_mode + if not stat.S_ISREG(mode): + raise LegalAgentBenchArtifactError(f"Expected a regular downloaded file, got {source}") + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, destination) + + +def ensure_runtime_archive(deps_dir: Path) -> Path: + """Create one immutable, process-safe transport archive for a cached runtime.""" + archive_path = deps_dir.parent / f"{deps_dir.name}.tar.gz" + if archive_path.is_file(): + return archive_path + lock_path = deps_dir.parent / f".{deps_dir.name}.archive.lock" + with lock_path.open("a+") as lock_file: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX) + try: + if archive_path.is_file(): + return archive_path + with tempfile.NamedTemporaryFile( + suffix=".tar.gz", + prefix=f".{deps_dir.name}-", + dir=deps_dir.parent, + delete=False, + ) as temporary: + temporary_path = Path(temporary.name) + try: + _create_archive(temporary_path, [(deps_dir, "agent_deps_mount")]) + temporary_path.replace(archive_path) + finally: + temporary_path.unlink(missing_ok=True) + finally: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN) + return archive_path + + def agent_key(agent_server_module: str) -> str: parts = agent_server_module.split(".") if len(parts) < 2 or parts[-1] != "app": @@ -652,11 +743,22 @@ async def _ensure_runtime(self, image: str) -> Path: agent_kwargs=self.config.agent_kwargs, image=image, docker_network=self.config.docker_network, + docker_platform=self.config.runtime_docker_platform, timeout_seconds=self.config.runtime_build_timeout_seconds, ) return self._deps_dir async def _ensure_image(self, task_dir: Path) -> str: + if self.config.sandbox_image: + if _provider_name(self._provider_config()) != "docker" and "@sha256:" not in self.config.sandbox_image: + raise LegalAgentBenchConfigurationError( + "Non-Docker LAB sandbox_image must use an immutable @sha256 digest" + ) + return self.config.sandbox_image + if _provider_name(self._provider_config()) != "docker": + raise LegalAgentBenchConfigurationError( + "Non-Docker LAB sandboxes require sandbox_image to reference an immutable registry image" + ) environment_dir = task_dir / "environment" image = f"{self.config.image_repository}:{_environment_hash(environment_dir)[:16]}" async with self._image_lock: @@ -682,6 +784,10 @@ async def _ensure_image(self, task_dir: Path) -> str: def _model_url(self, body: LegalAgentBenchRunRequest) -> str: if self.config.sandbox_model_base_url: return self.config.sandbox_model_base_url + if _provider_name(self._provider_config()) != "docker": + raise LegalAgentBenchConfigurationError( + "Non-Docker LAB sandboxes require sandbox_model_base_url reachable from the provider" + ) try: model_config = get_first_server_config_dict( self.server_client.global_config_dict, @@ -701,9 +807,8 @@ def _model_name(self) -> str: configured_name = global_config.get("policy_model_name") if isinstance(global_config, Mapping) else None return str(configured_name or self.config.model_server.name) - def _run_dirs(self, task_name: str) -> dict[str, Path]: - task_segment = _results_segment(task_name, fallback="unknown_task") - root = self._session_results_dir / f"{task_segment}_{uuid4().hex[:8]}" + @staticmethod + def _paths_for_root(root: Path, *, create: bool = False) -> dict[str, Path]: paths = { "root": root, "runtime": root / "runtime", @@ -714,11 +819,27 @@ def _run_dirs(self, task_name: str) -> dict[str, Path]: "workspace": root / "workspace", "verifier": root / "verifier", } - for path in paths.values(): - path.mkdir(parents=True, exist_ok=True) + if create: + for path in paths.values(): + path.mkdir(parents=True, exist_ok=True) + return paths + + def _run_root(self, task_name: str) -> Path: + task_segment = _results_segment(task_name, fallback="unknown_task") + return self._session_results_dir / f"{task_segment}_{uuid4().hex[:8]}" + + def _run_dirs(self, task_name: str) -> dict[str, Path]: + root = self._run_root(task_name) + paths = self._paths_for_root(root, create=True) print(f"LAB rollout artifacts: {root}", flush=True) return paths + def _publish_staged_run(self, staged: dict[str, Path], final_root: Path) -> dict[str, Path]: + final_root.parent.mkdir(parents=True, exist_ok=True) + shutil.copytree(staged["root"], final_root) + print(f"LAB rollout artifacts: {final_root}", flush=True) + return self._paths_for_root(final_root) + def _stage_agent_source(self, paths: dict[str, Path]) -> None: key = agent_key(self.config.agent_server_module) source = PARENT_DIR / "responses_api_agents" / key @@ -762,14 +883,31 @@ def _write_runner_config( } (paths["runtime"] / "runner.json").write_text(json.dumps(runner, indent=2)) - def _provider(self) -> DockerProvider: - create: dict[str, Any] = {"network": self.config.docker_network, "pids_limit": 4096} + def _provider_config(self) -> dict[str, Any]: + global_config = getattr(getattr(self, "server_client", None), "global_config_dict", None) + provider = resolve_provider_config(self.config.sandbox_provider, global_config) + if _provider_name(provider) != "docker": + return provider + + docker = dict(provider.get("docker") or {}) + create = dict(docker.get("create") or {}) + create.setdefault("network", self.config.docker_network) + create.setdefault("pids_limit", 4096) if sys.platform == "linux" and self.config.docker_network != "host": - create["extra_run_args"] = ["--add-host", "host.docker.internal:host-gateway"] - return DockerProvider( - create=create, - exec={"default_timeout_s": self.config.agent_timeout_seconds, "concurrency": 8}, - ) + extra_run_args = list(create.get("extra_run_args") or []) + host_gateway = ["--add-host", "host.docker.internal:host-gateway"] + if host_gateway[0] not in extra_run_args: + extra_run_args.extend(host_gateway) + create["extra_run_args"] = extra_run_args + execution = dict(docker.get("exec") or {}) + execution.setdefault("default_timeout_s", self.config.agent_timeout_seconds) + execution.setdefault("concurrency", 8) + docker.update({"create": create, "exec": execution}) + return {"docker": docker} + + def _sandbox_metadata(self) -> dict[str, Any]: + global_config = getattr(getattr(self, "server_client", None), "global_config_dict", None) + return resolve_provider_metadata(self.config.sandbox_provider, global_config) def _agent_sandbox( self, @@ -780,24 +918,14 @@ def _agent_sandbox( deps_dir: Path, paths: dict[str, Path], ) -> AsyncSandbox: - volumes = [ - f"{paths['agent_source']}:/agent_source_mount:ro", - f"{deps_dir}:/agent_deps_mount:ro", - f"{task_dir / 'documents'}:/workspace/vdr:ro", - f"{skills_dir}:/workspace/skills:ro", - f"{paths['runtime']}:/trajectories_mount", - f"{paths['agent']}:/logs/agent", - f"{paths['output']}:/workspace/output", - f"{paths['workspace']}:/workspace/workspace", - ] return AsyncSandbox( - self._provider(), + self._provider_config(), SandboxSpec( image=image, ttl_s=self.config.sandbox_ttl_seconds, workdir="/workspace/output", resources=_sandbox_resources(task_dir), - provider_options={"volumes": volumes}, + metadata=self._sandbox_metadata(), ), ) @@ -809,39 +937,119 @@ def _verifier_sandbox( paths: dict[str, Path], ) -> AsyncSandbox: return AsyncSandbox( - self._provider(), + self._provider_config(), SandboxSpec( image=image, ttl_s=self.config.sandbox_ttl_seconds, workdir="/logs/agent/artifacts/lab-run/output", resources=_sandbox_resources(task_dir), - provider_options={ - "volumes": [ - f"{paths['lab_run']}:/logs/agent/artifacts/lab-run:ro", - ] - }, + metadata=self._sandbox_metadata(), ), ) + async def _stage_agent_sandbox( + self, + sandbox: AsyncSandbox, + *, + task_dir: Path, + skills_dir: Path, + deps_dir: Path, + paths: dict[str, Path], + ) -> None: + with tempfile.NamedTemporaryFile(suffix=".tar.gz", delete=False) as temporary: + archive_path = Path(temporary.name) + try: + _create_archive( + archive_path, + [ + (paths["agent_source"], "agent_source_mount"), + (task_dir / "documents", "workspace/vdr"), + (skills_dir, "workspace/skills"), + (paths["runtime"], "trajectories_mount"), + ], + ) + await sandbox.upload(archive_path, "/tmp/legal-agent-bench-agent-input.tar.gz") + await sandbox.upload(ensure_runtime_archive(deps_dir), "/tmp/legal-agent-bench-runtime.tar.gz") + finally: + archive_path.unlink(missing_ok=True) + + command = ( + "mkdir -p /agent_source_mount /agent_deps_mount /trajectories_mount " + "/workspace/vdr /workspace/skills /workspace/output /workspace/workspace && " + "tar -xzf /tmp/legal-agent-bench-agent-input.tar.gz -C / && " + "tar -xzf /tmp/legal-agent-bench-runtime.tar.gz -C / && " + "chmod -R a+rX,a-w /agent_source_mount /agent_deps_mount /workspace/vdr /workspace/skills && " + "chown -R nobody:nogroup /trajectories_mount /workspace/output /workspace/workspace" + ) + result = await sandbox.exec(command, timeout_s=300, user="root") + if result.return_code != 0: + raise RuntimeError(result.stderr or result.stdout or "Failed to stage LAB agent sandbox") + + async def _collect_agent_sandbox(self, sandbox: AsyncSandbox, download_dir: Path) -> dict[str, Path]: + downloads: dict[str, Path] = {} + for filename in ("response.json", "runner_status.json"): + destination = download_dir / filename + try: + await sandbox.download(f"/trajectories_mount/{filename}", destination) + except Exception: + continue + downloads[filename] = destination + + archive_path = download_dir / "output.tar.gz" + archive_result = await sandbox.exec( + "tar -czf /tmp/legal-agent-bench-output.tar.gz -C /workspace/output .", + timeout_s=300, + user="root", + ) + if archive_result.return_code != 0: + raise RuntimeError(archive_result.stderr or archive_result.stdout or "Failed to collect LAB output") + await sandbox.download("/tmp/legal-agent-bench-output.tar.gz", archive_path) + downloads["output.tar.gz"] = archive_path + return downloads + + @staticmethod + def _materialize_agent_downloads( + paths: dict[str, Path], + downloads: dict[str, Path], + *, + stdout: str, + stderr: str, + ) -> None: + (paths["agent"] / "stdout.log").write_text(stdout) + (paths["agent"] / "stderr.log").write_text(stderr) + for filename in ("response.json", "runner_status.json"): + source = downloads.get(filename) + if source is not None: + _copy_downloaded_file(source, paths["runtime"] / filename) + output_archive = downloads.get("output.tar.gz") + if output_archive is None: + raise LegalAgentBenchArtifactError("Agent sandbox did not return an output archive") + _extract_untrusted_archive(output_archive, paths["output"]) + async def _stage_and_run_verifier( self, sandbox: AsyncSandbox, task_dir: Path, paths: dict[str, Path], - ) -> tuple[dict[str, Any], bool, Optional[str]]: + ) -> tuple[dict[str, bytes], bool, Optional[str]]: with tempfile.NamedTemporaryFile(suffix=".tar.gz", delete=False) as temporary: archive_path = Path(temporary.name) try: - with tarfile.open(archive_path, "w:gz") as archive: - for child in sorted((task_dir / "tests").iterdir(), key=lambda item: item.name): - archive.add(child, arcname=child.name) - await sandbox.upload(archive_path, "/tmp/legal-agent-bench-tests.tar.gz") + _create_archive( + archive_path, + [ + (paths["lab_run"], "logs/agent/artifacts/lab-run"), + (task_dir / "tests", "tests"), + ], + ) + await sandbox.upload(archive_path, "/tmp/legal-agent-bench-verifier-input.tar.gz") finally: archive_path.unlink(missing_ok=True) command = ( - "rm -rf /tests /logs/verifier && mkdir -p /tests /logs/verifier && " - "tar -xzf /tmp/legal-agent-bench-tests.tar.gz -C /tests && " + "rm -rf /tests /logs/agent /logs/verifier && mkdir -p /logs/verifier && " + "tar -xzf /tmp/legal-agent-bench-verifier-input.tar.gz -C / && " + "chmod -R a-w /logs/agent && " "bash /tests/test.sh" ) result = await sandbox.exec( @@ -852,17 +1060,36 @@ async def _stage_and_run_verifier( user="root", ) - for filename in ("reward.json", "scores.json", "transcript.jsonl", "report.html", "error.json"): - exists = await sandbox.exec(f"test -f /logs/verifier/{filename}", timeout_s=30, user="root") - if exists.return_code == 0: - await sandbox.download(f"/logs/verifier/{filename}", paths["verifier"] / filename) + downloaded: dict[str, bytes] = {} + with tempfile.TemporaryDirectory(prefix="legal-agent-bench-verifier-") as temporary_dir: + temporary_path = Path(temporary_dir) + for filename in ("reward.json", "scores.json", "transcript.jsonl", "report.html", "error.json"): + destination = temporary_path / filename + try: + await sandbox.download(f"/logs/verifier/{filename}", destination) + except Exception: + continue + if not stat.S_ISREG(destination.lstat().st_mode): + return {}, result.error_type == "timeout", f"Verifier returned unsafe {filename}" + downloaded[filename] = destination.read_bytes() timed_out = result.error_type == "timeout" - reward_path = paths["verifier"] / "reward.json" - if not reward_path.is_file(): + if "reward.json" not in downloaded: reason = result.stderr or result.stdout or "LAB verifier did not produce reward.json" return {}, timed_out, reason[-2000:] - return json.loads(reward_path.read_text(encoding="utf-8")), timed_out, None + return downloaded, timed_out, None + + @staticmethod + def _materialize_verifier_downloads(paths: dict[str, Path], downloaded: dict[str, bytes]) -> dict[str, Any]: + for filename, contents in downloaded.items(): + (paths["verifier"] / filename).write_bytes(contents) + try: + reward_data = json.loads(downloaded["reward.json"].decode("utf-8")) + except (KeyError, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise LegalAgentBenchArtifactError(f"Invalid verifier reward.json: {exc}") from exc + if not isinstance(reward_data, dict): + raise LegalAgentBenchArtifactError("Invalid verifier reward.json: expected an object") + return reward_data @staticmethod def _runner_status(paths: dict[str, Path]) -> dict[str, Any]: @@ -1042,6 +1269,9 @@ async def run(self, request: Request, body: LegalAgentBenchRunRequest) -> LegalA failure_reason: Optional[str] = None reward_data: dict[str, Any] = {} verifier_sandbox: Optional[AsyncSandbox] = None + staged_paths: Optional[dict[str, Path]] = None + final_root: Optional[Path] = None + staging_temp: Optional[tempfile.TemporaryDirectory[str]] = None try: model_name = self._model_name() @@ -1049,29 +1279,50 @@ async def run(self, request: Request, body: LegalAgentBenchRunRequest) -> LegalA task_dir = resolve_task_dir(self.config.runtime_tasks_dir, body.instance_id) skills_dir = resolve_repo_path(self.config.skills_dir) params = compose_agent_input(task_dir, skills_dir, params) - paths = self._run_dirs(task_dir.name) + final_root = self._run_root(task_dir.name) + staging_temp = tempfile.TemporaryDirectory(prefix="legal-agent-bench-stage-") + staged_paths = self._paths_for_root(Path(staging_temp.name), create=True) image = await self._ensure_image(task_dir) deps_dir = await self._ensure_runtime(image) - self._stage_agent_source(paths) - self._write_runner_config(paths, params, self._model_url(body)) + self._stage_agent_source(staged_paths) + self._write_runner_config(staged_paths, params, self._model_url(body)) sandbox = self._agent_sandbox( image=image, task_dir=task_dir, skills_dir=skills_dir, deps_dir=deps_dir, - paths=paths, + paths=staged_paths, ) await sandbox.start() - started = time.time() - agent_result = await sandbox.exec( - "/agent_deps_mount/bin/python /trajectories_mount/agent_runner.py", - cwd="/workspace/output", - timeout_s=self.config.agent_timeout_seconds, + await self._stage_agent_sandbox( + sandbox, + task_dir=task_dir, + skills_dir=skills_dir, + deps_dir=deps_dir, + paths=staged_paths, ) - agent_elapsed = time.time() - started - (paths["agent"] / "stdout.log").write_text(agent_result.stdout or "") - (paths["agent"] / "stderr.log").write_text(agent_result.stderr or "") + with tempfile.TemporaryDirectory(prefix="legal-agent-bench-agent-") as download_raw: + started = time.time() + agent_result = await sandbox.exec( + "/agent_deps_mount/bin/python /trajectories_mount/agent_runner.py", + cwd="/workspace/output", + timeout_s=self.config.agent_timeout_seconds, + user="nobody", + ) + agent_elapsed = time.time() - started + downloads = await self._collect_agent_sandbox(sandbox, Path(download_raw)) + await sandbox.stop() + sandbox = None + self._materialize_agent_downloads( + staged_paths, + downloads, + stdout=agent_result.stdout or "", + stderr=agent_result.stderr or "", + ) + paths = self._publish_staged_run(staged_paths, final_root) + staging_temp.cleanup() + staging_temp = None agent_timed_out = agent_result.error_type == "timeout" runner_status = self._runner_status(paths) if runner_status.get("ok") is False: @@ -1110,15 +1361,17 @@ async def run(self, request: Request, body: LegalAgentBenchRunRequest) -> LegalA response=response, failure_reason=failure_reason, ) - await sandbox.stop() - sandbox = None if not agent_failed: verifier_sandbox = self._verifier_sandbox(image=image, task_dir=task_dir, paths=paths) await verifier_sandbox.start() - reward_data, verifier_timed_out, verifier_failure = await self._stage_and_run_verifier( + verifier_downloads, verifier_timed_out, verifier_failure = await self._stage_and_run_verifier( verifier_sandbox, task_dir, paths ) + await verifier_sandbox.stop() + verifier_sandbox = None + if verifier_failure is None: + reward_data = self._materialize_verifier_downloads(paths, verifier_downloads) verifier_failed = verifier_failure is not None failure_reason = verifier_failure or failure_reason except LegalAgentBenchTaskError as exc: @@ -1143,6 +1396,14 @@ async def run(self, request: Request, body: LegalAgentBenchRunRequest) -> LegalA except Exception as exc: sandbox_failed = True failure_reason = failure_reason or f"Verifier sandbox cleanup failed: {exc}" + if staging_temp is not None: + if staged_paths is not None and final_root is not None: + try: + paths = self._publish_staged_run(staged_paths, final_root) + except Exception as exc: + sandbox_failed = True + failure_reason = failure_reason or f"Artifact publication failed: {exc}" + staging_temp.cleanup() result = self._response( body=body, diff --git a/responses_api_agents/legal_agent_bench_agent/tests/test_app.py b/responses_api_agents/legal_agent_bench_agent/tests/test_app.py index 7100d7026b..da2d6238dc 100644 --- a/responses_api_agents/legal_agent_bench_agent/tests/test_app.py +++ b/responses_api_agents/legal_agent_bench_agent/tests/test_app.py @@ -5,6 +5,9 @@ from __future__ import annotations import json +import multiprocessing +import os +import tarfile from pathlib import Path from types import SimpleNamespace @@ -16,6 +19,11 @@ from responses_api_agents.legal_agent_bench_agent import app +def _runtime_archive_worker(deps_dir: str) -> tuple[str, int]: + archive = app.ensure_runtime_archive(Path(deps_dir)) + return str(archive), archive.stat().st_size + + def _config(**overrides) -> app.LegalAgentBenchAgentConfig: values = { "host": "0.0.0.0", @@ -283,9 +291,28 @@ def fake_run(args, **kwargs): assert "CLAUDE_SPEC=@anthropic-ai/claude-code@2.1.212" in docker_runs[1] +def test_runtime_archive_is_published_once_across_processes(tmp_path) -> None: + deps = tmp_path / "hermes_agent" / "recipe" + deps.mkdir(parents=True) + (deps / ".installed").write_text("recipe") + (deps / "runtime.py").write_text("VALUE = 1\n") + context = multiprocessing.get_context("fork") + + with context.Pool(2) as pool: + results = pool.map(_runtime_archive_worker, [str(deps), str(deps)]) + + assert results[0] == results[1] + archive = Path(results[0][0]) + assert archive.is_file() + with tarfile.open(archive, "r:gz") as contents: + assert "agent_deps_mount/runtime.py" in contents.getnames() + + def test_config_defaults_are_docker_and_single_concurrency() -> None: config = _config() assert config.concurrency == 1 + assert config.sandbox_provider == {"docker": {}} + assert config.sandbox_image is None assert config.docker_network == "host" assert config.results_dir == "results/legal_agent_bench" assert config.agent_timeout_seconds == 10800 @@ -522,9 +549,9 @@ def test_linux_bridge_provider_registers_host_gateway(monkeypatch) -> None: runner = app.LegalAgentBenchAgent.model_construct(config=_config(docker_network=None)) monkeypatch.setattr(app.sys, "platform", "linux") - provider = runner._provider() + provider = runner._provider_config() - assert provider._create_config.extra_run_args == [ + assert provider["docker"]["create"]["extra_run_args"] == [ "--add-host", "host.docker.internal:host-gateway", ] @@ -637,7 +664,6 @@ async def test_run_classifies_bad_agent_configuration_without_sandbox_failure(mo runner = app.LegalAgentBenchAgent.model_construct(config=_config()) runner._sem = app.asyncio.Semaphore(1) runner._session_results_dir = tmp_path / "results" - runner._session_results_dir.mkdir() async def ensure_image(_task): return "lab:image" @@ -689,7 +715,49 @@ async def fake_run(args, *, cwd, timeout): assert sum(call[1] == "build" for call in calls) == 1 -def test_sandbox_mounts_inputs_read_only_and_output_writable(monkeypatch, tmp_path) -> None: +@pytest.mark.asyncio +async def test_two_run_requests_overlap_when_server_concurrency_is_two(monkeypatch, tmp_path) -> None: + _root, task = _task_tree(tmp_path) + skills = _skills(tmp_path) + runner = app.LegalAgentBenchAgent.model_construct(config=_config(concurrency=2)) + runner._sem = app.asyncio.Semaphore(2) + runner._session_results_dir = tmp_path / "results" + active = 0 + maximum_active = 0 + both_started = app.asyncio.Event() + + async def ensure_image(_task): + nonlocal active, maximum_active + active += 1 + maximum_active = max(maximum_active, active) + if active == 2: + both_started.set() + await app.asyncio.wait_for(both_started.wait(), timeout=1) + active -= 1 + return "lab:image" + + async def stop_after_overlap(_image): + raise app.LegalAgentBenchConfigurationError("stop after overlap assertion") + + monkeypatch.setattr(app, "resolve_task_dir", lambda runtime, instance: task) + monkeypatch.setattr(app, "resolve_repo_path", lambda path: skills) + monkeypatch.setattr(app, "compose_agent_input", lambda task_dir, skills_dir, params: params) + monkeypatch.setattr(runner, "_model_name", lambda: "policy") + monkeypatch.setattr(runner, "_ensure_image", ensure_image) + monkeypatch.setattr(runner, "_ensure_runtime", stop_after_overlap) + body = app.LegalAgentBenchRunRequest( + instance_id="legal_agent_bench::area__task", + responses_create_params=NeMoGymResponseCreateParamsNonStreaming(input=[]), + ) + + first, second = await app.asyncio.gather(runner.run(None, body), runner.run(None, body)) + + assert maximum_active == 2 + assert first.configuration_failed is True + assert second.configuration_failed is True + + +def test_agent_sandbox_has_no_host_mounts(monkeypatch, tmp_path) -> None: _root, task = _task_tree(tmp_path) skills = _skills(tmp_path) deps = tmp_path / "deps" @@ -705,7 +773,8 @@ def fake_sandbox(provider, spec): return SimpleNamespace() runner = app.LegalAgentBenchAgent.model_construct(config=_config()) - monkeypatch.setattr(runner, "_provider", lambda: "docker-provider") + monkeypatch.setattr(runner, "_provider_config", lambda: {"docker": {}}) + monkeypatch.setattr(runner, "_sandbox_metadata", lambda: {}) monkeypatch.setattr(app, "AsyncSandbox", fake_sandbox) runner._agent_sandbox( @@ -716,19 +785,11 @@ def fake_sandbox(provider, spec): paths=paths, ) - volumes = captured["spec"].provider_options["volumes"] - assert f"{paths['agent_source']}:/agent_source_mount:ro" in volumes - assert f"{task / 'documents'}:/workspace/vdr:ro" in volumes - assert f"{skills}:/workspace/skills:ro" in volumes - assert f"{deps}:/agent_deps_mount:ro" in volumes - assert f"{paths['output']}:/workspace/output" in volumes - assert str(app.PARENT_DIR) not in {volume.split(":", 1)[0] for volume in volumes} - assert all("resources_servers" not in volume for volume in volumes) - assert all("tests" not in volume for volume in volumes) - assert all("LAB_JUDGE" not in volume for volume in volumes) + assert captured["provider"] == {"docker": {}} + assert captured["spec"].provider_options == {} -def test_verifier_sandbox_mounts_only_completed_lab_run(monkeypatch, tmp_path) -> None: +def test_verifier_sandbox_has_no_host_mounts(monkeypatch, tmp_path) -> None: _root, task = _task_tree(tmp_path) paths = {"lab_run": tmp_path / "lab-run"} paths["lab_run"].mkdir() @@ -740,13 +801,14 @@ def fake_sandbox(provider, spec): return SimpleNamespace() runner = app.LegalAgentBenchAgent.model_construct(config=_config()) - monkeypatch.setattr(runner, "_provider", lambda: "docker-provider") + monkeypatch.setattr(runner, "_provider_config", lambda: {"docker": {}}) + monkeypatch.setattr(runner, "_sandbox_metadata", lambda: {}) monkeypatch.setattr(app, "AsyncSandbox", fake_sandbox) runner._verifier_sandbox(image="lab:image", task_dir=task, paths=paths) assert captured["spec"].workdir == "/logs/agent/artifacts/lab-run/output" - assert captured["spec"].provider_options["volumes"] == [f"{paths['lab_run']}:/logs/agent/artifacts/lab-run:ro"] + assert captured["spec"].provider_options == {} @pytest.mark.asyncio @@ -769,6 +831,8 @@ async def exec(self, *args, **kwargs): return SimpleNamespace(return_code=0, error_type=None, stdout="", stderr="") async def stop(self): + if self.phase == "agent": + assert not runner._session_results_dir.exists() events.append(f"{self.phase}:stop") agent_sandbox = PhaseSandbox("agent") @@ -776,7 +840,6 @@ async def stop(self): runner = app.LegalAgentBenchAgent.model_construct(config=_config()) runner._sem = app.asyncio.Semaphore(1) runner._session_results_dir = tmp_path / "results" - runner._session_results_dir.mkdir() async def ensure_image(_task): return "lab:image" @@ -790,7 +853,14 @@ def write_runner(paths, params, model_url): async def run_verifier(sandbox, task_dir, paths): assert sandbox is verifier_sandbox assert events == ["agent:start", "agent:exec", "agent:stop", "verifier:start"] - return {"reward": 1.0, "criteria_pass_rate": 1.0}, False, None + assert runner._session_results_dir.is_dir() + return {"reward.json": b'{"reward": 1.0, "criteria_pass_rate": 1.0}'}, False, None + + async def stage_agent(*args, **kwargs): + return None + + async def collect_agent(*args, **kwargs): + return {} monkeypatch.setattr(app, "resolve_task_dir", lambda runtime, instance: task) monkeypatch.setattr(app, "resolve_repo_path", lambda path: skills) @@ -802,6 +872,9 @@ async def run_verifier(sandbox, task_dir, paths): monkeypatch.setattr(runner, "_model_url", lambda body: "http://model") monkeypatch.setattr(runner, "_model_name", lambda: "policy") monkeypatch.setattr(runner, "_agent_sandbox", lambda **kwargs: agent_sandbox) + monkeypatch.setattr(runner, "_stage_agent_sandbox", stage_agent) + monkeypatch.setattr(runner, "_collect_agent_sandbox", collect_agent) + monkeypatch.setattr(runner, "_materialize_agent_downloads", lambda *args, **kwargs: None) monkeypatch.setattr(runner, "_verifier_sandbox", lambda **kwargs: verifier_sandbox) monkeypatch.setattr(runner, "_artifacts", lambda *args, **kwargs: None) monkeypatch.setattr(runner, "_stage_and_run_verifier", run_verifier) @@ -815,6 +888,7 @@ async def run_verifier(sandbox, task_dir, paths): assert events == ["agent:start", "agent:exec", "agent:stop", "verifier:start", "verifier:stop"] assert response.reward == 1.0 assert response.sandbox_failed is False + assert Path(response.artifact_dir).is_dir() terminal_output = capsys.readouterr().out assert "LAB rollout artifacts:" in terminal_output assert "LAB rollout complete:" in terminal_output @@ -861,6 +935,12 @@ def write_runner(paths, params, model_url): ) ) + async def stage_agent(*args, **kwargs): + return None + + async def collect_agent(*args, **kwargs): + return {} + monkeypatch.setattr(app, "resolve_task_dir", lambda runtime, instance: task) monkeypatch.setattr(app, "resolve_repo_path", lambda path: skills) monkeypatch.setattr(app, "compose_agent_input", lambda task_dir, skills_dir, params: params) @@ -871,6 +951,9 @@ def write_runner(paths, params, model_url): monkeypatch.setattr(runner, "_model_url", lambda body: "http://model") monkeypatch.setattr(runner, "_model_name", lambda: "policy") monkeypatch.setattr(runner, "_agent_sandbox", lambda **kwargs: AgentSandbox()) + monkeypatch.setattr(runner, "_stage_agent_sandbox", stage_agent) + monkeypatch.setattr(runner, "_collect_agent_sandbox", collect_agent) + monkeypatch.setattr(runner, "_materialize_agent_downloads", lambda *args, **kwargs: None) monkeypatch.setattr( runner, "_verifier_sandbox", @@ -927,6 +1010,8 @@ async def exec(self, command, **kwargs): )() async def download(self, source, destination): + if self.reward is None: + raise FileNotFoundError(source) payload = self.reward if source.endswith("reward.json") else {"summary": "ok"} Path(destination).write_text(json.dumps(payload)) @@ -934,14 +1019,16 @@ async def download(self, source, destination): @pytest.mark.asyncio async def test_verifier_is_staged_after_agent_and_receives_secrets_only_on_exec(tmp_path) -> None: _root, task = _task_tree(tmp_path) - paths = {"verifier": tmp_path / "verifier"} + paths = {"verifier": tmp_path / "verifier", "lab_run": tmp_path / "lab-run"} paths["verifier"].mkdir() + (paths["lab_run"] / "output").mkdir(parents=True) sandbox = _FakeSandbox({"reward": 1.0, "criteria_pass_rate": 1.0, "judge_error_count": 0}) runner = app.LegalAgentBenchAgent.model_construct(config=_config()) - reward, timed_out, failure = await runner._stage_and_run_verifier(sandbox, task, paths) + downloaded, timed_out, failure = await runner._stage_and_run_verifier(sandbox, task, paths) + reward = runner._materialize_verifier_downloads(paths, downloaded) - assert sandbox.uploaded == ["/tmp/legal-agent-bench-tests.tar.gz"] + assert sandbox.uploaded == ["/tmp/legal-agent-bench-verifier-input.tar.gz"] verifier_call = next(call for call in sandbox.exec_calls if "bash /tests/test.sh" in call[0]) assert verifier_call[1]["env"]["LAB_JUDGE_API_KEY"] == "secret" # pragma: allowlist secret assert reward["reward"] == 1.0 @@ -952,8 +1039,9 @@ async def test_verifier_is_staged_after_agent_and_receives_secrets_only_on_exec( @pytest.mark.asyncio async def test_missing_verifier_reward_returns_failure(tmp_path) -> None: _root, task = _task_tree(tmp_path) - paths = {"verifier": tmp_path / "verifier"} + paths = {"verifier": tmp_path / "verifier", "lab_run": tmp_path / "lab-run"} paths["verifier"].mkdir() + (paths["lab_run"] / "output").mkdir(parents=True) sandbox = _FakeSandbox(None, timed_out=True) runner = app.LegalAgentBenchAgent.model_construct(config=_config(verifier_timeout_seconds=1)) @@ -962,3 +1050,106 @@ async def test_missing_verifier_reward_returns_failure(tmp_path) -> None: assert reward == {} assert timed_out is True assert "reward.json" in failure + + +@pytest.mark.parametrize("link_type", [tarfile.SYMTYPE, tarfile.LNKTYPE]) +def test_untrusted_output_archive_rejects_links_without_touching_host(tmp_path, link_type) -> None: + victim = tmp_path / "victim.txt" + victim.write_text("safe") + archive_path = tmp_path / "malicious.tar.gz" + with tarfile.open(archive_path, "w:gz") as archive: + link = tarfile.TarInfo("stdout.log") + link.type = link_type + link.linkname = str(victim) + archive.addfile(link) + + with pytest.raises(app.LegalAgentBenchArtifactError, match="links"): + app._extract_untrusted_archive(archive_path, tmp_path / "output") + + assert victim.read_text() == "safe" + + +def test_materialized_output_is_owned_by_invoking_user(tmp_path) -> None: + source = tmp_path / "source" + source.mkdir() + (source / "memo.txt").write_text("complete") + archive_path = tmp_path / "output.tar.gz" + app._create_archive(archive_path, [(source, ".")]) + output = tmp_path / "materialized" + + app._extract_untrusted_archive(archive_path, output) + + assert (output / "memo.txt").stat().st_uid == os.getuid() + assert (output / "memo.txt").stat().st_gid == os.getgid() + + +def test_named_remote_provider_requires_immutable_image(tmp_path) -> None: + global_config = { + "sandbox": { + "default_metadata": {"sandbox-api": "remote"}, + "remote": {"endpoint": "sandbox.internal"}, + } + } + runner = app.LegalAgentBenchAgent.model_construct( + config=_config(sandbox_provider="sandbox", sandbox_image="registry.example/lab:latest"), + server_client=SimpleNamespace(global_config_dict=global_config), + ) + + assert "remote" in runner._provider_config() + assert runner._sandbox_metadata() == {"sandbox-api": "remote"} + body = app.LegalAgentBenchRunRequest( + instance_id="legal_agent_bench::area__task", + responses_create_params=NeMoGymResponseCreateParamsNonStreaming(input=[]), + ) + with pytest.raises(app.LegalAgentBenchConfigurationError, match="sandbox_model_base_url"): + runner._model_url(body) + with pytest.raises(app.LegalAgentBenchConfigurationError, match="immutable"): + app.asyncio.run(runner._ensure_image(tmp_path)) + + runner.config.sandbox_image = "registry.example/lab@sha256:" + "a" * 64 + assert app.asyncio.run(runner._ensure_image(tmp_path)) == runner.config.sandbox_image + + +@pytest.mark.asyncio +async def test_agent_staging_excludes_tests_and_judge_credentials(tmp_path) -> None: + _root, task = _task_tree(tmp_path) + skills = _skills(tmp_path) + deps = tmp_path / "cache" / "recipe" + deps.mkdir(parents=True) + (deps / ".installed").write_text("recipe") + paths = {"agent_source": tmp_path / "agent-source", "runtime": tmp_path / "runtime"} + for path in paths.values(): + path.mkdir() + (paths["agent_source"] / "responses_api_agents" / "hermes_agent").mkdir(parents=True) + (paths["agent_source"] / "responses_api_agents" / "hermes_agent" / "app.py").write_text("VALUE = 1\n") + (paths["runtime"] / "runner.json").write_text("{}") + + class CaptureSandbox: + def __init__(self): + self.members = {} + self.exec_commands = [] + + async def upload(self, source, destination): + with tarfile.open(source, "r:gz") as archive: + self.members[destination] = {member.name for member in archive.getmembers()} + + async def exec(self, command, **kwargs): + self.exec_commands.append(command) + return SimpleNamespace(return_code=0, stdout="", stderr="") + + sandbox = CaptureSandbox() + runner = app.LegalAgentBenchAgent.model_construct(config=_config()) + + await runner._stage_agent_sandbox( + sandbox, + task_dir=task, + skills_dir=skills, + deps_dir=deps, + paths=paths, + ) + + staged = sandbox.members["/tmp/legal-agent-bench-agent-input.tar.gz"] + assert any(name.startswith("workspace/vdr") for name in staged) + assert any(name.startswith("workspace/skills") for name in staged) + assert not any("task.toml" in name or "/tests" in name or "LAB_JUDGE" in name for name in staged) + assert "chmod -R a+rX,a-w /agent_source_mount /agent_deps_mount" in sandbox.exec_commands[0] From 907c9ec93e93bf7a7159324e2311cbf5300a2e48 Mon Sep 17 00:00:00 2001 From: Ryan Stewart Date: Mon, 3 Aug 2026 14:22:21 -0400 Subject: [PATCH 06/26] support LAB evaluations on ECS Fargate Signed-off-by: Ryan Stewart --- benchmarks/legal_agent_bench/README.md | 83 +++++++++++++++- .../sandbox/providers/ecs_fargate/provider.py | 4 +- .../legal_agent_bench_agent/README.md | 16 ++-- .../legal_agent_bench_agent/app.py | 40 +++++++- .../legal_agent_bench_claude_code.yaml | 5 + .../configs/legal_agent_bench_codex.yaml | 5 + .../configs/legal_agent_bench_hermes.yaml | 5 + .../legal_agent_bench_agent/tests/test_app.py | 96 +++++++++++++++++++ tests/unit_tests/test_ecs_fargate_provider.py | 8 ++ 9 files changed, 248 insertions(+), 14 deletions(-) diff --git a/benchmarks/legal_agent_bench/README.md b/benchmarks/legal_agent_bench/README.md index a34c60e6dd..f808e49b98 100644 --- a/benchmarks/legal_agent_bench/README.md +++ b/benchmarks/legal_agent_bench/README.md @@ -24,7 +24,9 @@ agent during dataset collation. ## Requirements - Python 3.13.13 and the repository environment installed with `uv` -- Docker with a running daemon +- Docker with a running daemon for the default local backend and portable + harness provisioning +- The `sandbox` dependency extra when using ECS Fargate - Authorized OpenAI-compatible policy and judge endpoints in the root `env.yaml` - At least 10 GB of free working space @@ -56,6 +58,12 @@ uv sync --extra dev docker info >/dev/null ``` +For a remote sandbox provider, also install its dependencies with: + +```bash +uv sync --extra dev --extra sandbox +``` + Configure the policy and judge endpoints in the gitignored root `env.yaml` as shown in the [resource-server README](../../resources_servers/legal_agent_bench/README.md#requirements). @@ -201,6 +209,77 @@ Remove `--limit 1` from the desired smoke command. Choose a new output filename and increase `--concurrency` only after confirming that Docker, the policy endpoint, and the judge endpoint can sustain it. +## Use ECS Fargate + +Gym's native `ecs_fargate` provider can run the native, Hermes, Claude Code, +or Codex LAB variants without changing their agent loops. The reference ECS +infrastructure is discovered from `//ecs-sandbox/config` in SSM; +`ssm_project` defaults to `harbor`. Configure an AWS profile locally and export +the profile and region before starting Gym: + +```bash +export AWS_PROFILE=gym-ecs +export AWS_REGION=us-east-1 +export AWS_DEFAULT_REGION="$AWS_REGION" +``` + +The orchestrator must be able to reach the ECS task SSH sidecar on TCP port +`52222`. The SSM configuration must provide the cluster, subnets, security +groups, task roles, ECR repository, S3 staging bucket, and SSH-sidecar key +ARNs. Install the `sandbox` dependency extra as shown above. + +Build the shared LAB environment for `linux/amd64`, push it to ECR, and use its +immutable digest. The prepared LAB tasks currently share this environment: + +```bash +LAB_ENV_DIR=resources_servers/legal_agent_bench/data/runtime/harbor_tasks/legal_agent_bench/antitrust-competition__analyze-antitrust-hsr-strategy/environment +ECR_REPOSITORY=.dkr.ecr..amazonaws.com/ +ECR_REGISTRY=${ECR_REPOSITORY%%/*} +ECR_REPOSITORY_NAME=${ECR_REPOSITORY#*/} +LAB_IMAGE_TAG=legal-agent-bench-smoke + +aws ecr get-login-password --region "$AWS_REGION" | + docker login --username AWS --password-stdin "$ECR_REGISTRY" +docker buildx build \ + --platform linux/amd64 \ + --tag "$ECR_REPOSITORY:$LAB_IMAGE_TAG" \ + --push \ + "$LAB_ENV_DIR" +LAB_IMAGE_DIGEST=$(aws ecr describe-images \ + --region "$AWS_REGION" \ + --repository-name "$ECR_REPOSITORY_NAME" \ + --image-ids "imageTag=$LAB_IMAGE_TAG" \ + --query 'imageDetails[0].imageDigest' \ + --output text) +export LAB_ECS_IMAGE="$ECR_REPOSITORY@$LAB_IMAGE_DIGEST" +docker pull --platform linux/amd64 "$LAB_ECS_IMAGE" +``` + +Run one native smoke through ECS: + +```bash +gym eval run \ + --model-type vllm_model \ + --benchmark legal_agent_bench \ + --config nemo_gym/sandbox/providers/ecs_fargate/configs/ecs_fargate.yaml \ + --split benchmark \ + --output results/legal_agent_bench_native_ecs_smoke.jsonl \ + --concurrency 1 \ + --limit 1 \ + +legal_agent_bench_benchmark_native_agent.responses_api_agents.legal_agent_bench_agent.sandbox_provider=sandbox \ + "+legal_agent_bench_benchmark_native_agent.responses_api_agents.legal_agent_bench_agent.sandbox_image=${LAB_ECS_IMAGE}" \ + +legal_agent_bench_benchmark_native_agent.responses_api_agents.legal_agent_bench_agent.runtime_docker_platform=linux/amd64 +``` + +ECS tunnels the rollout-scoped Gym policy-model URL into the agent task, so +model credentials remain in Gym and `sandbox_model_base_url` is unnecessary. +An explicit `sandbox_model_base_url` still bypasses the tunnel when the endpoint +is already reachable from the task. The verifier contacts the configured judge +endpoint directly, so that endpoint must be reachable from the Fargate VPC. +Each completed rollout creates one agent task followed by one verifier task; +both are stopped during cleanup. Artifacts use the same local paths documented +above. + ## Manage servers separately To manage the servers separately, start the desired variant first: @@ -244,7 +323,7 @@ For configurable variants, the rollout also reports `agent_failed`, `task_failed` identifies an unsafe, unknown, incomplete, or malformed task; `configuration_failed` identifies an invalid harness selection or missing required pin. The runner checks policy-model connectivity from inside the -Docker sandbox before starting the harness. A connectivity or harness failure +selected sandbox before starting the harness. A connectivity or harness failure is masked and is not sent to the judge. A normal model/task result can still receive zero reward without those flags. Infrastructure, configuration, task-loading, or judge failures set `mask_sample`; do not treat those zeroes as diff --git a/nemo_gym/sandbox/providers/ecs_fargate/provider.py b/nemo_gym/sandbox/providers/ecs_fargate/provider.py index b586363940..10d8ab974e 100644 --- a/nemo_gym/sandbox/providers/ecs_fargate/provider.py +++ b/nemo_gym/sandbox/providers/ecs_fargate/provider.py @@ -94,7 +94,9 @@ def _apply_spec_overrides(cfg: engine.EcsFargateConfig, spec: SandboxSpec) -> en overrides["cpu"] = str(int(resources.cpu * 1024)) if resources.memory_mib is not None: overrides["memory"] = str(int(resources.memory_mib)) - if resources.disk_gib is not None: + # Fargate supplies 20 GiB when ephemeralStorage is omitted and accepts only + # 21-200 GiB explicitly. The implicit default satisfies smaller requests. + if resources.disk_gib is not None and resources.disk_gib > 20: overrides["ephemeral_storage_gib"] = int(resources.disk_gib) return replace(cfg, **overrides) if overrides else cfg diff --git a/responses_api_agents/legal_agent_bench_agent/README.md b/responses_api_agents/legal_agent_bench_agent/README.md index 00e8dff792..09eac4447b 100644 --- a/responses_api_agents/legal_agent_bench_agent/README.md +++ b/responses_api_agents/legal_agent_bench_agent/README.md @@ -48,11 +48,13 @@ process namespace or writable filesystem with the agent. The default provider is Docker. The runner translates derived host-loopback model URLs to `host.docker.internal` on Docker Desktop and registers the -equivalent host gateway for Linux bridge networking. A named Gym provider can -be selected through `sandbox_provider`; non-Docker providers require an -immutable registry `sandbox_image` and an explicit reachable -`sandbox_model_base_url`. Before importing the selected harness, the -in-container runner +equivalent host gateway for Linux bridge networking. Gym's ECS Fargate provider +can be selected through `sandbox_provider`; non-Docker providers require an +immutable registry `sandbox_image`. ECS uses +Gym's provider-native SSH reverse tunnel to expose the rollout-scoped policy +proxy as `LAB_POLICY_MODEL_URL`, keeping model credentials out of the sandbox. +An explicit `sandbox_model_base_url` is used unchanged and disables that ECS +tunnel. Before importing the selected harness, the in-container runner performs a bounded TCP connectivity check and writes `runtime/runner_status.json`. Connectivity or harness failures set the `mask_sample` flag on the row, skip the judge, and are distinct from an ordinary zero-reward @@ -92,5 +94,5 @@ that inherits it. CLI versions belong in the harness configuration's `agent_kwargs`; provisioning derives its exact package specification from that pin and rejects unpinned Claude Code or Codex configurations. -Docker remains the zero-configuration local backend. Non-Docker providers -require an immutable registry image and a model URL reachable from the sandbox. +Docker remains the zero-configuration local backend. See the benchmark README +for ECS Fargate requirements and provider overrides. diff --git a/responses_api_agents/legal_agent_bench_agent/app.py b/responses_api_agents/legal_agent_bench_agent/app.py index f0102a91f6..197421113d 100644 --- a/responses_api_agents/legal_agent_bench_agent/app.py +++ b/responses_api_agents/legal_agent_bench_agent/app.py @@ -99,7 +99,9 @@ from nemo_gym.server_utils import ServerClient runner = json.loads(Path("/trajectories_mount/runner.json").read_text()) -model_url = runner["model_url"].rstrip("/") +# ECS Fargate injects this value after resolving the host-side policy URL through +# its SSH reverse tunnel. Other providers use the URL persisted in runner.json. +model_url = os.environ.get("LAB_POLICY_MODEL_URL", runner["model_url"]).rstrip("/") model_url_v1 = model_url if model_url.endswith("/v1") else model_url + "/v1" status_path = Path("/trajectories_mount/runner_status.json") @@ -552,6 +554,16 @@ def sandbox_model_url( return urlunsplit(parsed._replace(netloc=f"{userinfo}host.docker.internal{port}")) +def host_tunnel_model_url(model_url: str) -> str: + """Translate wildcard listeners into a concrete host endpoint for an ECS reverse tunnel.""" + parsed = urlsplit(model_url) + if (parsed.hostname or "").lower() not in {"0.0.0.0", "::"}: + return model_url + userinfo = parsed.netloc.rsplit("@", 1)[0] + "@" if "@" in parsed.netloc else "" + port = f":{parsed.port}" if parsed.port is not None else "" + return urlunsplit(parsed._replace(netloc=f"{userinfo}127.0.0.1{port}")) + + def _response_output_text(response: NeMoGymResponse) -> str: parts: list[str] = [] for item in response.output: @@ -784,7 +796,8 @@ async def _ensure_image(self, task_dir: Path) -> str: def _model_url(self, body: LegalAgentBenchRunRequest) -> str: if self.config.sandbox_model_base_url: return self.config.sandbox_model_base_url - if _provider_name(self._provider_config()) != "docker": + provider_name = _provider_name(self._provider_config()) + if provider_name not in {"docker", "ecs_fargate"}: raise LegalAgentBenchConfigurationError( "Non-Docker LAB sandboxes require sandbox_model_base_url reachable from the provider" ) @@ -800,7 +813,22 @@ def _model_url(self, body: LegalAgentBenchRunRequest) -> str: ) from exc rollout_id = self.rollout_id_from_run(body) prefixed_url = apply_rollout_prefix(base_url, rollout_id) if rollout_id else base_url - return sandbox_model_url(prefixed_url, docker_network=self.config.docker_network) + if provider_name == "docker": + return sandbox_model_url(prefixed_url, docker_network=self.config.docker_network) + return host_tunnel_model_url(prefixed_url) + + def _agent_provider_options(self, model_url: str) -> dict[str, Any]: + """Return provider-specific routing without exposing policy credentials to the sandbox.""" + if _provider_name(self._provider_config()) != "ecs_fargate" or self.config.sandbox_model_base_url: + return {} + return { + "outside_endpoints": [ + { + "url": model_url, + "env_var": "LAB_POLICY_MODEL_URL", + } + ] + } def _model_name(self) -> str: global_config = getattr(getattr(self, "server_client", None), "global_config_dict", None) @@ -917,6 +945,7 @@ def _agent_sandbox( skills_dir: Path, deps_dir: Path, paths: dict[str, Path], + model_url: str, ) -> AsyncSandbox: return AsyncSandbox( self._provider_config(), @@ -926,6 +955,7 @@ def _agent_sandbox( workdir="/workspace/output", resources=_sandbox_resources(task_dir), metadata=self._sandbox_metadata(), + provider_options=self._agent_provider_options(model_url), ), ) @@ -1285,7 +1315,8 @@ async def run(self, request: Request, body: LegalAgentBenchRunRequest) -> LegalA image = await self._ensure_image(task_dir) deps_dir = await self._ensure_runtime(image) self._stage_agent_source(staged_paths) - self._write_runner_config(staged_paths, params, self._model_url(body)) + model_url = self._model_url(body) + self._write_runner_config(staged_paths, params, model_url) sandbox = self._agent_sandbox( image=image, @@ -1293,6 +1324,7 @@ async def run(self, request: Request, body: LegalAgentBenchRunRequest) -> LegalA skills_dir=skills_dir, deps_dir=deps_dir, paths=staged_paths, + model_url=model_url, ) await sandbox.start() await self._stage_agent_sandbox( diff --git a/responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_claude_code.yaml b/responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_claude_code.yaml index 7d446f2968..5c498b5113 100644 --- a/responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_claude_code.yaml +++ b/responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_claude_code.yaml @@ -19,6 +19,11 @@ legal_agent_bench_claude_code_agent: concurrency: 1 agent_timeout_seconds: 10800 verifier_timeout_seconds: 3600 + sandbox_provider: + docker: {} + sandbox_image: null + runtime_docker_platform: null + sandbox_model_base_url: null agent_server_module: responses_api_agents.claude_code_agent.app agent_server_class: ClaudeCodeAgent agent_config_class: ClaudeCodeAgentConfig diff --git a/responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_codex.yaml b/responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_codex.yaml index dfe39f106b..5e8861a56e 100644 --- a/responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_codex.yaml +++ b/responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_codex.yaml @@ -19,6 +19,11 @@ legal_agent_bench_codex_agent: concurrency: 1 agent_timeout_seconds: 10800 verifier_timeout_seconds: 3600 + sandbox_provider: + docker: {} + sandbox_image: null + runtime_docker_platform: null + sandbox_model_base_url: null agent_server_module: responses_api_agents.codex_agent.app agent_server_class: CodexAgent agent_config_class: CodexAgentConfig diff --git a/responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_hermes.yaml b/responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_hermes.yaml index 0fb6d8852e..abd3a74b7d 100644 --- a/responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_hermes.yaml +++ b/responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_hermes.yaml @@ -19,6 +19,11 @@ legal_agent_bench_hermes_agent: concurrency: 1 agent_timeout_seconds: 10800 verifier_timeout_seconds: 3600 + sandbox_provider: + docker: {} + sandbox_image: null + runtime_docker_platform: null + sandbox_model_base_url: null agent_server_module: responses_api_agents.hermes_agent.app agent_server_class: HermesAgent agent_config_class: HermesAgentConfig diff --git a/responses_api_agents/legal_agent_bench_agent/tests/test_app.py b/responses_api_agents/legal_agent_bench_agent/tests/test_app.py index da2d6238dc..d3ca27feeb 100644 --- a/responses_api_agents/legal_agent_bench_agent/tests/test_app.py +++ b/responses_api_agents/legal_agent_bench_agent/tests/test_app.py @@ -464,6 +464,7 @@ def test_runner_config_preserves_dynamic_agent_configuration(tmp_path) -> None: assert "inspect.signature(agent.responses)" in (tmp_path / "agent_runner.py").read_text() assert "SimpleNamespace(path_params={})" in (tmp_path / "agent_runner.py").read_text() assert "socket.create_connection" in (tmp_path / "agent_runner.py").read_text() + assert 'os.environ.get("LAB_POLICY_MODEL_URL", runner["model_url"])' in (tmp_path / "agent_runner.py").read_text() assert "runner_status.json" in (tmp_path / "agent_runner.py").read_text() assert "hermes_model_metadata.fetch_endpoint_model_metadata" in (tmp_path / "agent_runner.py").read_text() assert "hermes_usage_pricing.fetch_endpoint_model_metadata" in (tmp_path / "agent_runner.py").read_text() @@ -526,6 +527,30 @@ def test_model_url_uses_override_without_server_lookup() -> None: assert runner._model_url(body) == "http://host.docker.internal:9000" +def test_ecs_model_url_uses_host_policy_proxy_for_reverse_tunnel(monkeypatch) -> None: + runner = app.LegalAgentBenchAgent.model_construct( + config=_config( + sandbox_provider={ + "ecs_fargate": { + "region": "us-east-1", + "cluster": "test", + } + } + ), + server_client=SimpleNamespace( + global_config_dict={}, + _build_server_base_url=lambda _config: "http://0.0.0.0:16300", + ), + ) + monkeypatch.setattr(app, "get_first_server_config_dict", lambda *_args: {}) + body = app.LegalAgentBenchRunRequest( + instance_id="legal_agent_bench::area__task", + responses_create_params=NeMoGymResponseCreateParamsNonStreaming(input=[]), + ) + + assert runner._model_url(body) == "http://127.0.0.1:16300" + + @pytest.mark.parametrize( ("url", "network", "platform_name", "expected"), [ @@ -783,12 +808,83 @@ def fake_sandbox(provider, spec): skills_dir=skills, deps_dir=deps, paths=paths, + model_url="http://model", ) assert captured["provider"] == {"docker": {}} assert captured["spec"].provider_options == {} +def test_ecs_agent_sandbox_tunnels_derived_policy_model_url(monkeypatch, tmp_path) -> None: + _root, task = _task_tree(tmp_path) + skills = _skills(tmp_path) + deps = tmp_path / "deps" + deps.mkdir() + paths = {name: tmp_path / name for name in ("runtime", "agent_source", "agent", "output", "workspace")} + for path in paths.values(): + path.mkdir() + captured = {} + + def fake_sandbox(provider, spec): + captured["provider"] = provider + captured["spec"] = spec + return SimpleNamespace() + + provider = {"ecs_fargate": {"region": "us-east-1", "cluster": "test"}} + runner = app.LegalAgentBenchAgent.model_construct(config=_config(sandbox_provider=provider)) + monkeypatch.setattr(runner, "_sandbox_metadata", lambda: {}) + monkeypatch.setattr(app, "AsyncSandbox", fake_sandbox) + + runner._agent_sandbox( + image="registry.example/lab@sha256:" + "a" * 64, + task_dir=task, + skills_dir=skills, + deps_dir=deps, + paths=paths, + model_url="http://127.0.0.1:16300/rollout/run-1", + ) + + assert captured["provider"] == provider + assert captured["spec"].provider_options == { + "outside_endpoints": [ + { + "url": "http://127.0.0.1:16300/rollout/run-1", + "env_var": "LAB_POLICY_MODEL_URL", + } + ] + } + + +def test_ecs_agent_sandbox_uses_explicit_reachable_model_url_without_tunnel(monkeypatch, tmp_path) -> None: + _root, task = _task_tree(tmp_path) + captured = {} + + def fake_sandbox(provider, spec): + captured["spec"] = spec + return SimpleNamespace() + + provider = {"ecs_fargate": {"region": "us-east-1", "cluster": "test"}} + runner = app.LegalAgentBenchAgent.model_construct( + config=_config( + sandbox_provider=provider, + sandbox_model_base_url="https://model.example/v1", + ) + ) + monkeypatch.setattr(runner, "_sandbox_metadata", lambda: {}) + monkeypatch.setattr(app, "AsyncSandbox", fake_sandbox) + + runner._agent_sandbox( + image="registry.example/lab@sha256:" + "a" * 64, + task_dir=task, + skills_dir=tmp_path / "skills", + deps_dir=tmp_path / "deps", + paths={}, + model_url="https://model.example/v1", + ) + + assert captured["spec"].provider_options == {} + + def test_verifier_sandbox_has_no_host_mounts(monkeypatch, tmp_path) -> None: _root, task = _task_tree(tmp_path) paths = {"lab_run": tmp_path / "lab-run"} diff --git a/tests/unit_tests/test_ecs_fargate_provider.py b/tests/unit_tests/test_ecs_fargate_provider.py index 5832c1b62f..cf89666040 100644 --- a/tests/unit_tests/test_ecs_fargate_provider.py +++ b/tests/unit_tests/test_ecs_fargate_provider.py @@ -419,6 +419,14 @@ def test_apply_spec_overrides_maps_resources_and_ttl(): assert _apply_spec_overrides(cfg, SandboxSpec(image="img")) is cfg +def test_apply_spec_overrides_uses_implicit_storage_for_requests_up_to_20_gib(): + cfg = engine.EcsFargateConfig(region="us-east-1") + + for requested in (1, 10, 20): + out = _apply_spec_overrides(cfg, SandboxSpec(image="img", resources={"disk_gib": requested})) + assert out.ephemeral_storage_gib is None + + def test_apply_spec_overrides_rejects_gpu(): from nemo_gym.sandbox.providers import SandboxCreateError From 10a7564c143ba5ead5f523753d3b07bca2ed44f7 Mon Sep 17 00:00:00 2001 From: Ryan Stewart Date: Mon, 3 Aug 2026 14:22:40 -0400 Subject: [PATCH 07/26] document matching LAB concurrency limits Signed-off-by: Ryan Stewart --- benchmarks/legal_agent_bench/README.md | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/benchmarks/legal_agent_bench/README.md b/benchmarks/legal_agent_bench/README.md index f808e49b98..b8870e1719 100644 --- a/benchmarks/legal_agent_bench/README.md +++ b/benchmarks/legal_agent_bench/README.md @@ -206,8 +206,24 @@ configurable runner's top-level flags. ## Run a larger evaluation Remove `--limit 1` from the desired smoke command. Choose a new output filename -and increase `--concurrency` only after confirming that Docker, the policy -endpoint, and the judge endpoint can sustain it. +and increase client and agent-server concurrency together only after confirming +that the sandbox backend, policy endpoint, and judge endpoint can sustain it. +For example, a two-way Hermes run uses both settings: + +```bash +gym eval run \ + --model-type vllm_model \ + --benchmark legal_agent_bench/config_hermes \ + --split benchmark \ + --output results/legal_agent_bench_hermes.jsonl \ + --concurrency 2 \ + +legal_agent_bench_benchmark_hermes_agent.responses_api_agents.legal_agent_bench_agent.concurrency=2 +``` + +Use the corresponding `legal_agent_bench_benchmark_claude_code_agent` or +`legal_agent_bench_benchmark_codex_agent` prefix for those variants. The +agent-server default stays at `1`; changing only `--concurrency` leaves the +server-side semaphore serial. ## Use ECS Fargate From 6a3dc5bf4e85e7586690eea8462cf05183d70a64 Mon Sep 17 00:00:00 2001 From: Ryan Stewart Date: Mon, 3 Aug 2026 14:23:13 -0400 Subject: [PATCH 08/26] provide Codex model metadata in LAB sandboxes Signed-off-by: Ryan Stewart --- .../legal_agent_bench_agent/app.py | 36 +++++++++++++++- .../legal_agent_bench_agent/tests/test_app.py | 43 +++++++++++++++++++ 2 files changed, 78 insertions(+), 1 deletion(-) diff --git a/responses_api_agents/legal_agent_bench_agent/app.py b/responses_api_agents/legal_agent_bench_agent/app.py index 197421113d..365b22f7a8 100644 --- a/responses_api_agents/legal_agent_bench_agent/app.py +++ b/responses_api_agents/legal_agent_bench_agent/app.py @@ -56,6 +56,33 @@ "claude_code_agent": ("claude_code_version", "CLAUDE_SPEC", "@anthropic-ai/claude-code"), "codex_agent": ("codex_version", "CODEX_SPEC", "@openai/codex"), } +CODEX_MODEL_CATALOG_PATH = "/trajectories_mount/codex_model_catalog.json" +CODEX_MODEL_CATALOG = { + "models": [ + { + "slug": "gym-policy-model", + "display_name": "Gym policy model", + "description": "Model selected by the configured Gym policy server.", + "supported_reasoning_levels": [], + "shell_type": "default", + "visibility": "none", + "supported_in_api": True, + "priority": 99, + "availability_nux": None, + "upgrade": None, + "base_instructions": "", + "supports_reasoning_summaries": True, + "support_verbosity": False, + "default_verbosity": None, + "apply_patch_tool_type": None, + "truncation_policy": {"mode": "bytes", "limit": 10_000}, + "supports_parallel_tool_calls": False, + "context_window": 272_000, + "max_context_window": 272_000, + "experimental_supported_tools": [], + } + ] +} GENERIC_HARNESS_PREAMBLE = """\ You are an AI agent running in an automated Legal Agent Bench evaluation. @@ -899,11 +926,18 @@ def _write_runner_config( model_url: str, ) -> None: (paths["runtime"] / "agent_runner.py").write_text(_RUNNER_SOURCE) + agent_kwargs = dict(self.config.agent_kwargs) + if agent_key(self.config.agent_server_module) == "codex_agent": + extra_config = dict(agent_kwargs.get("extra_config") or {}) + if "model_catalog_json" not in extra_config: + (paths["runtime"] / "codex_model_catalog.json").write_text(json.dumps(CODEX_MODEL_CATALOG, indent=2)) + extra_config["model_catalog_json"] = CODEX_MODEL_CATALOG_PATH + agent_kwargs["extra_config"] = extra_config runner = { "agent_server_module": self.config.agent_server_module, "agent_server_class": self.config.agent_server_class, "agent_config_class": self.config.agent_config_class, - "agent_kwargs": self.config.agent_kwargs, + "agent_kwargs": agent_kwargs, "model_url": model_url, "model_connect_timeout_seconds": self.config.model_connect_timeout_seconds, "disable_endpoint_metadata_probe": agent_key(self.config.agent_server_module) == "hermes_agent", diff --git a/responses_api_agents/legal_agent_bench_agent/tests/test_app.py b/responses_api_agents/legal_agent_bench_agent/tests/test_app.py index d3ca27feeb..0c1588f58c 100644 --- a/responses_api_agents/legal_agent_bench_agent/tests/test_app.py +++ b/responses_api_agents/legal_agent_bench_agent/tests/test_app.py @@ -487,6 +487,49 @@ def test_runner_config_keeps_endpoint_metadata_for_non_hermes_harnesses(tmp_path assert payload["disable_endpoint_metadata_probe"] is False +def test_runner_config_stages_default_codex_model_catalog(tmp_path) -> None: + runner = app.LegalAgentBenchAgent.model_construct( + config=_config( + agent_server_module="responses_api_agents.codex_agent.app", + agent_server_class="CodexAgent", + agent_config_class="CodexAgentConfig", + ) + ) + + runner._write_runner_config( + {"runtime": tmp_path}, + NeMoGymResponseCreateParamsNonStreaming(input="Do the work"), + "http://model.internal:8000", + ) + + payload = json.loads((tmp_path / "runner.json").read_text()) + assert payload["agent_kwargs"]["extra_config"]["model_catalog_json"] == app.CODEX_MODEL_CATALOG_PATH + catalog = json.loads((tmp_path / "codex_model_catalog.json").read_text()) + assert catalog["models"][0]["slug"] == "gym-policy-model" + assert catalog["models"][0]["base_instructions"] == "" + + +def test_runner_config_preserves_explicit_codex_model_catalog(tmp_path) -> None: + runner = app.LegalAgentBenchAgent.model_construct( + config=_config( + agent_server_module="responses_api_agents.codex_agent.app", + agent_server_class="CodexAgent", + agent_config_class="CodexAgentConfig", + agent_kwargs={"extra_config": {"model_catalog_json": "/custom/catalog.json"}}, + ) + ) + + runner._write_runner_config( + {"runtime": tmp_path}, + NeMoGymResponseCreateParamsNonStreaming(input="Do the work"), + "http://model.internal:8000", + ) + + payload = json.loads((tmp_path / "runner.json").read_text()) + assert payload["agent_kwargs"]["extra_config"]["model_catalog_json"] == "/custom/catalog.json" + assert not (tmp_path / "codex_model_catalog.json").exists() + + def test_stage_agent_source_copies_only_selected_runtime_package(monkeypatch, tmp_path) -> None: repository = tmp_path / "repository" package = repository / "responses_api_agents" / "hermes_agent" From bccbfcdb66eaa430238ef671e46df2df8d62c40b Mon Sep 17 00:00:00 2001 From: Ryan Stewart Date: Mon, 3 Aug 2026 15:09:03 -0400 Subject: [PATCH 09/26] add the Gym-native LAB runner and make it the default Signed-off-by: Ryan Stewart --- benchmarks/legal_agent_bench/README.md | 90 +++-- benchmarks/legal_agent_bench/config.yaml | 18 +- .../legal_agent_bench/config_harbor.yaml | 25 ++ .../legal_agent_bench/tests/test_prepare.py | 28 +- resources_servers/legal_agent_bench/README.md | 53 +-- .../legal_agent_bench_agent/README.md | 15 +- .../legal_agent_bench_agent/app.py | 73 +++- .../configs/legal_agent_bench_native.yaml | 43 +++ .../legal_agent_bench_agent/tests/test_app.py | 27 ++ .../legal_agent_bench_native_agent/README.md | 23 ++ .../__init__.py | 1 + .../legal_agent_bench_native_agent/app.py | 307 ++++++++++++++++ .../requirements.txt | 1 + .../legal_agent_bench_native_agent_deps.sh | 8 + .../tests/test_app.py | 338 ++++++++++++++++++ 15 files changed, 966 insertions(+), 84 deletions(-) create mode 100644 benchmarks/legal_agent_bench/config_harbor.yaml create mode 100644 responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_native.yaml create mode 100644 responses_api_agents/legal_agent_bench_native_agent/README.md create mode 100644 responses_api_agents/legal_agent_bench_native_agent/__init__.py create mode 100644 responses_api_agents/legal_agent_bench_native_agent/app.py create mode 100644 responses_api_agents/legal_agent_bench_native_agent/requirements.txt create mode 100755 responses_api_agents/legal_agent_bench_native_agent/scripts/legal_agent_bench_native_agent_deps.sh create mode 100644 responses_api_agents/legal_agent_bench_native_agent/tests/test_app.py diff --git a/benchmarks/legal_agent_bench/README.md b/benchmarks/legal_agent_bench/README.md index b8870e1719..a306bc8b7b 100644 --- a/benchmarks/legal_agent_bench/README.md +++ b/benchmarks/legal_agent_bench/README.md @@ -5,13 +5,15 @@ This benchmark registers the with Gym's benchmark catalog. Every variant evaluates the same 1,749 public Harvey LAB tasks with the same skills and verifier. -This benchmark uses the Harbor integration by default. Three additional variants use the LAB-owned -[configurable runner](../../responses_api_agents/legal_agent_bench_agent/README.md) -with Gym's built-in agent harnesses: +The unqualified benchmark uses a LAB-owned, Gym-native implementation of the +upstream LAB model/tool loop. Four explicit variants run the same tasks through +Harbor or one of Gym's built-in agent harnesses. All non-Harbor choices use the +[configurable runner](../../responses_api_agents/legal_agent_bench_agent/README.md). | Benchmark | Agent | | --- | --- | -| `legal_agent_bench` | Harbor (default) | +| `legal_agent_bench` | LAB Gym-native loop (default) | +| `legal_agent_bench/config_harbor` | Harbor compatibility integration | | `legal_agent_bench/config_hermes` | Hermes | | `legal_agent_bench/config_claude_code` | Claude Code | | `legal_agent_bench/config_codex` | Codex | @@ -81,17 +83,18 @@ From the repository root, run: gym eval prepare --benchmark legal_agent_bench ``` -Preparation is shared by all four variants. It validates or prepares the task +Preparation is shared by all five variants. It validates or prepares the task and skill caches, then writes the deterministic benchmark index to `benchmarks/legal_agent_bench/data/legal_agent_bench_benchmark.jsonl`. Repeated preparation reuses valid caches and does not download a second copy of LAB. -Validate that all four configurations resolve before spending time on a +Validate that all five configurations resolve before spending time on a rollout: ```bash gym env validate --model-type vllm_model --benchmark legal_agent_bench +gym env validate --model-type vllm_model --benchmark legal_agent_bench/config_harbor gym env validate --model-type vllm_model --benchmark legal_agent_bench/config_hermes gym env validate --model-type vllm_model --benchmark legal_agent_bench/config_claude_code gym env validate --model-type vllm_model --benchmark legal_agent_bench/config_codex @@ -99,19 +102,32 @@ gym env validate --model-type vllm_model --benchmark legal_agent_bench/config_co ## Test the various harnesses -Run these one at a time for each harness you want to test. Each command starts the required Gym services, runs -the first benchmark task, writes one JSONL result row, and stops the services. +Run these one at a time for each harness you want to test. Each command starts +the required Gym services, runs the first benchmark task, writes one JSONL +result row, and stops the services. Alongside the path passed to `--output`, Gym writes `_materialized_inputs.jsonl` and `_aggregate_metrics.json`. -Harbor: +Gym-native LAB loop: ```bash gym eval run \ --model-type vllm_model \ --benchmark legal_agent_bench \ --split benchmark \ + --output results/legal_agent_bench_native_smoke.jsonl \ + --concurrency 1 \ + --limit 1 +``` + +Harbor compatibility integration: + +```bash +gym eval run \ + --model-type vllm_model \ + --benchmark legal_agent_bench/config_harbor \ + --split benchmark \ --output results/legal_agent_bench_harbor_smoke.jsonl \ --concurrency 1 \ --limit 1 @@ -153,17 +169,20 @@ gym eval run \ --limit 1 ``` -Hermes uses chat completions. Claude Code and Codex use their respective CLI -adapters against the configured OpenAI-compatible policy endpoint. The first -run of each configurable harness provisions its pinned portable runtime, so it -can be much slower than a cached run. The LAB runner disables Hermes's optional -pricing and context-metadata lookups because Gym already supplies the model and -its internal policy proxy does not expose `/models`; real model-call access -logging is enabled. +The default native loop sends LAB's canonical function tools through Gym's +Responses API and executes them directly inside the task sandbox. Hermes uses +chat completions. Claude Code and Codex use their respective CLI adapters +against the configured OpenAI-compatible policy endpoint. The first run of +each non-Harbor harness provisions its portable runtime, so it can be much +slower than a cached run. The LAB runner disables Hermes's optional pricing and +context-metadata lookups because Gym already supplies the model and its +internal policy proxy does not expose `/models`; real model-call access logging +is enabled. -## Check a configurable result +## Check a native or configurable result -For Hermes, for example, inspect the result row and the persisted artifact bundle with: +For the default native loop, inspect the result row and persisted artifact +bundle with: ```bash jq '{ @@ -176,15 +195,16 @@ jq '{ judge_error_count, verifier_error, artifact_dir -}' results/legal_agent_bench_hermes_smoke.jsonl +}' results/legal_agent_bench_native_smoke.jsonl -ARTIFACT_DIR=$(jq -r '.artifact_dir' results/legal_agent_bench_hermes_smoke.jsonl) +ARTIFACT_DIR=$(jq -r '.artifact_dir' results/legal_agent_bench_native_smoke.jsonl) jq . "$ARTIFACT_DIR/run_summary.json" jq . "$ARTIFACT_DIR/agent/trajectory.json" open "$ARTIFACT_DIR/verifier/report.html" # macOS; use xdg-open on Linux ``` -Substitute the Claude Code or Codex output filename to inspect those runs. +Substitute the Hermes, Claude Code, or Codex output filename to inspect those +runs. A reliable completed rollout has `mask_sample: false`, no failure flags, `judge_error_count: 0`, and `verifier_error: 0`. Check `output_files` in `run_summary.json` for the deliverables. A `reward` of `0.0` can still be a @@ -192,8 +212,9 @@ valid run: the default `full_task` reward requires every rubric criterion to pass, while `criteria_pass_rate` shows partial success. Harbor keeps its detailed trial files under -`results/legal_agent_bench/harbor_jobs`; configurable variants use the -corresponding `hermes_jobs`, `claude_code_jobs`, or `codex_jobs` directory. +`results/legal_agent_bench/harbor_jobs`; the native and configurable variants +use the corresponding `native_jobs`, `hermes_jobs`, `claude_code_jobs`, or +`codex_jobs` directory. Their model directory comes from `policy_model_name`, with path separators and other unsafe characters normalized for the filesystem. A session directory is created only when its first rollout starts. Each configurable rollout is stored @@ -220,7 +241,8 @@ gym eval run \ +legal_agent_bench_benchmark_hermes_agent.responses_api_agents.legal_agent_bench_agent.concurrency=2 ``` -Use the corresponding `legal_agent_bench_benchmark_claude_code_agent` or +Use `legal_agent_bench_benchmark_native_agent` for the default native loop, or +the corresponding `legal_agent_bench_benchmark_claude_code_agent` or `legal_agent_bench_benchmark_codex_agent` prefix for those variants. The agent-server default stays at `1`; changing only `--concurrency` leaves the server-side semaphore serial. @@ -330,9 +352,15 @@ pass rate instead, add this override to `gym env start` or the one-shot ``` This changes only the reported reward; it does not change the tasks, agent, or -judge criteria. - -For configurable variants, the rollout also reports `agent_failed`, +judge criteria. The command above uses the native default's resource-server +prefix. For an explicit variant, replace +`legal_agent_bench_benchmark_resources_server` with the corresponding +`legal_agent_bench_benchmark_harbor_resources_server`, +`legal_agent_bench_benchmark_hermes_resources_server`, +`legal_agent_bench_benchmark_claude_code_resources_server`, or +`legal_agent_bench_benchmark_codex_resources_server` prefix. + +For the native and configurable variants, the rollout also reports `agent_failed`, `model_connection_failed`, `agent_timed_out`, `verifier_failed`, `verifier_timed_out`, `sandbox_failed`, `task_failed`, `configuration_failed`, `judge_error_count`, and `verifier_error`. @@ -346,9 +374,10 @@ task-loading, or judge failures set `mask_sample`; do not treat those zeroes as model-quality results. Configurable-runner artifacts are grouped by harness, model, dated session, and -task. The default roots are `results/legal_agent_bench/hermes_jobs`, +task. The default roots are `results/legal_agent_bench/native_jobs`, +`results/legal_agent_bench/hermes_jobs`, `results/legal_agent_bench/claude_code_jobs`, and -`results/legal_agent_bench/codex_jobs`. Harbor continues to use +`results/legal_agent_bench/codex_jobs`. Harbor uses `results/legal_agent_bench/harbor_jobs`. Each configurable output row includes `artifact_dir`, `run_summary_path`, @@ -368,7 +397,8 @@ Run the benchmark and resource-server tests with: uv run pytest -q \ benchmarks/legal_agent_bench/tests \ resources_servers/legal_agent_bench/tests \ - responses_api_agents/legal_agent_bench_agent/tests + responses_api_agents/legal_agent_bench_agent/tests \ + responses_api_agents/legal_agent_bench_native_agent/tests ``` Generated indexes, collation metrics, Harbor jobs, source documents, and skills diff --git a/benchmarks/legal_agent_bench/config.yaml b/benchmarks/legal_agent_bench/config.yaml index cadf64def6..ec8a4c8b32 100644 --- a/benchmarks/legal_agent_bench/config.yaml +++ b/benchmarks/legal_agent_bench/config.yaml @@ -2,20 +2,20 @@ # SPDX-License-Identifier: Apache-2.0 config_paths: - - resources_servers/legal_agent_bench/configs/legal_agent_bench.yaml + - responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_native.yaml legal_agent_bench_benchmark_resources_server: _inherit_from: legal_agent_bench -legal_agent_bench_benchmark_harbor_agent: - _inherit_from: legal_agent_bench_harbor_agent +legal_agent_bench_benchmark_native_agent: + _inherit_from: legal_agent_bench_native_agent responses_api_agents: - harbor_agent: - harbor_datasets: - legal_agent_bench: - local_dataset_path: ${legal_agent_bench_benchmark_resources_server.resources_servers.legal_agent_bench.harbor_tasks_dir} - harbor_agent_kwargs: - skills_dir: ${legal_agent_bench_benchmark_resources_server.resources_servers.legal_agent_bench.harness_skills_dir} + legal_agent_bench_agent: + resources_server: + type: resources_servers + name: legal_agent_bench_benchmark_resources_server + runtime_tasks_dir: ${legal_agent_bench_benchmark_resources_server.resources_servers.legal_agent_bench.harbor_tasks_dir} + skills_dir: ${legal_agent_bench_benchmark_resources_server.resources_servers.legal_agent_bench.harness_skills_dir} datasets: - name: legal_agent_bench type: benchmark diff --git a/benchmarks/legal_agent_bench/config_harbor.yaml b/benchmarks/legal_agent_bench/config_harbor.yaml new file mode 100644 index 0000000000..883659ebc8 --- /dev/null +++ b/benchmarks/legal_agent_bench/config_harbor.yaml @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +config_paths: + - resources_servers/legal_agent_bench/configs/legal_agent_bench.yaml + +legal_agent_bench_benchmark_harbor_resources_server: + _inherit_from: legal_agent_bench + +legal_agent_bench_benchmark_harbor_agent: + _inherit_from: legal_agent_bench_harbor_agent + responses_api_agents: + harbor_agent: + harbor_datasets: + legal_agent_bench: + local_dataset_path: ${legal_agent_bench_benchmark_harbor_resources_server.resources_servers.legal_agent_bench.harbor_tasks_dir} + harbor_agent_kwargs: + skills_dir: ${legal_agent_bench_benchmark_harbor_resources_server.resources_servers.legal_agent_bench.harness_skills_dir} + datasets: + - name: legal_agent_bench + type: benchmark + jsonl_fpath: benchmarks/legal_agent_bench/data/legal_agent_bench_benchmark.jsonl + prompt_config: null + prepare_script: benchmarks/legal_agent_bench/prepare.py + num_repeats: 1 diff --git a/benchmarks/legal_agent_bench/tests/test_prepare.py b/benchmarks/legal_agent_bench/tests/test_prepare.py index 391cd8386a..962c140096 100644 --- a/benchmarks/legal_agent_bench/tests/test_prepare.py +++ b/benchmarks/legal_agent_bench/tests/test_prepare.py @@ -120,7 +120,7 @@ def test_benchmark_config_is_isolated_and_resolves_shared_cache_paths() -> None: benchmark = BenchmarkConfig.from_config_path(CONFIG_FPATH, strict=False) assert benchmark is not None assert benchmark.name == "legal_agent_bench" - assert benchmark.agent_name == "legal_agent_bench_benchmark_harbor_agent" + assert benchmark.agent_name == "legal_agent_bench_benchmark_native_agent" assert benchmark.num_repeats == 1 assert benchmark.dataset.prompt_config is None assert benchmark.dataset.jsonl_fpath == Path("benchmarks/legal_agent_bench/data/legal_agent_bench_benchmark.jsonl") @@ -132,14 +132,35 @@ def test_benchmark_config_is_isolated_and_resolves_shared_cache_paths() -> None: ) resolved = GlobalConfigDictParser().parse_no_environment(initial_global_config_dict=initial_config) assert "legal_agent_bench" not in resolved - assert "legal_agent_bench_harbor_agent" not in resolved + assert "legal_agent_bench_native_agent" not in resolved resource = resolved.legal_agent_bench_benchmark_resources_server.resources_servers.legal_agent_bench + agent = resolved.legal_agent_bench_benchmark_native_agent.responses_api_agents.legal_agent_bench_agent + assert agent.agent_server_module == "responses_api_agents.legal_agent_bench_native_agent.app" + assert agent.runtime_tasks_dir == resource.harbor_tasks_dir + assert agent.skills_dir == resource.harness_skills_dir + assert agent.agent_kwargs.max_turns == 60 + assert len(agent.datasets) == 1 + assert agent.datasets[0].type == "benchmark" + + +def test_harbor_compatibility_variant_resolves() -> None: + config_path = BENCHMARK_DIR / "config_harbor.yaml" + benchmark = BenchmarkConfig.from_config_path(config_path, strict=False) + assert benchmark is not None + assert benchmark.name == "legal_agent_bench" + assert benchmark.agent_name == "legal_agent_bench_benchmark_harbor_agent" + + initial_config = OmegaConf.merge( + OmegaConf.load(config_path), + GlobalConfigDictParserConfig.NO_MODEL_GLOBAL_CONFIG_DICT, + ) + resolved = GlobalConfigDictParser().parse_no_environment(initial_global_config_dict=initial_config) + resource = resolved.legal_agent_bench_benchmark_harbor_resources_server.resources_servers.legal_agent_bench agent = resolved.legal_agent_bench_benchmark_harbor_agent.responses_api_agents.harbor_agent assert agent.harbor_datasets.legal_agent_bench.local_dataset_path == resource.harbor_tasks_dir assert agent.harbor_agent_kwargs.skills_dir == resource.harness_skills_dir assert agent.harbor_agent_kwargs.max_turns == 60 - assert len(agent.datasets) == 1 assert agent.datasets[0].type == "benchmark" @@ -189,6 +210,7 @@ def test_configurable_benchmark_variants_resolve(filename, expected_agent, expec @pytest.mark.parametrize( "agent_name", [ + "legal_agent_bench_benchmark_native_agent", "legal_agent_bench_benchmark_harbor_agent", "legal_agent_bench_benchmark_hermes_agent", "legal_agent_bench_benchmark_claude_code_agent", diff --git a/resources_servers/legal_agent_bench/README.md b/resources_servers/legal_agent_bench/README.md index 33a73b80ed..b58e9dbb9d 100644 --- a/resources_servers/legal_agent_bench/README.md +++ b/resources_servers/legal_agent_bench/README.md @@ -2,22 +2,25 @@ This resource server runs the public Harvey [Legal Agent Benchmark (LAB)](https://github.com/harveyai/harvey-labs/tree/f46ef86e4788545622db25dcffa3aebb7a139929) -through NeMo Gym. The compatibility configuration uses Harbor, and optional -configurations run Gym's Hermes, Claude Code, or Codex harness through a -LAB-owned Docker runner. The integration is pinned to upstream commit +through NeMo Gym. The benchmark default is a direct Gym-native implementation +of LAB's model/tool loop. Compatibility configurations run Harbor or Gym's +Hermes, Claude Code, and Codex harnesses through the same LAB-owned sandbox +runner. The integration is pinned to upstream commit `f46ef86e4788545622db25dcffa3aebb7a139929`: 1,749 tasks and the public `docx`, `pptx`, and `xlsx` skills. NeMo Gym schedules rollouts, the selected agent works with each task's -documents inside Docker, and the task-local verifier scores every rubric -criterion with an OpenAI-compatible judge model. The default configuration uses -the Harbor agent harness, but it can easily be overriden through configuration -settings. +documents inside a sandbox, and the task-local verifier scores every rubric +criterion with an OpenAI-compatible judge model. `--benchmark legal_agent_bench` +selects the native loop; Harbor is available explicitly as +`legal_agent_bench/config_harbor`. ## Requirements - Python 3.13.13 and [uv](https://docs.astral.sh/uv/) -- Docker with a running daemon (the only supported container backend) +- Docker with a running daemon for local execution and portable runtime + provisioning. ECS Fargate execution is also supported with an immutable + registry image; see the benchmark README. - An OpenAI-compatible policy endpoint and judge endpoint - At least 10 GB of free working space for preparation and the first Docker build @@ -129,13 +132,14 @@ gym env test --resources-server legal_agent_bench ## Run and smoke test -For the shortest copy-paste path that prepares and tests Harbor, Hermes, Claude -Code, and Codex as benchmark variants, use the -[benchmark README](../../benchmarks/legal_agent_bench/README.md#smoke-test-every-harness). +For the shortest copy-paste path that prepares and tests the native loop, +Harbor, Hermes, Claude Code, and Codex as benchmark variants, use the +[benchmark README](../../benchmarks/legal_agent_bench/README.md#test-the-various-harnesses). The commands below describe the lower-level resource-server and standalone agent workflow. -The default configuration starts the Harbor agent: +The resource-server discovery config predates the benchmark wrapper and starts +the Harbor compatibility agent: ```bash gym env start \ @@ -165,6 +169,7 @@ configurations and agent names are: | Config path | Direct-run agent | | --- | --- | | `resources_servers/legal_agent_bench/configs/legal_agent_bench.yaml` | `legal_agent_bench_harbor_agent` | +| `responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_native.yaml` | `legal_agent_bench_native_agent` | | `responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_hermes.yaml` | `legal_agent_bench_hermes_agent` | | `responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_claude_code.yaml` | `legal_agent_bench_claude_code_agent` | | `responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_codex.yaml` | `legal_agent_bench_codex_agent` | @@ -173,26 +178,27 @@ The configurable files include `resources_servers/legal_agent_bench/configs/resources_only.yaml`, which starts the LAB resource server without also launching the compatibility Harbor agent. -For example, start and run Hermes with: +For example, start and run the Gym-native loop with: ```bash gym env start \ - --config responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_hermes.yaml \ + --config responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_native.yaml \ --model-type vllm_model gym eval run --no-serve \ - --config responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_hermes.yaml \ - --agent legal_agent_bench_hermes_agent \ + --config responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_native.yaml \ + --agent legal_agent_bench_native_agent \ --input resources_servers/legal_agent_bench/data/example.jsonl \ - --output results/legal_agent_bench_hermes_smoke.jsonl \ + --output results/legal_agent_bench_native_smoke.jsonl \ --concurrency 1 \ --limit 1 ``` -Use the corresponding config and agent name for Claude Code or Codex. All -three harnesses use the configured Gym policy endpoint. That endpoint must -support the selected harness's protocol. Their CLI/runtime dependencies are -pinned and provisioned into a portable cache on the first rollout. +Use the corresponding config and agent name for Hermes, Claude Code, or Codex. +All four non-Harbor choices use the configured Gym policy endpoint. That +endpoint must support the selected harness's protocol. Their runtime +dependencies are provisioned into a portable cache on the first rollout; CLI +dependencies are pinned where applicable. The default `full_task` reward is LAB's official all-criteria score: a task earns `1.0` only when every criterion passes. For diagnostic partial credit, @@ -231,6 +237,7 @@ The default paths are: - Credential-bearing runtime tasks: `data/runtime/harbor_tasks/legal_agent_bench` - Harbor jobs: `results/legal_agent_bench/harbor_jobs` - Configurable agent runtimes: `responses_api_agents/legal_agent_bench_agent/.deps` +- Gym-native rollout artifacts: `results/legal_agent_bench/native_jobs` - Hermes rollout artifacts: `results/legal_agent_bench/hermes_jobs` - Claude Code rollout artifacts: `results/legal_agent_bench/claude_code_jobs` - Codex rollout artifacts: `results/legal_agent_bench/codex_jobs` @@ -264,10 +271,10 @@ sees only its selected Gym package and public task inputs, while the verifier starts after that container is destroyed and receives the completed LAB run read-only. -To inspect a configurable smoke run: +To inspect a native or configurable smoke run: ```bash -ARTIFACT_DIR=$(jq -r '.artifact_dir' results/legal_agent_bench_hermes_smoke.jsonl) +ARTIFACT_DIR=$(jq -r '.artifact_dir' results/legal_agent_bench_native_smoke.jsonl) jq . "$ARTIFACT_DIR/run_summary.json" jq . "$ARTIFACT_DIR/agent/trajectory.json" open "$ARTIFACT_DIR/verifier/report.html" # macOS; use xdg-open on Linux diff --git a/responses_api_agents/legal_agent_bench_agent/README.md b/responses_api_agents/legal_agent_bench_agent/README.md index 09eac4447b..52dc8cbf31 100644 --- a/responses_api_agents/legal_agent_bench_agent/README.md +++ b/responses_api_agents/legal_agent_bench_agent/README.md @@ -4,8 +4,8 @@ This task-driven Gym agent runs a configured Gym Responses API agent inside a Legal Agent Bench (LAB) sandbox. It keeps the benchmark task set, skills, and verifier fixed while allowing the agent harness and sandbox provider to vary. -The committed configurations select Gym's Hermes, Claude Code, and Codex agents -through: +The default configuration selects LAB's direct Gym-native loop. Additional +configurations select Gym's Hermes, Claude Code, and Codex agents through: - `agent_server_module` - `agent_server_class` @@ -38,7 +38,10 @@ Agent source, documents, skills, runtime, and runner inputs are transferred through Gym's sandbox file API. The sandbox has no writable host mounts. Only the task instructions and public skill manuals are supplied to the -configured agent. Its results are downloaded to a private temporary directory, +configured agent. The native choice reproduces upstream LAB's model → function +tools → tool results loop using Gym's Responses API; it uses LAB's canonical +`bash`, `read`, `write`, `write_docx`, `edit`, `glob`, and `grep` tools. Its +results are downloaded to a private temporary directory, the sandbox is destroyed, and links, devices, traversal paths, and other unsafe archive entries are rejected before Gym creates host artifacts. A fresh verifier-only sandbox then receives a separately staged, sanitized `lab-run` @@ -64,7 +67,9 @@ and actual model-call access logging is enabled. Artifacts are written below `results/legal_agent_bench/_jobs///_/`. -For example, Hermes uses `results/legal_agent_bench/hermes_jobs`. The model +For example, the default native loop uses +`results/legal_agent_bench/native_jobs`, while Hermes uses +`results/legal_agent_bench/hermes_jobs`. The model segment is the configured `policy_model_name`, normalized into one safe path segment. The task name is also normalized, and the run ID is an eight-character unique suffix. A session directory is created only when its first rollout starts. @@ -74,7 +79,7 @@ top-level `run_summary.json`. The rollout response also includes direct paths for the summary, trajectory, stdout, stderr, output directory, and verifier report. -For setup, four copy-paste smoke commands, and result inspection, see the +For setup, five copy-paste smoke commands, and result inspection, see the [benchmark README](../../benchmarks/legal_agent_bench/README.md#test-the-various-harnesses). Run `gym` from an activated repository environment rather than prefixing server-starting commands with `uv run`. diff --git a/responses_api_agents/legal_agent_bench_agent/app.py b/responses_api_agents/legal_agent_bench_agent/app.py index 365b22f7a8..88e9d5bfef 100644 --- a/responses_api_agents/legal_agent_bench_agent/app.py +++ b/responses_api_agents/legal_agent_bench_agent/app.py @@ -46,12 +46,19 @@ resolve_repo_path, validate_harness_skills, ) +from resources_servers.legal_agent_bench.vendor.harvey_labs.lab_harbor.tools import ( + get_all_tool_definitions, +) PACKAGE_DIR = Path(__file__).resolve().parent PORTABLE_PYTHON_SH = PACKAGE_DIR / "setup_scripts" / "_portable_python.sh" DATASET_ALIAS = "legal_agent_bench" INITIAL_USER_PROMPT = "Please begin working on the task described in the system prompt." +NATIVE_AGENT_MODULE = "responses_api_agents.legal_agent_bench_native_agent.app" +LAB_SYSTEM_PROMPT = ( + PARENT_DIR / "resources_servers" / "legal_agent_bench" / "vendor" / "harvey_labs" / "harness" / "system-prompt.md" +).read_text(encoding="utf-8") AGENT_CLI_PINS = { "claude_code_agent": ("claude_code_version", "CLAUDE_SPEC", "@anthropic-ai/claude-code"), "codex_agent": ("codex_version", "CODEX_SPEC", "@openai/codex"), @@ -119,6 +126,10 @@ os.environ["WORKSPACE_DIR"] = "/workspace/workspace" os.environ["SKILLS_DIR"] = "/workspace/skills" os.environ["HOME"] = "/workspace/workspace" +# The inner runner receives its complete configuration below. Prevent Gym's +# shared aiohttp client from invoking Hydra's CLI config loader in the output +# directory when the native agent makes its first model request. +os.environ.setdefault("NEMO_GYM_CONFIG_DICT", "{}") os.chdir("/workspace/output") from nemo_gym.config_types import ModelServerRef, ResourcesServerRef @@ -173,7 +184,15 @@ def empty_endpoint_model_metadata(*args, **kwargs): hermes_model_metadata.fetch_endpoint_model_metadata = empty_endpoint_model_metadata hermes_usage_pricing.fetch_endpoint_model_metadata = empty_endpoint_model_metadata - client = ServerClient.model_construct(global_config_dict={}) + client = ServerClient.model_construct( + global_config_dict={ + "policy_model": { + "responses_api_models": { + "policy_model": {}, + } + } + } + ) client._build_server_base_url = lambda _cfg: model_url base = { @@ -383,7 +402,8 @@ def _results_session_dir( timestamp: float, session_id: str, ) -> Path: - harness = agent_key(agent_server_module).removesuffix("_agent") + key = agent_key(agent_server_module) + harness = "native" if key == "legal_agent_bench_native_agent" else key.removesuffix("_agent") date = time.strftime("%Y%m%d", time.localtime(timestamp)) clock = time.strftime("%H%M%S", time.localtime(timestamp)) model = _results_segment(model_name, fallback="unknown_model") @@ -668,10 +688,26 @@ def _load_skill_prompt(skills_dir: Path) -> str: return "".join(sections) +def native_tool_definitions() -> list[dict[str, Any]]: + """Translate LAB's canonical tools to OpenAI Responses function tools.""" + return [ + { + "type": "function", + "name": tool["name"], + "description": tool["description"], + "parameters": tool["parameters"], + "strict": False, + } + for tool in get_all_tool_definitions() + ] + + def compose_agent_input( task_dir: Path, skills_dir: Path, params: NeMoGymResponseCreateParamsNonStreaming, + *, + native: bool = False, ) -> NeMoGymResponseCreateParamsNonStreaming: try: task = json.loads((task_dir / "task.json").read_text(encoding="utf-8")) @@ -679,17 +715,23 @@ def compose_agent_input( instructions = task["instructions"] except (OSError, KeyError, TypeError, json.JSONDecodeError) as exc: raise LegalAgentBenchTaskError(f"Invalid LAB task configuration in {task_dir}: {exc}") from exc - system_prompt = ( - GENERIC_HARNESS_PREAMBLE + _load_skill_prompt(skills_dir) + "\n\n## Task\n\n" + f"# {title}\n\n{instructions}" - ) - return params.model_copy( - update={ - "input": [ - {"role": "system", "content": system_prompt}, - {"role": "user", "content": INITIAL_USER_PROMPT}, - ] - } - ) + preamble = LAB_SYSTEM_PROMPT if native else GENERIC_HARNESS_PREAMBLE + system_prompt = preamble + _load_skill_prompt(skills_dir) + "\n\n## Task\n\n" + f"# {title}\n\n{instructions}" + update: dict[str, Any] = { + "input": [ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": INITIAL_USER_PROMPT}, + ] + } + if native: + update.update( + { + "tools": native_tool_definitions(), + "tool_choice": "auto", + "parallel_tool_calls": False, + } + ) + return params.model_copy(update=update) def _task_toml(task_dir: Path) -> dict[str, Any]: @@ -1342,7 +1384,10 @@ async def run(self, request: Request, body: LegalAgentBenchRunRequest) -> LegalA response = _empty_response(model_name) task_dir = resolve_task_dir(self.config.runtime_tasks_dir, body.instance_id) skills_dir = resolve_repo_path(self.config.skills_dir) - params = compose_agent_input(task_dir, skills_dir, params) + if self.config.agent_server_module == NATIVE_AGENT_MODULE: + params = compose_agent_input(task_dir, skills_dir, params, native=True) + else: + params = compose_agent_input(task_dir, skills_dir, params) final_root = self._run_root(task_dir.name) staging_temp = tempfile.TemporaryDirectory(prefix="legal-agent-bench-stage-") staged_paths = self._paths_for_root(Path(staging_temp.name), create=True) diff --git a/responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_native.yaml b/responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_native.yaml new file mode 100644 index 0000000000..94cb62e3a5 --- /dev/null +++ b/responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_native.yaml @@ -0,0 +1,43 @@ +config_paths: + - resources_servers/legal_agent_bench/configs/resources_only.yaml + +legal_agent_bench_native_agent: + responses_api_agents: + legal_agent_bench_agent: + entrypoint: app.py + domain: agent + description: Legal Agent Bench with its direct Gym-native tool loop. + value: Evaluate legal-document agents without an external agent harness. + resources_server: + type: resources_servers + name: legal_agent_bench + model_server: + type: responses_api_models + name: policy_model + runtime_tasks_dir: ${legal_agent_bench.resources_servers.legal_agent_bench.harbor_tasks_dir} + skills_dir: ${legal_agent_bench.resources_servers.legal_agent_bench.harness_skills_dir} + concurrency: 1 + agent_timeout_seconds: 10800 + verifier_timeout_seconds: 3600 + sandbox_provider: + docker: {} + sandbox_image: null + runtime_docker_platform: null + sandbox_model_base_url: null + agent_server_module: responses_api_agents.legal_agent_bench_native_agent.app + agent_server_class: LegalAgentBenchNativeAgent + agent_config_class: LegalAgentBenchNativeAgentConfig + agent_kwargs: + max_turns: 60 + shell_timeout: 60 + model_timeout_seconds: 1800 + max_output_chars: 16384 + datasets: + - name: example + type: example + jsonl_fpath: resources_servers/legal_agent_bench/data/example.jsonl + license: MIT + - name: legal_agent_bench + type: validation + jsonl_fpath: resources_servers/legal_agent_bench/data/generated/all.jsonl + license: MIT diff --git a/responses_api_agents/legal_agent_bench_agent/tests/test_app.py b/responses_api_agents/legal_agent_bench_agent/tests/test_app.py index 0c1588f58c..3bfb94e7fd 100644 --- a/responses_api_agents/legal_agent_bench_agent/tests/test_app.py +++ b/responses_api_agents/legal_agent_bench_agent/tests/test_app.py @@ -112,6 +112,7 @@ def _successful_response(text: str = "Done") -> app.NeMoGymResponse: @pytest.mark.parametrize( ("module", "expected"), [ + ("responses_api_agents.legal_agent_bench_native_agent.app", "legal_agent_bench_native_agent"), ("responses_api_agents.hermes_agent.app", "hermes_agent"), ("responses_api_agents.claude_code_agent.app", "claude_code_agent"), ("responses_api_agents.codex_agent.app", "codex_agent"), @@ -129,6 +130,7 @@ def test_agent_key_rejects_invalid_modules(module) -> None: def test_all_supported_agents_have_dependency_scripts() -> None: for module in ( + "responses_api_agents.legal_agent_bench_native_agent.app", "responses_api_agents.hermes_agent.app", "responses_api_agents.claude_code_agent.app", "responses_api_agents.codex_agent.app", @@ -324,6 +326,7 @@ def test_config_defaults_are_docker_and_single_concurrency() -> None: @pytest.mark.parametrize( ("module", "job_dir"), [ + ("responses_api_agents.legal_agent_bench_native_agent.app", "native_jobs"), ("responses_api_agents.hermes_agent.app", "hermes_jobs"), ("responses_api_agents.claude_code_agent.app", "claude_code_jobs"), ("responses_api_agents.codex_agent.app", "codex_jobs"), @@ -417,6 +420,28 @@ def test_compose_agent_input_uses_task_and_skills_without_rubric(monkeypatch, tm assert result.input[1]["content"] == app.INITIAL_USER_PROMPT +def test_native_agent_input_uses_upstream_prompt_and_canonical_tools(monkeypatch, tmp_path) -> None: + _root, task = _task_tree(tmp_path) + skills = _skills(tmp_path) + monkeypatch.setattr(app, "validate_harness_skills", lambda path: Path(path)) + + result = app.compose_agent_input( + task, + skills, + NeMoGymResponseCreateParamsNonStreaming(input=[]), + native=True, + ) + + system = result.input[0]["content"] + assert system.startswith(app.LAB_SYSTEM_PROMPT) + assert "Write a memo." in system + assert '"criteria"' not in system + assert '"id": "1"' not in system + assert [tool["name"] for tool in result.tools] == ["bash", "read", "write", "write_docx", "edit", "glob", "grep"] + assert all(tool["type"] == "function" and tool["strict"] is False for tool in result.tools) + assert result.parallel_tool_calls is False + + def test_verifier_credentials_are_read_separately_from_sandbox_resources(tmp_path) -> None: _root, task = _task_tree(tmp_path) assert app._verifier_env(task) == { @@ -466,6 +491,8 @@ def test_runner_config_preserves_dynamic_agent_configuration(tmp_path) -> None: assert "socket.create_connection" in (tmp_path / "agent_runner.py").read_text() assert 'os.environ.get("LAB_POLICY_MODEL_URL", runner["model_url"])' in (tmp_path / "agent_runner.py").read_text() assert "runner_status.json" in (tmp_path / "agent_runner.py").read_text() + assert '"responses_api_models"' in (tmp_path / "agent_runner.py").read_text() + assert 'os.environ.setdefault("NEMO_GYM_CONFIG_DICT", "{}")' in (tmp_path / "agent_runner.py").read_text() assert "hermes_model_metadata.fetch_endpoint_model_metadata" in (tmp_path / "agent_runner.py").read_text() assert "hermes_usage_pricing.fetch_endpoint_model_metadata" in (tmp_path / "agent_runner.py").read_text() diff --git a/responses_api_agents/legal_agent_bench_native_agent/README.md b/responses_api_agents/legal_agent_bench_native_agent/README.md new file mode 100644 index 0000000000..c4c97b5a6f --- /dev/null +++ b/responses_api_agents/legal_agent_bench_native_agent/README.md @@ -0,0 +1,23 @@ +# Legal Agent Bench Native Agent + +This agent is the direct Gym-native implementation of Harvey LAB's model/tool +loop. It runs only as the inner agent of the hardened +[`legal_agent_bench_agent`](../legal_agent_bench_agent/README.md) sandbox +runner. + +For each turn it sends the accumulated Responses API trajectory and LAB's +canonical function-tool definitions to the configured Gym policy model. It +executes returned `bash`, `read`, `write`, `write_docx`, `edit`, `glob`, and +`grep` calls inside the task sandbox, appends their results, and continues until +the model returns a final assistant message or reaches the configured turn +limit. + +The outer runner owns task resolution, prompt and skill composition, sandbox +staging, result collection, verifier isolation, and artifact publication. A +model error, timeout, repeated empty response, failed runtime preflight, or +turn-limit exhaustion returns an explicit failed response while preserving any +partial trajectory. The outer runner masks that sample and skips verification. + +Use `--benchmark legal_agent_bench` for this default. See the +[benchmark README](../../benchmarks/legal_agent_bench/README.md) for setup, +smoke-test, and artifact-inspection commands. diff --git a/responses_api_agents/legal_agent_bench_native_agent/__init__.py b/responses_api_agents/legal_agent_bench_native_agent/__init__.py new file mode 100644 index 0000000000..42458d2190 --- /dev/null +++ b/responses_api_agents/legal_agent_bench_native_agent/__init__.py @@ -0,0 +1 @@ +"""Gym-native agent loop for Legal Agent Bench.""" diff --git a/responses_api_agents/legal_agent_bench_native_agent/app.py b/responses_api_agents/legal_agent_bench_native_agent/app.py new file mode 100644 index 0000000000..09c5d84be1 --- /dev/null +++ b/responses_api_agents/legal_agent_bench_native_agent/app.py @@ -0,0 +1,307 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Gym-native implementation of the Legal Agent Bench tool loop.""" + +from __future__ import annotations + +import asyncio +import json +import os +import signal +from pathlib import Path +from typing import Any, Optional +from uuid import uuid4 + +from fastapi import Body, Request +from pydantic import ConfigDict, Field + +from nemo_gym.base_resources_server import BaseRunRequest +from nemo_gym.base_responses_api_agent import BaseResponsesAPIAgentConfig, SimpleResponsesAPIAgent +from nemo_gym.config_types import ModelServerRef, ResourcesServerRef +from nemo_gym.openai_utils import ( + NeMoGymEasyInputMessage, + NeMoGymFunctionCallOutput, + NeMoGymResponse, + NeMoGymResponseCreateParamsNonStreaming, + NeMoGymResponseFunctionToolCall, + NeMoGymResponseOutputMessage, +) +from nemo_gym.server_utils import get_response_json, raise_for_status + + +INITIAL_EMPTY_RESPONSE_NUDGE = ( + "Your last response was empty and did not call any tools. Continue the task. " + "Use the available tools to inspect the documents and write the required deliverables." +) +CONTAINER_TOOL_RUNNER = Path("/opt/legal-agent-bench/container_tool_runner.py") +SUPPORTED_TOOLS = frozenset({"bash", "read", "write", "write_docx", "edit", "glob", "grep"}) + + +class LegalAgentBenchNativeAgentConfig(BaseResponsesAPIAgentConfig): + resources_server: ResourcesServerRef + model_server: ModelServerRef + max_turns: int = Field(default=60, ge=1) + shell_timeout: int = Field(default=60, ge=1) + model_timeout_seconds: int = Field(default=1800, ge=1) + max_output_chars: int = Field(default=16_384, ge=1) + + +class LegalAgentBenchNativeRunRequest(BaseRunRequest): + model_config = ConfigDict(extra="allow") + + +class LabToolExecutor: + """Execute the upstream LAB tools in the current task sandbox.""" + + def __init__(self, *, timeout_seconds: int, max_output_chars: int) -> None: + self.timeout_seconds = timeout_seconds + self.max_output_chars = max_output_chars + + async def preflight(self) -> None: + result = await self.execute("preflight", {}) + if result.startswith("Error:"): + raise RuntimeError(result) + + async def execute(self, name: str, arguments: str | dict[str, Any]) -> str: + if name != "preflight" and name not in SUPPORTED_TOOLS: + return f"Error: unknown tool: {name}" + if isinstance(arguments, str): + try: + parsed = json.loads(arguments) + except json.JSONDecodeError as exc: + return f"Error: invalid JSON arguments for {name}: {exc}" + else: + parsed = arguments + if not isinstance(parsed, dict): + return f"Error: arguments for {name} must be a JSON object" + + if name == "bash": + command = parsed.get("command") + if not isinstance(command, str) or not command: + return "Error: command is required" + result = await self._run(["/bin/bash", "-lc", command]) + else: + result = await self._run(["/usr/local/bin/python", str(CONTAINER_TOOL_RUNNER), name, json.dumps(parsed)]) + full_read = name == "read" and "limit" in parsed and parsed.get("limit") in {0, None} + return result if full_read else self._truncate(result) + + async def _run(self, command: list[str]) -> str: + process = await asyncio.create_subprocess_exec( + *command, + cwd="/workspace/output", + env=os.environ.copy(), + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + start_new_session=True, + ) + try: + stdout, stderr = await asyncio.wait_for(process.communicate(), timeout=self.timeout_seconds) + except asyncio.TimeoutError: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + await process.communicate() + return f"Error: tool timed out after {self.timeout_seconds}s" + + output = stdout.decode(errors="replace") + error = stderr.decode(errors="replace") + command_name = Path(command[1]).name if len(command) > 1 else "tool" + if command[0] == "/usr/local/bin/python": + try: + payload = json.loads(next(line for line in reversed(output.splitlines()) if line.strip())) + except (StopIteration, json.JSONDecodeError): + payload = None + if isinstance(payload, dict) and "result" in payload: + output = str(payload["result"]) + if process.returncode: + detail = error or output or f"{command_name} exited with code {process.returncode}" + return f"Error: {detail.strip()}" + if error: + output = f"{output}\nSTDERR:\n{error}" if output else error + return output or "(no output)" + + def _truncate(self, value: str) -> str: + if len(value) <= self.max_output_chars: + return value + return value[: self.max_output_chars] + "\n[output truncated]" + + +def _output_text(messages: list[NeMoGymResponseOutputMessage]) -> str: + parts: list[str] = [] + for message in messages: + for content in message.content: + text = getattr(content, "text", None) + if text: + parts.append(str(text)) + return "\n".join(parts).strip() + + +def _merge_usage(total: Any, current: Any) -> Any: + if current is None: + return total + if total is None: + return current.model_copy(deep=True) + total.input_tokens += current.input_tokens + total.output_tokens += current.output_tokens + total.total_tokens += current.total_tokens + total.input_tokens_details.cached_tokens += current.input_tokens_details.cached_tokens + total.output_tokens_details.reasoning_tokens += current.output_tokens_details.reasoning_tokens + return total + + +def _failed_response( + *, + body: NeMoGymResponseCreateParamsNonStreaming, + response: Optional[NeMoGymResponse], + output: list[Any], + usage: Any, + message: str, +) -> NeMoGymResponse: + base = response or NeMoGymResponse( + id=f"resp_{uuid4().hex}", + created_at=0, + model=body.model or "policy_model", + object="response", + output=[], + parallel_tool_calls=body.parallel_tool_calls, + tool_choice=body.tool_choice, + tools=body.tools, + ) + return NeMoGymResponse.model_validate( + base.model_dump(mode="json") + | { + "status": "failed", + "error": {"code": "server_error", "message": message[-2000:]}, + "output": output, + "usage": usage, + } + ) + + +class LegalAgentBenchNativeAgent(SimpleResponsesAPIAgent): + """Run LAB's canonical tool loop through Gym's Responses API model server.""" + + config: LegalAgentBenchNativeAgentConfig + + async def responses( + self, + request: Request, + body: NeMoGymResponseCreateParamsNonStreaming = Body(), + ) -> NeMoGymResponse: + body = body.model_copy(deep=True) + if isinstance(body.input, str): + body.input = [NeMoGymEasyInputMessage(role="user", content=body.input)] + + executor = LabToolExecutor( + timeout_seconds=self.config.shell_timeout, + max_output_chars=self.config.max_output_chars, + ) + try: + await executor.preflight() + except Exception as exc: + return _failed_response( + body=body, + response=None, + output=[], + usage=None, + message=f"LAB tool preflight failed: {type(exc).__name__}: {exc}", + ) + + trajectory: list[Any] = [] + usage = None + last_response: Optional[NeMoGymResponse] = None + model_cookies = None + empty_responses = 0 + + for _turn in range(self.config.max_turns): + model_input = body.model_copy(update={"input": list(body.input) + trajectory}) + try: + raw_response = await asyncio.wait_for( + self.server_client.post( + server_name=self.config.model_server.name, + url_path=self.url_path_for_request("/v1/responses", request), + json=model_input, + cookies=model_cookies, + ), + timeout=self.config.model_timeout_seconds, + ) + await raise_for_status(raw_response) + model_response = NeMoGymResponse.model_validate(await get_response_json(raw_response)) + model_cookies = raw_response.cookies + except Exception as exc: + return _failed_response( + body=body, + response=last_response, + output=trajectory, + usage=usage, + message=f"LAB model call failed: {type(exc).__name__}: {exc}", + ) + + last_response = model_response + usage = _merge_usage(usage, model_response.usage) + trajectory.extend(model_response.output) + if model_response.error is not None or model_response.incomplete_details is not None: + return model_response.model_copy(update={"output": trajectory, "usage": usage}) + + function_calls = [ + item for item in model_response.output if isinstance(item, NeMoGymResponseFunctionToolCall) + ] + assistant_messages = [ + item for item in model_response.output if isinstance(item, NeMoGymResponseOutputMessage) + ] + if not function_calls and assistant_messages: + if _output_text(assistant_messages): + return model_response.model_copy( + update={"status": "completed", "output": trajectory, "usage": usage} + ) + empty_responses += 1 + if empty_responses > 2: + return _failed_response( + body=body, + response=model_response, + output=trajectory, + usage=usage, + message="LAB model repeatedly returned empty responses", + ) + trajectory.append(NeMoGymEasyInputMessage(role="user", content=INITIAL_EMPTY_RESPONSE_NUDGE)) + continue + + if not function_calls and not model_response.output: + empty_responses += 1 + if empty_responses > 2: + return _failed_response( + body=body, + response=model_response, + output=trajectory, + usage=usage, + message="LAB model repeatedly returned empty responses", + ) + trajectory.append(NeMoGymEasyInputMessage(role="user", content=INITIAL_EMPTY_RESPONSE_NUDGE)) + continue + + empty_responses = 0 + for call in function_calls: + result = await executor.execute(call.name, call.arguments) + trajectory.append( + NeMoGymFunctionCallOutput( + type="function_call_output", + call_id=call.call_id, + output=result, + ) + ) + + return _failed_response( + body=body, + response=last_response, + output=trajectory, + usage=usage, + message=f"LAB agent reached the {self.config.max_turns}-turn limit", + ) + + async def run(self, request: Request, body: LegalAgentBenchNativeRunRequest): + raise NotImplementedError("The LAB-native agent runs inside the task-driven LAB sandbox") + + +if __name__ == "__main__": + LegalAgentBenchNativeAgent.run_webserver() diff --git a/responses_api_agents/legal_agent_bench_native_agent/requirements.txt b/responses_api_agents/legal_agent_bench_native_agent/requirements.txt new file mode 100644 index 0000000000..e16136d43f --- /dev/null +++ b/responses_api_agents/legal_agent_bench_native_agent/requirements.txt @@ -0,0 +1 @@ +# The LAB-native loop depends only on nemo_gym, installed by the setup script. diff --git a/responses_api_agents/legal_agent_bench_native_agent/scripts/legal_agent_bench_native_agent_deps.sh b/responses_api_agents/legal_agent_bench_native_agent/scripts/legal_agent_bench_native_agent_deps.sh new file mode 100755 index 0000000000..4476b84f9f --- /dev/null +++ b/responses_api_agents/legal_agent_bench_native_agent/scripts/legal_agent_bench_native_agent_deps.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -euo pipefail + +source "${PORTABLE_PYTHON_SH:?PORTABLE_PYTHON_SH is required}" + +mkdir -p "${DEPS_DIR:?DEPS_DIR is required}" +install_portable_python +install_nemo_gym_deps diff --git a/responses_api_agents/legal_agent_bench_native_agent/tests/test_app.py b/responses_api_agents/legal_agent_bench_native_agent/tests/test_app.py new file mode 100644 index 0000000000..ca3fb7d9ea --- /dev/null +++ b/responses_api_agents/legal_agent_bench_native_agent/tests/test_app.py @@ -0,0 +1,338 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Tests for the Gym-native Legal Agent Bench loop.""" + +from __future__ import annotations + +import asyncio +import json +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from nemo_gym.config_types import ModelServerRef, ResourcesServerRef +from nemo_gym.openai_utils import ( + NeMoGymFunctionCallOutput, + NeMoGymResponseCreateParamsNonStreaming, + NeMoGymResponseFunctionToolCall, + NeMoGymResponseOutputMessage, +) +from nemo_gym.server_utils import ServerClient +from responses_api_agents.legal_agent_bench_native_agent import app + + +_REAL_PREFLIGHT = app.LabToolExecutor.preflight + + +def _config(**overrides) -> app.LegalAgentBenchNativeAgentConfig: + values = { + "host": "0.0.0.0", + "port": 10000, + "name": "legal_agent_bench_native_agent", + "entrypoint": "app.py", + "resources_server": ResourcesServerRef(name="lab", type="resources_servers"), + "model_server": ModelServerRef(name="policy_model", type="responses_api_models"), + } + values.update(overrides) + return app.LegalAgentBenchNativeAgentConfig(**values) + + +def _model_response(output: list[dict], *, response_id: str = "response", usage: bool = True) -> dict: + payload = { + "id": response_id, + "created_at": 1, + "model": "policy", + "object": "response", + "output": output, + "parallel_tool_calls": False, + "tool_choice": "auto", + "tools": [], + } + if usage: + payload["usage"] = { + "input_tokens": 3, + "input_tokens_details": {"cached_tokens": 1}, + "output_tokens": 2, + "output_tokens_details": {"reasoning_tokens": 1}, + "total_tokens": 5, + } + return payload + + +def _function_call(name: str = "glob", arguments: str = '{"pattern":"**/*"}') -> dict: + return { + "id": "fc-1", + "call_id": "call-1", + "name": name, + "arguments": arguments, + "type": "function_call", + "status": "completed", + } + + +def _assistant_message(text: str = "Done") -> dict: + return { + "id": "message-1", + "content": [{"annotations": [], "text": text, "type": "output_text"}], + "role": "assistant", + "status": "completed", + "type": "message", + } + + +def _raw_response(payload: dict) -> SimpleNamespace: + return SimpleNamespace(ok=True, read=AsyncMock(return_value=json.dumps(payload).encode()), cookies={}) + + +def _process(*, stdout: bytes = b"", stderr: bytes = b"", returncode: int = 0) -> SimpleNamespace: + return SimpleNamespace( + pid=123, + returncode=returncode, + communicate=AsyncMock(return_value=(stdout, stderr)), + ) + + +def _agent(*, max_turns: int = 60) -> app.LegalAgentBenchNativeAgent: + client = MagicMock(spec=ServerClient) + return app.LegalAgentBenchNativeAgent(config=_config(max_turns=max_turns), server_client=client) + + +@pytest.fixture(autouse=True) +def _successful_preflight(monkeypatch) -> None: + monkeypatch.setattr(app.LabToolExecutor, "preflight", AsyncMock()) + + +async def test_tool_loop_returns_full_responses_trajectory_and_usage(monkeypatch) -> None: + agent = _agent() + agent.server_client.post = AsyncMock( + side_effect=[ + _raw_response(_model_response([_function_call()], response_id="tool-turn")), + _raw_response(_model_response([_assistant_message()], response_id="final-turn")), + ] + ) + execute = AsyncMock(return_value="contract.docx") + monkeypatch.setattr(app.LabToolExecutor, "execute", execute) + body = NeMoGymResponseCreateParamsNonStreaming(input=[{"role": "user", "content": "Do the task"}]) + + result = await agent.responses(SimpleNamespace(path_params={}), body) + + assert result.status == "completed" + assert result.error is None + assert result.usage.total_tokens == 10 + assert [type(item) for item in result.output] == [ + NeMoGymResponseFunctionToolCall, + NeMoGymFunctionCallOutput, + NeMoGymResponseOutputMessage, + ] + execute.assert_awaited_once_with("glob", '{"pattern":"**/*"}') + second_input = agent.server_client.post.await_args_list[1].kwargs["json"].input + assert isinstance(second_input[-1], NeMoGymFunctionCallOutput) + assert second_input[-1].output == "contract.docx" + + +async def test_model_failure_preserves_partial_trajectory(monkeypatch) -> None: + agent = _agent() + agent.server_client.post = AsyncMock( + side_effect=[ + _raw_response(_model_response([_function_call()])), + RuntimeError("model disconnected"), + ] + ) + monkeypatch.setattr(app.LabToolExecutor, "execute", AsyncMock(return_value="file.txt")) + + result = await agent.responses( + SimpleNamespace(path_params={}), + NeMoGymResponseCreateParamsNonStreaming(input="Do the task"), + ) + + assert result.status == "failed" + assert result.error is not None + assert "model disconnected" in result.error.message + assert any(isinstance(item, NeMoGymResponseFunctionToolCall) for item in result.output) + assert any(isinstance(item, NeMoGymFunctionCallOutput) for item in result.output) + + +async def test_turn_limit_is_an_explicit_failure(monkeypatch) -> None: + agent = _agent(max_turns=1) + agent.server_client.post = AsyncMock(return_value=_raw_response(_model_response([_function_call()]))) + monkeypatch.setattr(app.LabToolExecutor, "execute", AsyncMock(return_value="file.txt")) + + result = await agent.responses( + SimpleNamespace(path_params={}), + NeMoGymResponseCreateParamsNonStreaming(input="Do the task"), + ) + + assert result.status == "failed" + assert result.error is not None + assert "1-turn limit" in result.error.message + assert len(result.output) == 2 + + +async def test_malformed_tool_arguments_are_returned_to_the_model(monkeypatch) -> None: + executor = app.LabToolExecutor(timeout_seconds=1, max_output_chars=100) + run = AsyncMock() + monkeypatch.setattr(executor, "_run", run) + + result = await executor.execute("read", "not-json") + + assert result.startswith("Error: invalid JSON arguments") + run.assert_not_awaited() + + +async def test_tool_executor_validates_calls_and_truncates_output(monkeypatch) -> None: + executor = app.LabToolExecutor(timeout_seconds=1, max_output_chars=5) + run = AsyncMock(return_value="complete output") + monkeypatch.setattr(executor, "_run", run) + + assert await executor.execute("unknown", {}) == "Error: unknown tool: unknown" + assert await executor.execute("read", []) == "Error: arguments for read must be a JSON object" + assert await executor.execute("bash", {}) == "Error: command is required" + assert await executor.execute("bash", {"command": "pwd"}) == "compl\n[output truncated]" + run.assert_awaited_once_with(["/bin/bash", "-lc", "pwd"]) + + run.reset_mock(return_value=True) + run.return_value = "short" + assert await executor.execute("read", {"file_path": "memo.txt"}) == "short" + + +async def test_preflight_rejects_container_tool_error(monkeypatch) -> None: + executor = app.LabToolExecutor(timeout_seconds=1, max_output_chars=100) + monkeypatch.setattr(executor, "execute", AsyncMock(return_value="Error: missing document tooling")) + + with pytest.raises(RuntimeError, match="missing document tooling"): + await _REAL_PREFLIGHT(executor) + + +async def test_explicit_full_read_is_not_truncated(monkeypatch) -> None: + executor = app.LabToolExecutor(timeout_seconds=1, max_output_chars=5) + monkeypatch.setattr(executor, "_run", AsyncMock(return_value="complete document")) + + result = await executor.execute("read", {"file_path": "memo.txt", "limit": 0}) + + assert result == "complete document" + + +async def test_process_runner_parses_container_result_and_stderr(monkeypatch) -> None: + executor = app.LabToolExecutor(timeout_seconds=1, max_output_chars=100) + process = _process(stdout=b'log line\n{"result":"document text"}\n', stderr=b"tool warning") + create = AsyncMock(return_value=process) + monkeypatch.setattr(app.asyncio, "create_subprocess_exec", create) + + result = await executor._run( + ["/usr/local/bin/python", str(app.CONTAINER_TOOL_RUNNER), "read", '{"file_path":"memo.docx"}'] + ) + + assert result == "document text\nSTDERR:\ntool warning" + assert create.await_args.kwargs["cwd"] == "/workspace/output" + assert create.await_args.kwargs["start_new_session"] is True + + +@pytest.mark.parametrize( + ("stdout", "returncode", "expected"), + [ + (b"not json", 0, "not json"), + (b"", 0, "(no output)"), + (b"command failed", 2, "Error: command failed"), + ], +) +async def test_process_runner_handles_unstructured_and_failed_output( + monkeypatch, + stdout: bytes, + returncode: int, + expected: str, +) -> None: + executor = app.LabToolExecutor(timeout_seconds=1, max_output_chars=100) + monkeypatch.setattr( + app.asyncio, + "create_subprocess_exec", + AsyncMock(return_value=_process(stdout=stdout, returncode=returncode)), + ) + + result = await executor._run(["/usr/local/bin/python", str(app.CONTAINER_TOOL_RUNNER), "preflight", "{}"]) + + assert result == expected + + +async def test_process_runner_kills_timed_out_process_group(monkeypatch) -> None: + executor = app.LabToolExecutor(timeout_seconds=7, max_output_chars=100) + process = _process() + monkeypatch.setattr(app.asyncio, "create_subprocess_exec", AsyncMock(return_value=process)) + + async def time_out(awaitable, *, timeout): + awaitable.close() + assert timeout == 7 + raise asyncio.TimeoutError + + killpg = MagicMock(side_effect=ProcessLookupError) + monkeypatch.setattr(app.asyncio, "wait_for", time_out) + monkeypatch.setattr(app.os, "killpg", killpg) + + result = await executor._run(["/bin/bash", "-lc", "sleep 60"]) + + assert result == "Error: tool timed out after 7s" + killpg.assert_called_once_with(123, app.signal.SIGKILL) + assert process.communicate.call_count == 2 + assert process.communicate.await_count == 1 + + +def test_merge_usage_ignores_missing_current_usage() -> None: + response = app.NeMoGymResponse.model_validate(_model_response([])) + + assert app._merge_usage(response.usage, None) is response.usage + + +async def test_model_error_response_is_returned_with_trajectory() -> None: + agent = _agent() + payload = _model_response([_assistant_message("partial")]) + payload.update( + { + "status": "failed", + "error": {"code": "server_error", "message": "policy backend failed"}, + } + ) + agent.server_client.post = AsyncMock(return_value=_raw_response(payload)) + + result = await agent.responses( + SimpleNamespace(path_params={}), + NeMoGymResponseCreateParamsNonStreaming(input="Do the task"), + ) + + assert result.status == "failed" + assert result.error is not None + assert result.error.message == "policy backend failed" + assert any(isinstance(item, NeMoGymResponseOutputMessage) for item in result.output) + + +@pytest.mark.parametrize("empty_output", [[], [_assistant_message("")]]) +async def test_repeated_empty_model_responses_fail_after_two_nudges(empty_output) -> None: + agent = _agent(max_turns=3) + agent.server_client.post = AsyncMock( + side_effect=[_raw_response(_model_response(empty_output, response_id=f"empty-{index}")) for index in range(3)] + ) + + result = await agent.responses( + SimpleNamespace(path_params={}), + NeMoGymResponseCreateParamsNonStreaming(input="Do the task"), + ) + + assert result.status == "failed" + assert result.error is not None + assert "repeatedly returned empty responses" in result.error.message + second_input = agent.server_client.post.await_args_list[1].kwargs["json"].input + assert any(getattr(item, "content", None) == app.INITIAL_EMPTY_RESPONSE_NUDGE for item in second_input) + + +async def test_preflight_failure_returns_failed_response(monkeypatch) -> None: + agent = _agent() + monkeypatch.setattr(app.LabToolExecutor, "preflight", AsyncMock(side_effect=RuntimeError("missing pandoc"))) + + result = await agent.responses( + SimpleNamespace(path_params={}), + NeMoGymResponseCreateParamsNonStreaming(input="Do the task"), + ) + + assert result.status == "failed" + assert result.error is not None + assert "missing pandoc" in result.error.message + agent.server_client.post.assert_not_called() From 362b522a38aae02243f5654e0e93a1310ca512ff Mon Sep 17 00:00:00 2001 From: Ryan Stewart Date: Mon, 3 Aug 2026 23:50:37 -0400 Subject: [PATCH 10/26] validate LAB runner boundary inputs Signed-off-by: Ryan Stewart --- .../legal_agent_bench_agent/app.py | 79 +++++++++++++++---- 1 file changed, 65 insertions(+), 14 deletions(-) diff --git a/responses_api_agents/legal_agent_bench_agent/app.py b/responses_api_agents/legal_agent_bench_agent/app.py index 88e9d5bfef..2aa152ddd1 100644 --- a/responses_api_agents/legal_agent_bench_agent/app.py +++ b/responses_api_agents/legal_agent_bench_agent/app.py @@ -8,7 +8,9 @@ import fcntl import hashlib import json +import math import os +import re import shutil import stat import subprocess @@ -63,6 +65,12 @@ "claude_code_agent": ("claude_code_version", "CLAUDE_SPEC", "@anthropic-ai/claude-code"), "codex_agent": ("codex_version", "CODEX_SPEC", "@openai/codex"), } +PINNED_NPM_VERSION = re.compile( + r"^(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)" + r"(?:-[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?" + r"(?:\+[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?$" +) +IMMUTABLE_IMAGE = re.compile(r"^[^@\s]+@sha256:[0-9a-f]{64}$") CODEX_MODEL_CATALOG_PATH = "/trajectories_mount/codex_model_catalog.json" CODEX_MODEL_CATALOG = { "models": [ @@ -444,11 +452,12 @@ def agent_runtime_env(agent_server_module: str, agent_kwargs: dict[str, Any]) -> return {} field, environment_variable, package = pin version = agent_kwargs.get(field) - if not isinstance(version, str) or not version.strip(): + normalized_version = version.strip() if isinstance(version, str) else "" + if not PINNED_NPM_VERSION.fullmatch(normalized_version): raise LegalAgentBenchConfigurationError( - f"Configurable LAB agent {key!r} requires a pinned agent_kwargs.{field}" + f"Configurable LAB agent {key!r} requires agent_kwargs.{field} to be an exact npm version" ) - return {environment_variable: f"{package}@{version.strip()}"} + return {environment_variable: f"{package}@{normalized_version}"} def ensure_agent_runtime( @@ -731,7 +740,37 @@ def compose_agent_input( "parallel_tool_calls": False, } ) - return params.model_copy(update=update) + payload = params.model_dump(mode="json") + payload.update(update) + return NeMoGymResponseCreateParamsNonStreaming.model_validate(payload) + + +def _normalized_reward_data(reward_data: Mapping[str, Any]) -> dict[str, Any]: + """Validate verifier metrics before they can reach response construction or host artifacts.""" + + def metric(name: str) -> float: + value = reward_data.get(name, 0.0) + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise LegalAgentBenchArtifactError(f"Invalid verifier {name}: expected a number") + normalized = float(value) + if not math.isfinite(normalized) or not 0.0 <= normalized <= 1.0: + raise LegalAgentBenchArtifactError(f"Invalid verifier {name}: expected a finite value in [0, 1]") + return normalized + + def count(name: str) -> int: + value = reward_data.get(name, 0) + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise LegalAgentBenchArtifactError(f"Invalid verifier {name}: expected a non-negative integer") + return value + + normalized = dict(reward_data) + normalized.update( + reward=metric("reward"), + criteria_pass_rate=metric("criteria_pass_rate"), + verifier_error=count("verifier_error"), + judge_error_count=count("judge_error_count"), + ) + return normalized def _task_toml(task_dir: Path) -> dict[str, Any]: @@ -831,9 +870,11 @@ async def _ensure_runtime(self, image: str) -> Path: async def _ensure_image(self, task_dir: Path) -> str: if self.config.sandbox_image: - if _provider_name(self._provider_config()) != "docker" and "@sha256:" not in self.config.sandbox_image: + if _provider_name(self._provider_config()) != "docker" and not IMMUTABLE_IMAGE.fullmatch( + self.config.sandbox_image + ): raise LegalAgentBenchConfigurationError( - "Non-Docker LAB sandbox_image must use an immutable @sha256 digest" + "Non-Docker LAB sandbox_image must use a complete immutable @sha256 digest" ) return self.config.sandbox_image if _provider_name(self._provider_config()) != "docker": @@ -1187,15 +1228,16 @@ async def _stage_and_run_verifier( @staticmethod def _materialize_verifier_downloads(paths: dict[str, Path], downloaded: dict[str, bytes]) -> dict[str, Any]: - for filename, contents in downloaded.items(): - (paths["verifier"] / filename).write_bytes(contents) try: reward_data = json.loads(downloaded["reward.json"].decode("utf-8")) except (KeyError, UnicodeDecodeError, json.JSONDecodeError) as exc: raise LegalAgentBenchArtifactError(f"Invalid verifier reward.json: {exc}") from exc if not isinstance(reward_data, dict): raise LegalAgentBenchArtifactError("Invalid verifier reward.json: expected an object") - return reward_data + normalized = _normalized_reward_data(reward_data) + for filename, contents in downloaded.items(): + (paths["verifier"] / filename).write_bytes(contents) + return normalized @staticmethod def _runner_status(paths: dict[str, Path]) -> dict[str, Any]: @@ -1317,8 +1359,14 @@ def _response( configuration_failed: bool = False, failure_reason: Optional[str] = None, ) -> LegalAgentBenchAgentResponse: - verifier_error = int(reward_data.get("verifier_error") or 0) - judge_errors = int(reward_data.get("judge_error_count") or 0) + try: + normalized_reward = _normalized_reward_data(reward_data) + except LegalAgentBenchArtifactError as exc: + normalized_reward = _normalized_reward_data({}) + verifier_failed = True + failure_reason = failure_reason or str(exc) + verifier_error = normalized_reward["verifier_error"] + judge_errors = normalized_reward["judge_error_count"] unreliable = bool( agent_failed or model_connection_failed @@ -1334,9 +1382,9 @@ def _response( return LegalAgentBenchAgentResponse( responses_create_params=params, response=response, - reward=float(reward_data.get("reward") or 0.0), + reward=normalized_reward["reward"], instance_id=body.instance_id, - criteria_pass_rate=float(reward_data.get("criteria_pass_rate") or 0.0), + criteria_pass_rate=normalized_reward["criteria_pass_rate"], judge_error_count=judge_errors, verifier_error=verifier_error, mask_sample=unreliable, @@ -1482,7 +1530,10 @@ async def run(self, request: Request, body: LegalAgentBenchRunRequest) -> LegalA await verifier_sandbox.stop() verifier_sandbox = None if verifier_failure is None: - reward_data = self._materialize_verifier_downloads(paths, verifier_downloads) + try: + reward_data = self._materialize_verifier_downloads(paths, verifier_downloads) + except LegalAgentBenchArtifactError as exc: + verifier_failure = str(exc) verifier_failed = verifier_failure is not None failure_reason = verifier_failure or failure_reason except LegalAgentBenchTaskError as exc: From b6ed2b1fb7e268a56460932c9d9a4de734446a38 Mon Sep 17 00:00:00 2001 From: Ryan Stewart Date: Mon, 3 Aug 2026 23:50:44 -0400 Subject: [PATCH 11/26] test: cover LAB runner boundary and failure paths Signed-off-by: Ryan Stewart --- .../legal_agent_bench_agent/tests/test_app.py | 368 +++++++++++++++++- 1 file changed, 361 insertions(+), 7 deletions(-) diff --git a/responses_api_agents/legal_agent_bench_agent/tests/test_app.py b/responses_api_agents/legal_agent_bench_agent/tests/test_app.py index 3bfb94e7fd..5ee076d4ca 100644 --- a/responses_api_agents/legal_agent_bench_agent/tests/test_app.py +++ b/responses_api_agents/legal_agent_bench_agent/tests/test_app.py @@ -7,6 +7,7 @@ import json import multiprocessing import os +import sys import tarfile from pathlib import Path from types import SimpleNamespace @@ -161,6 +162,12 @@ def test_dependency_runtime_cache_is_harness_and_recipe_specific(monkeypatch, tm monkeypatch.setattr(app, "resolve_agent_setup_script", lambda module: script) def fake_run(args, **kwargs): + if len(args) > 1 and args[1] == "run": + mounted_bundle = next( + Path(arg.split(":", 1)[0]) for arg in args if ":/nemo_gym_mount:ro" in arg + ) + assert mounted_bundle != tmp_path + assert not (mounted_bundle / "env.yaml").exists() calls.append((args, kwargs)) return SimpleNamespace(stdout="sha256:image:linux/arm64\n") @@ -250,6 +257,15 @@ def test_agent_runtime_env_rejects_missing_harness_pin(module, field) -> None: app.agent_runtime_env(module, {}) +@pytest.mark.parametrize("version", ["latest", "next", "^1.2.3", "1.2", "1.2.3 || 2.0.0"]) +def test_agent_runtime_env_rejects_non_exact_harness_pin(version) -> None: + with pytest.raises(app.LegalAgentBenchConfigurationError, match="exact npm version"): + app.agent_runtime_env( + "responses_api_agents.codex_agent.app", + {"codex_version": version}, + ) + + def test_dependency_provisioning_uses_pin_and_invalidates_when_it_changes(monkeypatch, tmp_path) -> None: package_dir = tmp_path / "legal_agent_bench_agent" package_dir.mkdir() @@ -308,6 +324,60 @@ def test_runtime_archive_is_published_once_across_processes(tmp_path) -> None: assert archive.is_file() with tarfile.open(archive, "r:gz") as contents: assert "agent_deps_mount/runtime.py" in contents.getnames() + assert app.ensure_runtime_archive(deps) == archive + + +def test_archive_helpers_reject_missing_unsafe_and_nonregular_inputs(tmp_path) -> None: + with pytest.raises(FileNotFoundError): + app._create_archive(tmp_path / "missing.tar.gz", [(tmp_path / "absent", "input")]) + + traversal = tarfile.TarInfo("../outside") + with pytest.raises(app.LegalAgentBenchArtifactError, match="Unsafe"): + app._validate_archive_member(traversal) + + device = tarfile.TarInfo("device") + device.type = tarfile.CHRTYPE + with pytest.raises(app.LegalAgentBenchArtifactError, match="regular files"): + app._validate_archive_member(device) + + with pytest.raises(app.LegalAgentBenchArtifactError, match="regular downloaded file"): + app._copy_downloaded_file(tmp_path, tmp_path / "copy") + + +def test_provider_and_installer_validation_rejects_ambiguous_or_missing_configuration(monkeypatch, tmp_path) -> None: + with pytest.raises(app.LegalAgentBenchConfigurationError, match="exactly one provider"): + app._provider_name({"docker": {}, "ecs_fargate": {}}) + + monkeypatch.setattr(app, "PARENT_DIR", tmp_path) + with pytest.raises(app.LegalAgentBenchConfigurationError, match="requires dependency setup script"): + app.resolve_agent_setup_script("responses_api_agents.hermes_agent.app") + + +def test_recipe_hash_ignores_transient_bytecode(tmp_path) -> None: + package = tmp_path / "package" + (package / "__pycache__").mkdir(parents=True) + (package / "app.py").write_text("VALUE = 1\n") + bytecode = package / "__pycache__" / "app.pyc" + bytecode.write_bytes(b"first") + first = app._recipe_hash([package]) + + bytecode.write_bytes(b"second") + + assert app._recipe_hash([package]) == first + + +def test_runtime_provisioning_requires_docker(monkeypatch) -> None: + monkeypatch.setattr(app, "resolve_agent_setup_script", lambda _module: Path("installer.sh")) + monkeypatch.setattr(app.shutil, "which", lambda _name: None) + + with pytest.raises(FileNotFoundError, match="Docker CLI"): + app.ensure_agent_runtime( + "responses_api_agents.hermes_agent.app", + agent_kwargs={}, + image="lab:image", + docker_network="host", + timeout_seconds=60, + ) def test_config_defaults_are_docker_and_single_concurrency() -> None: @@ -412,12 +482,13 @@ def test_compose_agent_input_uses_task_and_skills_without_rubric(monkeypatch, tm assert result.temperature == 0.5 assert len(result.input) == 2 - system = result.input[0]["content"] + serialized = result.model_dump(mode="json", warnings="error") + system = serialized["input"][0]["content"] assert "Write a memo." in system assert "Skill: docx" in system assert "/workspace/vdr" in system assert "criteria" not in system - assert result.input[1]["content"] == app.INITIAL_USER_PROMPT + assert serialized["input"][1]["content"] == app.INITIAL_USER_PROMPT def test_native_agent_input_uses_upstream_prompt_and_canonical_tools(monkeypatch, tmp_path) -> None: @@ -432,7 +503,8 @@ def test_native_agent_input_uses_upstream_prompt_and_canonical_tools(monkeypatch native=True, ) - system = result.input[0]["content"] + serialized = result.model_dump(mode="json", warnings="error") + system = serialized["input"][0]["content"] assert system.startswith(app.LAB_SYSTEM_PROMPT) assert "Write a memo." in system assert '"criteria"' not in system @@ -733,6 +805,37 @@ def test_response_masks_harness_and_verifier_failures() -> None: assert response.mask_sample is True +@pytest.mark.parametrize( + "reward_data", + [ + {"reward": float("nan")}, + {"criteria_pass_rate": 1.1}, + {"judge_error_count": "not-a-number"}, + {"verifier_error": -1}, + ], +) +def test_response_masks_malformed_verifier_metrics_instead_of_raising(reward_data) -> None: + runner = app.LegalAgentBenchAgent.model_construct(config=_config()) + params = NeMoGymResponseCreateParamsNonStreaming(input=[]) + body = app.LegalAgentBenchRunRequest( + instance_id="legal_agent_bench::area__task", + responses_create_params=params, + ) + + response = runner._response( + body=body, + params=params, + response=_successful_response(), + reward_data=reward_data, + paths=None, + ) + + assert response.reward == 0.0 + assert response.verifier_failed is True + assert response.mask_sample is True + assert response.failure_reason.startswith("Invalid verifier") + + @pytest.mark.asyncio async def test_run_classifies_invalid_task_without_sandbox_failure(monkeypatch) -> None: runner = app.LegalAgentBenchAgent.model_construct(config=_config()) @@ -978,7 +1081,16 @@ def fake_sandbox(provider, spec): @pytest.mark.asyncio -async def test_run_stops_agent_sandbox_before_starting_verifier_sandbox(monkeypatch, tmp_path, capsys) -> None: +@pytest.mark.parametrize( + ("reward_blob", "expected_reward", "expected_verifier_failed"), + [ + (b'{"reward": 1.0, "criteria_pass_rate": 1.0}', 1.0, False), + (b'{"reward": "invalid", "criteria_pass_rate": 1.0}', 0.0, True), + ], +) +async def test_run_stops_agent_sandbox_before_starting_verifier_sandbox( + monkeypatch, tmp_path, capsys, reward_blob, expected_reward, expected_verifier_failed +) -> None: _root, task = _task_tree(tmp_path) skills = _skills(tmp_path) deps = tmp_path / "deps" @@ -1020,7 +1132,7 @@ async def run_verifier(sandbox, task_dir, paths): assert sandbox is verifier_sandbox assert events == ["agent:start", "agent:exec", "agent:stop", "verifier:start"] assert runner._session_results_dir.is_dir() - return {"reward.json": b'{"reward": 1.0, "criteria_pass_rate": 1.0}'}, False, None + return {"reward.json": reward_blob}, False, None async def stage_agent(*args, **kwargs): return None @@ -1052,12 +1164,14 @@ async def collect_agent(*args, **kwargs): response = await runner.run(None, body) assert events == ["agent:start", "agent:exec", "agent:stop", "verifier:start", "verifier:stop"] - assert response.reward == 1.0 + assert response.reward == expected_reward + assert response.verifier_failed is expected_verifier_failed assert response.sandbox_failed is False assert Path(response.artifact_dir).is_dir() terminal_output = capsys.readouterr().out assert "LAB rollout artifacts:" in terminal_output - assert "LAB rollout complete:" in terminal_output + expected_terminal_status = "failed" if expected_verifier_failed else "complete" + assert f"LAB rollout {expected_terminal_status}:" in terminal_output @pytest.mark.asyncio @@ -1276,6 +1390,27 @@ def test_named_remote_provider_requires_immutable_image(tmp_path) -> None: assert app.asyncio.run(runner._ensure_image(tmp_path)) == runner.config.sandbox_image +@pytest.mark.parametrize( + "image", + [ + "registry.example/lab@sha256:", + "registry.example/lab@sha256:abc123", + "registry.example/lab@sha512:" + "a" * 128, + "registry.example/lab:latest@sha256:" + "g" * 64, + ], +) +def test_named_remote_provider_rejects_incomplete_or_invalid_digest(image, tmp_path) -> None: + runner = app.LegalAgentBenchAgent.model_construct( + config=_config( + sandbox_provider={"ecs_fargate": {"region": "us-east-1"}}, + sandbox_image=image, + ) + ) + + with pytest.raises(app.LegalAgentBenchConfigurationError, match="complete immutable"): + app.asyncio.run(runner._ensure_image(tmp_path)) + + @pytest.mark.asyncio async def test_agent_staging_excludes_tests_and_judge_credentials(tmp_path) -> None: _root, task = _task_tree(tmp_path) @@ -1319,3 +1454,222 @@ async def exec(self, command, **kwargs): assert any(name.startswith("workspace/skills") for name in staged) assert not any("task.toml" in name or "/tests" in name or "LAB_JUDGE" in name for name in staged) assert "chmod -R a+rX,a-w /agent_source_mount /agent_deps_mount" in sandbox.exec_commands[0] + + +@pytest.mark.asyncio +async def test_run_process_returns_output_and_terminates_on_timeout(tmp_path) -> None: + code, stdout, stderr = await app._run_process( + [sys.executable, "-c", "import sys; print('out'); print('err', file=sys.stderr)"], + cwd=tmp_path, + timeout=5, + ) + assert (code, stdout.strip(), stderr.strip()) == (0, "out", "err") + + with pytest.raises(TimeoutError, match="Command timed out"): + await app._run_process( + [sys.executable, "-c", "import time; time.sleep(10)"], + cwd=tmp_path, + timeout=0.01, + ) + + +@pytest.mark.asyncio +async def test_runtime_is_provisioned_once_per_agent_instance(monkeypatch, tmp_path) -> None: + runner = app.LegalAgentBenchAgent.model_construct(config=_config()) + runner._runtime_lock = app.asyncio.Lock() + runner._deps_dir = None + calls = [] + + def provision(*args, **kwargs): + calls.append((args, kwargs)) + return tmp_path / "deps" + + monkeypatch.setattr(app, "ensure_agent_runtime", provision) + + first, second = await app.asyncio.gather(runner._ensure_runtime("lab:image"), runner._ensure_runtime("lab:image")) + + assert first == second == tmp_path / "deps" + assert len(calls) == 1 + + +@pytest.mark.asyncio +async def test_docker_image_resolution_handles_missing_cached_and_failed_builds(monkeypatch, tmp_path) -> None: + _root, task = _task_tree(tmp_path) + runner = app.LegalAgentBenchAgent.model_construct(config=_config()) + runner._image_lock = app.asyncio.Lock() + + monkeypatch.setattr(app.shutil, "which", lambda _name: None) + with pytest.raises(FileNotFoundError, match="Docker CLI"): + await runner._ensure_image(task) + + monkeypatch.setattr(app.shutil, "which", lambda _name: "/usr/bin/docker") + + async def cached(*args, **kwargs): + return 0, "", "" + + monkeypatch.setattr(app, "_run_process", cached) + assert await runner._ensure_image(task) == ( + f"{runner.config.image_repository}:" + app._environment_hash(task / "environment")[:16] + ) + + async def failed_build(args, **kwargs): + return (1, "", "missing") if args[1] == "image" else (1, "", "build exploded") + + monkeypatch.setattr(app, "_run_process", failed_build) + with pytest.raises(RuntimeError, match="build exploded"): + await runner._ensure_image(task) + + +def test_model_url_reports_server_lookup_failures_and_routes_docker(monkeypatch) -> None: + body = app.LegalAgentBenchRunRequest( + instance_id="legal_agent_bench::area__task", + responses_create_params=NeMoGymResponseCreateParamsNonStreaming(input=[]), + ) + runner = app.LegalAgentBenchAgent.model_construct( + config=_config(), + server_client=SimpleNamespace(global_config_dict={}), + ) + with pytest.raises(app.LegalAgentBenchConfigurationError, match="Unable to resolve policy model"): + runner._model_url(body) + + runner.server_client = SimpleNamespace( + global_config_dict={"policy_model": {}}, + _build_server_base_url=lambda _config: "http://127.0.0.1:8000", + ) + monkeypatch.setattr(app, "get_first_server_config_dict", lambda *_args: {}) + monkeypatch.setattr(app.LegalAgentBenchAgent, "rollout_id_from_run", lambda _self, _body: None) + assert runner._model_url(body).startswith("http://host.docker.internal:8000") + + +@pytest.mark.asyncio +async def test_remote_provider_requires_an_image_and_missing_agent_source_is_configuration_error(tmp_path) -> None: + runner = app.LegalAgentBenchAgent.model_construct( + config=_config(sandbox_provider={"ecs_fargate": {"region": "us-east-1"}}) + ) + with pytest.raises(app.LegalAgentBenchConfigurationError, match="require sandbox_image"): + await runner._ensure_image(tmp_path) + + paths = runner._paths_for_root(tmp_path / "staging", create=True) + runner.config.agent_server_module = "responses_api_agents.nonexistent_agent.app" + with pytest.raises(app.LegalAgentBenchConfigurationError, match="source not found"): + runner._stage_agent_source(paths) + + +def test_failure_detection_covers_empty_and_zero_activity_responses() -> None: + empty = app._empty_response("policy") + assert app.agent_response_failure(empty, "responses_api_agents.codex_agent.app") == ( + "Agent produced an empty trajectory" + ) + + silent = _successful_response("") + silent.usage.total_tokens = 0 + assert app.agent_response_failure(silent, "responses_api_agents.codex_agent.app") == ( + "codex_agent produced no model activity" + ) + + assert app.host_tunnel_model_url("https://model.example/v1") == "https://model.example/v1" + + +def test_task_prompt_and_skills_report_invalid_source_files(tmp_path) -> None: + _root, task = _task_tree(tmp_path) + skills = _skills(tmp_path) + params = NeMoGymResponseCreateParamsNonStreaming(input=[]) + + (task / "task.json").write_text("not json") + with pytest.raises(app.LegalAgentBenchTaskError, match="Invalid LAB task configuration"): + app.compose_agent_input(task, skills, params) + + (task / "task.toml").write_text("not = [valid") + with pytest.raises(app.LegalAgentBenchTaskError, match="Invalid LAB task.toml"): + app._task_toml(task) + + (skills / app.REQUIRED_SKILLS[0] / "SKILL.md").unlink() + with pytest.raises(app.LegalAgentBenchConfigurationError, match="Invalid LAB skills configuration"): + app._load_skill_prompt(skills) + + +@pytest.mark.asyncio +async def test_agent_staging_and_collection_surface_sandbox_failures(tmp_path) -> None: + _root, task = _task_tree(tmp_path) + skills = _skills(tmp_path) + deps = tmp_path / "deps" + deps.mkdir() + (deps / "runtime.py").write_text("pass\n") + paths = app.LegalAgentBenchAgent._paths_for_root(tmp_path / "staged", create=True) + (paths["agent_source"] / "app.py").write_text("pass\n") + + class Sandbox: + async def upload(self, source, destination): + return None + + async def exec(self, command, **kwargs): + return SimpleNamespace(return_code=1, stdout="", stderr="sandbox command failed") + + runner = app.LegalAgentBenchAgent.model_construct(config=_config()) + with pytest.raises(RuntimeError, match="sandbox command failed"): + await runner._stage_agent_sandbox(Sandbox(), task_dir=task, skills_dir=skills, deps_dir=deps, paths=paths) + + with pytest.raises(RuntimeError, match="sandbox command failed"): + await runner._collect_agent_sandbox(Sandbox(), tmp_path / "downloads") + + +@pytest.mark.asyncio +async def test_agent_collection_and_materialization_preserve_optional_files(tmp_path) -> None: + download_dir = tmp_path / "downloads" + download_dir.mkdir() + output_source = tmp_path / "output-source" + output_source.mkdir() + (output_source / "memo.txt").write_text("complete") + + class Sandbox: + async def exec(self, command, **kwargs): + return SimpleNamespace(return_code=0, stdout="", stderr="") + + async def download(self, source, destination): + destination = Path(destination) + if source.endswith("response.json"): + destination.write_text(_successful_response().model_dump_json()) + elif source.endswith("runner_status.json"): + raise FileNotFoundError(source) + else: + app._create_archive(destination, [(output_source, ".")]) + + runner = app.LegalAgentBenchAgent.model_construct(config=_config()) + downloads = await runner._collect_agent_sandbox(Sandbox(), download_dir) + paths = runner._paths_for_root(tmp_path / "result", create=True) + runner._materialize_agent_downloads(paths, downloads, stdout="stdout", stderr="stderr") + + assert set(downloads) == {"response.json", "output.tar.gz"} + assert (paths["runtime"] / "response.json").is_file() + assert (paths["output"] / "memo.txt").read_text() == "complete" + assert (paths["agent"] / "stdout.log").read_text() == "stdout" + + with pytest.raises(app.LegalAgentBenchArtifactError, match="output archive"): + runner._materialize_agent_downloads(paths, {}, stdout="", stderr="") + + +def test_verifier_materialization_rejects_bad_reward_before_writing_files(tmp_path) -> None: + paths = {"verifier": tmp_path / "verifier"} + paths["verifier"].mkdir() + + with pytest.raises(app.LegalAgentBenchArtifactError, match="expected an object"): + app.LegalAgentBenchAgent._materialize_verifier_downloads( + paths, + {"reward.json": b"[]", "report.html": b"untrusted"}, + ) + assert list(paths["verifier"].iterdir()) == [] + + with pytest.raises(app.LegalAgentBenchArtifactError, match="Invalid verifier reward.json"): + app.LegalAgentBenchAgent._materialize_verifier_downloads(paths, {"reward.json": b"not-json"}) + + +def test_runner_status_reports_invalid_json_and_nonobject(tmp_path) -> None: + paths = {"runtime": tmp_path} + assert app.LegalAgentBenchAgent._runner_status(paths) == {} + + status = tmp_path / "runner_status.json" + status.write_text("not-json") + assert app.LegalAgentBenchAgent._runner_status(paths)["ok"] is False + + status.write_text("[]") + assert app.LegalAgentBenchAgent._runner_status(paths)["error"].endswith("expected an object") From d756acb0b639964d1fff53f92652526b2f26ba7d Mon Sep 17 00:00:00 2001 From: Ryan Stewart Date: Tue, 4 Aug 2026 09:27:43 -0400 Subject: [PATCH 12/26] propagate LAB Harbor agent failures Signed-off-by: Ryan Stewart --- benchmarks/legal_agent_bench/README.md | 8 +- resources_servers/legal_agent_bench/README.md | 4 + .../configs/legal_agent_bench.yaml | 1 + .../legal_agent_bench/legal_harbor_agent.py | 21 ++- .../tests/test_harbor_integration.py | 160 ++++++++++++++++++ responses_api_agents/harbor_agent/app.py | 64 ++++++- .../harbor_agent/tests/test_app.py | 87 +++++++++- 7 files changed, 338 insertions(+), 7 deletions(-) diff --git a/benchmarks/legal_agent_bench/README.md b/benchmarks/legal_agent_bench/README.md index a306bc8b7b..32ff087724 100644 --- a/benchmarks/legal_agent_bench/README.md +++ b/benchmarks/legal_agent_bench/README.md @@ -221,8 +221,10 @@ created only when its first rollout starts. Each configurable rollout is stored under `_jobs///_/`, where the task name is normalized and the run ID is an eight-character unique suffix. -Harbor's result row uses the established Harbor response format rather than the -configurable runner's top-level flags. +Harbor retains its established response format and additionally reports +`mask_sample`, `agent_failed`, `model_connection_failed`, `agent_timed_out`, and +`failure_reason` for agent-phase failures. It preserves any partial trajectory, +but skips judging and forces reward to zero when one of those failures occurs. ## Run a larger evaluation @@ -372,6 +374,8 @@ is masked and is not sent to the judge. A normal model/task result can still receive zero reward without those flags. Infrastructure, configuration, task-loading, or judge failures set `mask_sample`; do not treat those zeroes as model-quality results. +The Harbor variant reports the agent-phase subset described above and applies +the same masking and no-judge behavior to those failures. Configurable-runner artifacts are grouped by harness, model, dated session, and task. The default roots are `results/legal_agent_bench/native_jobs`, diff --git a/resources_servers/legal_agent_bench/README.md b/resources_servers/legal_agent_bench/README.md index b58e9dbb9d..f7e1d167eb 100644 --- a/resources_servers/legal_agent_bench/README.md +++ b/resources_servers/legal_agent_bench/README.md @@ -305,6 +305,10 @@ or more criteria fail; use `criteria_pass_rate` to see partial success. sandbox failures. A zero reward without an infrastructure/judge flag is an ordinary model/task result; a flagged result should be excluded from model-quality comparisons. +- Harbor agent, adapter, connection, and timeout failures preserve any partial + trajectory but skip judging. Their result rows report `mask_sample`, + `agent_failed`, `model_connection_failed`, `agent_timed_out`, and + `failure_reason`, and their reward is forced to zero. - Hermes normally probes `/v1/models` and `/models` for optional pricing and context metadata. The LAB runner disables those lookups because Gym supplies the model explicitly and its internal policy proxy does not implement model diff --git a/resources_servers/legal_agent_bench/configs/legal_agent_bench.yaml b/resources_servers/legal_agent_bench/configs/legal_agent_bench.yaml index 3344f78106..8211aa28a0 100644 --- a/resources_servers/legal_agent_bench/configs/legal_agent_bench.yaml +++ b/resources_servers/legal_agent_bench/configs/legal_agent_bench.yaml @@ -69,6 +69,7 @@ legal_agent_bench_harbor_agent: harbor_agent_override_timeout: 10800 harbor_agent_max_timeout: null + harbor_skip_verification_on_agent_failure: true harbor_verifier_override_timeout: 3600 harbor_verifier_max_timeout: null harbor_timeout_multiplier: null diff --git a/resources_servers/legal_agent_bench/legal_harbor_agent.py b/resources_servers/legal_agent_bench/legal_harbor_agent.py index 7961937c92..08c4ff22b0 100644 --- a/resources_servers/legal_agent_bench/legal_harbor_agent.py +++ b/resources_servers/legal_agent_bench/legal_harbor_agent.py @@ -58,6 +58,10 @@ def __init__(self, logs_dir: Path, model_name: str | None = None, logger=None, * SYSTEM_PROMPT_PREAMBLE = SYSTEM_PROMPT_PATH.read_text(encoding="utf-8") +class LegalAgentBenchHarnessError(RuntimeError): + """The LAB Harbor harness failed after preserving its partial artifacts.""" + + class HarborToolExecutor: """Execute LAB tools inside a Harbor environment.""" @@ -310,8 +314,11 @@ async def run(self, instruction: str, environment: BaseEnvironment, context: Age model_name=self.model, agent_name=self.name(), ) + agent_failed = bool(result.get("model_error")) _write_agent_error_flags(Path(self.logs_dir), metrics) _populate_context(context, result, metrics, task_id, self.agent_id, artifact_dir) + if agent_failed: + raise LegalAgentBenchHarnessError(result["model_error"]) async def _hydrate_environment(self, environment: BaseEnvironment, docs_dir: Path) -> None: validate_harness_skills(self.skills_dir) @@ -391,6 +398,9 @@ async def _run_agent_async( finished_cleanly = False context_overflow = False model_error = None + model_error_type = None + model_connection_failed = False + agent_timed_out = False empty_response_count = 0 start_time = time.time() @@ -404,10 +414,13 @@ async def _run_agent_async( except Exception as exc: err_msg = str(exc) _log_model_error(transcript_file, turn_count, exc) + model_error = err_msg + model_error_type = type(exc).__name__ + agent_timed_out = isinstance(exc, TimeoutError) + model_connection_failed = isinstance(exc, (ConnectionError, OSError)) and not agent_timed_out if _is_context_overflow_error(err_msg): context_overflow = True break - model_error = err_msg break messages.append(response.message) @@ -453,6 +466,9 @@ async def _run_agent_async( "finished_cleanly": (not context_overflow and finished_cleanly), "context_overflow": context_overflow, "model_error": model_error, + "model_error_type": model_error_type, + "model_connection_failed": model_connection_failed, + "agent_timed_out": agent_timed_out, "tool_metrics": tool_executor.get_metrics(), } @@ -717,6 +733,9 @@ def _populate_context( "lab_run_id": metrics["run_id"], "artifact_dir": str(artifact_dir), "finished_cleanly": metrics["finished_cleanly"], + "agent_failed": bool(result.get("model_error")), + "model_connection_failed": bool(result.get("model_connection_failed")), + "agent_timed_out": bool(result.get("agent_timed_out")), "model_error": metrics.get("model_error"), "turn_count": metrics["turn_count"], "tool_metrics": { diff --git a/resources_servers/legal_agent_bench/tests/test_harbor_integration.py b/resources_servers/legal_agent_bench/tests/test_harbor_integration.py index c969081115..fe81c52e33 100644 --- a/resources_servers/legal_agent_bench/tests/test_harbor_integration.py +++ b/resources_servers/legal_agent_bench/tests/test_harbor_integration.py @@ -10,6 +10,7 @@ from asyncio import Semaphore from datetime import datetime, timezone from pathlib import Path +from types import SimpleNamespace from unittest.mock import MagicMock import pytest @@ -18,8 +19,11 @@ from resources_servers.legal_agent_bench.harbor_bridge import REPO_ROOT, LegalAgentBenchHarborBridge from resources_servers.legal_agent_bench.legal_harbor_agent import ( LegalAgentBenchHarborAgent, + LegalAgentBenchHarnessError, + ModelResponse, OpenAICompatibleAdapter, _chat_with_timeout, + _run_agent_async, ) from resources_servers.legal_agent_bench.prepare import ( EXPECTED_TASK_COUNT, @@ -80,6 +84,7 @@ def test_folder_config_is_public_docker_only() -> None: assert resource["auto_prepare_assets"] is True assert agent["harbor_environment_type"] == "docker" assert agent["harbor_environment_import_path"] is None + assert agent["harbor_skip_verification_on_agent_failure"] is True assert "docker_image" not in json.dumps(agent) @@ -272,6 +277,161 @@ async def slow_chat(_messages, _tools): await _chat_with_timeout(adapter, [], []) +@pytest.mark.asyncio +async def test_harbor_agent_preserves_partial_trajectory_then_propagates_failure(monkeypatch, tmp_path) -> None: + task_dir = tmp_path / "task" + (task_dir / "documents").mkdir(parents=True) + (task_dir / "environment").mkdir() + (task_dir / "task.toml").write_text('version = "1.0"\n', encoding="utf-8") + (task_dir / "task.json").write_text( + json.dumps( + { + "title": "Test task", + "instructions": "Write the deliverable.", + "criteria": [{"id": "C-1", "title": "Done", "match_criteria": "Pass."}], + } + ), + encoding="utf-8", + ) + logs_dir = tmp_path / "logs" / "agent" + agent = LegalAgentBenchHarborAgent( + logs_dir=logs_dir, + model_name="policy-model", + api_base="http://policy/v1", + skills_dir=tmp_path / "skills", + ) + monkeypatch.setattr(agent, "_skill_names", lambda: []) + + async def hydrate(_environment, _docs_dir): + return None + + monkeypatch.setattr(agent, "_hydrate_environment", hydrate) + monkeypatch.setattr(agent, "_create_adapter", MagicMock()) + monkeypatch.setattr( + "resources_servers.legal_agent_bench.legal_harbor_agent.get_all_tool_definitions", + lambda: [], + ) + + class ToolExecutor: + def __init__(self, *_args, **_kwargs): + pass + + async def preflight(self): + return None + + monkeypatch.setattr( + "resources_servers.legal_agent_bench.legal_harbor_agent.HarborToolExecutor", + ToolExecutor, + ) + + async def partial_failure(*, transcript_path, **_kwargs): + transcript_path.write_text( + json.dumps( + { + "turn": 1, + "role": "assistant", + "message": {"role": "assistant", "content": "Partial work"}, + "text": "Partial work", + "tool_calls": None, + "input_tokens": 11, + "output_tokens": 3, + } + ) + + "\n", + encoding="utf-8", + ) + return { + "messages": [], + "turn_count": 1, + "input_tokens": 11, + "output_tokens": 3, + "wall_clock_seconds": 0.1, + "finished_cleanly": False, + "context_overflow": False, + "model_error": "adapter failed after partial output", + "model_error_type": "ConnectionError", + "model_connection_failed": True, + "agent_timed_out": False, + "tool_metrics": {}, + } + + monkeypatch.setattr( + "resources_servers.legal_agent_bench.legal_harbor_agent._run_agent_async", + partial_failure, + ) + + class Environment: + environment_dir = task_dir / "environment" + + async def download_dir(self, _source, target): + Path(target).mkdir(parents=True, exist_ok=True) + + context = SimpleNamespace() + with pytest.raises(LegalAgentBenchHarnessError, match="adapter failed after partial output"): + await agent.run("", Environment(), context) + + trajectory = json.loads((logs_dir / "trajectory.json").read_text(encoding="utf-8")) + assert trajectory["steps"][-1]["message"] == "Partial work" + assert context.metadata["agent_failed"] is True + assert context.metadata["model_connection_failed"] is True + assert context.metadata["agent_timed_out"] is False + assert context.metadata["model_error"] == "adapter failed after partial output" + + +@pytest.mark.asyncio +async def test_harbor_loop_classifies_timeout_after_partial_output(tmp_path) -> None: + class Adapter: + timeout_seconds = None + + def __init__(self): + self.calls = 0 + + def make_system_message(self, content): + return {"role": "system", "content": content} + + def make_user_message(self, content): + return {"role": "user", "content": content} + + def make_tool_result_messages(self, results): + return [{"role": "tool", "content": result} for _call_id, result in results] + + async def chat(self, _messages, _tools): + self.calls += 1 + if self.calls == 1: + return ModelResponse( + message={"role": "assistant", "content": "Working"}, + tool_calls=[SimpleNamespace(id="call-1", name="read", arguments='{"path":"input.docx"}')], + text="Working", + input_tokens=10, + output_tokens=2, + ) + raise TimeoutError("model request timed out") + + class ToolExecutor: + async def execute(self, _name, _arguments): + return "document text" + + def get_metrics(self): + return {} + + transcript = tmp_path / "transcript.jsonl" + result = await _run_agent_async( + adapter=Adapter(), + system_prompt="system", + tool_executor=ToolExecutor(), + tools=[], + max_turns=3, + transcript_path=transcript, + ) + + entries = [json.loads(line) for line in transcript.read_text(encoding="utf-8").splitlines()] + assert any(entry.get("role") == "assistant" for entry in entries) + assert any(entry.get("role") == "model_error" for entry in entries) + assert result["model_error"] == "model request timed out" + assert result["agent_timed_out"] is True + assert result["model_connection_failed"] is False + + @pytest.mark.parametrize( "message, expected", [ diff --git a/responses_api_agents/harbor_agent/app.py b/responses_api_agents/harbor_agent/app.py index 43159df562..67e4aca48e 100644 --- a/responses_api_agents/harbor_agent/app.py +++ b/responses_api_agents/harbor_agent/app.py @@ -111,6 +111,10 @@ class HarborAgentConfig(BaseResponsesAPIAgentConfig): # Keep Docker runtime images between trials (maps to EnvironmentConfig.delete=False). harbor_no_delete: bool = True + # Harbor normally proceeds to verification after an agent timeout. Benchmarks + # that cannot score partial agent output can opt out through a trial hook. + harbor_skip_verification_on_agent_failure: bool = False + # --- Model routing --- # NeMo Gym model server reference used to resolve Harbor model base URL. model_server: ModelServerRef @@ -134,7 +138,13 @@ def _find_trial_dir_with_result(job_dir: Path) -> Optional[Path]: return None -async def run_harbor_job(job_config_dict: dict) -> str: +async def _stop_verification_after_agent_failure(event: Any) -> None: + result = event.result + if result is not None and result.exception_info is not None: + raise RuntimeError(f"Skipping verification after agent failure: {result.exception_info.exception_type}") + + +async def run_harbor_job(job_config_dict: dict, skip_verification_on_agent_failure: bool = False) -> str: """Runs a single Harbor Job and returns the *absolute* trial directory path. The trial directory contains: @@ -164,6 +174,8 @@ async def run_harbor_job(job_config_dict: dict) -> str: config = JobConfig(**job_config_dict) job = Job(config) + if skip_verification_on_agent_failure: + job.on_verification_started(_stop_verification_after_agent_failure) job_error = None try: @@ -189,7 +201,7 @@ async def run_harbor_job(job_config_dict: dict) -> str: _RAY_WORKER_EVENT_LOOP: Optional[asyncio.AbstractEventLoop] = None -def _run_harbor_job_sync(job_config_dict: dict) -> str: +def _run_harbor_job_sync(job_config_dict: dict, skip_verification_on_agent_failure: bool = False) -> str: """Synchronous wrapper for run_harbor_job for use in Ray remote. Ray workers are long-lived processes. Reusing a single event loop per worker @@ -200,7 +212,9 @@ def _run_harbor_job_sync(job_config_dict: dict) -> str: if _RAY_WORKER_EVENT_LOOP is None or _RAY_WORKER_EVENT_LOOP.is_closed(): _RAY_WORKER_EVENT_LOOP = asyncio.new_event_loop() asyncio.set_event_loop(_RAY_WORKER_EVENT_LOOP) - return _RAY_WORKER_EVENT_LOOP.run_until_complete(run_harbor_job(job_config_dict)) + return _RAY_WORKER_EVENT_LOOP.run_until_complete( + run_harbor_job(job_config_dict, skip_verification_on_agent_failure) + ) @ray.remote( @@ -270,6 +284,7 @@ async def run(self, body: HarborRunRequest) -> HarborVerifyResponse: try: params = dict( job_config_dict=job_config_dict, + skip_verification_on_agent_failure=self.config.harbor_skip_verification_on_agent_failure, ) runner = runner_ray_remote if self.config.harbor_ray_task_num_cpus is not None: @@ -297,6 +312,39 @@ def _read_trial_files(): verifier_result = trial_result.get("verifier_result") reward = HarborAgentUtils.extract_reward(verifier_result) + failure_fields = {} + if self.config.harbor_skip_verification_on_agent_failure: + exception_info = trial_result.get("exception_info") or {} + agent_metadata = (trial_result.get("agent_result") or {}).get("metadata") or {} + failed_during_agent_phase = bool( + exception_info + and trial_result.get("agent_execution") is not None + and trial_result.get("verifier") is None + ) + agent_timed_out = bool( + exception_info.get("exception_type") == "AgentTimeoutError" + or agent_metadata.get("agent_timed_out", False) + ) + agent_failed = bool( + agent_metadata.get("agent_failed", False) or failed_during_agent_phase or agent_timed_out + ) + model_connection_failed = bool(agent_metadata.get("model_connection_failed", False)) + failure_reason = agent_metadata.get("model_error") + if agent_timed_out: + failure_reason = exception_info.get("exception_message") or failure_reason + elif agent_failed and not failure_reason: + failure_reason = exception_info.get("exception_message") + mask_sample = bool(agent_failed or model_connection_failed or agent_timed_out) + failure_fields = { + "mask_sample": mask_sample, + "agent_failed": agent_failed, + "model_connection_failed": model_connection_failed, + "agent_timed_out": agent_timed_out, + "failure_reason": failure_reason, + } + if mask_sample: + reward = 0.0 + # Convert Harbor outputs to NeMo Gym response items: # keep rich trajectory details, then overlay rollout token details when present. output_items = HarborAgentUtils.trial_result_to_responses(trial_result, trajectory) @@ -316,6 +364,15 @@ def _read_trial_files(): input_messages = [] usage = None reward = 0.0 + failure_fields = {} + if self.config.harbor_skip_verification_on_agent_failure: + failure_fields = { + "mask_sample": True, + "agent_failed": True, + "model_connection_failed": False, + "agent_timed_out": False, + "failure_reason": str(e), + } response = HarborAgentUtils.get_default_response_object() response["model"] = policy_model_name @@ -341,6 +398,7 @@ def _read_trial_files(): agent_timeout_error=int( ((trial_result or {}).get("exception_info") or {}).get("exception_type") == "AgentTimeoutError" ), + **failure_fields, ) # Save result to disk (folder = run_id, file = task name) diff --git a/responses_api_agents/harbor_agent/tests/test_app.py b/responses_api_agents/harbor_agent/tests/test_app.py index f7404bb57d..ff75083a26 100644 --- a/responses_api_agents/harbor_agent/tests/test_app.py +++ b/responses_api_agents/harbor_agent/tests/test_app.py @@ -18,6 +18,7 @@ from contextlib import contextmanager from datetime import datetime, timezone from pathlib import Path +from types import SimpleNamespace from typing import Any, Dict, List, Optional from unittest.mock import MagicMock, patch @@ -29,6 +30,7 @@ HarborAgent, HarborAgentConfig, HarborRunRequest, + _stop_verification_after_agent_failure, ) from responses_api_agents.harbor_agent.utils import HarborAgentUtils @@ -309,7 +311,7 @@ async def _run_inline(fn, *args, **kwargs): mock_to_thread.side_effect = _run_inline - yield + yield mock_ray # =========================================================================== @@ -444,6 +446,89 @@ async def test_run_failed_execution(self): assert response.responses_create_params.temperature == 0.3 assert response.responses_create_params.input == [] + async def test_run_masks_partial_agent_failure_and_ignores_verifier_reward(self): + server = _make_server(harbor_skip_verification_on_agent_failure=True) + trial_result = { + **DEFAULT_TRIAL_RESULT, + "agent_result": { + **DEFAULT_TRIAL_RESULT["agent_result"], + "metadata": { + "agent_failed": True, + "model_connection_failed": True, + "agent_timed_out": False, + "model_error": "adapter failed after partial output", + }, + }, + "exception_info": { + "exception_type": "LegalAgentBenchHarnessError", + "exception_message": "adapter failed after partial output", + }, + "agent_execution": {"started_at": "2026-01-01T00:00:00Z"}, + "verifier": None, + } + with _harbor_run_mocks(trial_result=trial_result, trajectory=DEFAULT_TRAJECTORY) as mock_ray: + response = await server.run(_make_run_request()) + + assert response.response.output + assert response.reward == 0.0 + assert response.mask_sample is True + assert response.agent_failed is True + assert response.model_connection_failed is True + assert response.agent_timed_out is False + assert response.failure_reason == "adapter failed after partial output" + params = mock_ray.remote.call_args.args[1] + assert params["skip_verification_on_agent_failure"] is True + + async def test_verification_hook_rejects_agent_failure(self): + event = SimpleNamespace( + result=SimpleNamespace(exception_info=SimpleNamespace(exception_type="AgentTimeoutError")) + ) + + with pytest.raises(RuntimeError, match="Skipping verification after agent failure: AgentTimeoutError"): + await _stop_verification_after_agent_failure(event) + + await _stop_verification_after_agent_failure(SimpleNamespace(result=SimpleNamespace(exception_info=None))) + + async def test_run_masks_timeout_with_partial_output(self): + server = _make_server(harbor_skip_verification_on_agent_failure=True) + trial_result = { + **DEFAULT_TRIAL_RESULT, + "exception_info": { + "exception_type": "AgentTimeoutError", + "exception_message": "Agent execution timed out", + }, + "agent_execution": {"started_at": "2026-01-01T00:00:00Z"}, + "verifier": None, + } + with _harbor_run_mocks(trial_result=trial_result, trajectory=DEFAULT_TRAJECTORY): + response = await server.run(_make_run_request()) + + assert response.response.output + assert response.reward == 0.0 + assert response.mask_sample is True + assert response.agent_failed is True + assert response.agent_timed_out is True + assert response.failure_reason == "Agent execution timed out" + + async def test_default_harbor_timeout_contract_is_unchanged(self): + server = _make_server() + trial_result = { + **DEFAULT_TRIAL_RESULT, + "exception_info": { + "exception_type": "AgentTimeoutError", + "exception_message": "Agent execution timed out", + }, + "agent_execution": {"started_at": "2026-01-01T00:00:00Z"}, + "verifier": {"started_at": "2026-01-01T00:01:00Z"}, + } + with _harbor_run_mocks(trial_result=trial_result, trajectory=DEFAULT_TRAJECTORY): + response = await server.run(_make_run_request()) + + assert response.reward == 1.0 + assert response.agent_timeout_error == 1 + assert "mask_sample" not in response.model_dump() + assert "agent_failed" not in response.model_dump() + @pytest.mark.parametrize( "model_name, expected", [ From d2c890a1ff8f2363e7a4d061f916d4f4a23296b8 Mon Sep 17 00:00:00 2001 From: Ryan Stewart Date: Wed, 5 Aug 2026 09:32:46 -0400 Subject: [PATCH 13/26] route LAB operational failures for retry Signed-off-by: Ryan Stewart --- resources_servers/legal_agent_bench/README.md | 22 +- .../legal_agent_bench/legal_harbor_agent.py | 8 +- .../tests/test_harbor_integration.py | 101 ++++++++++ responses_api_agents/claude_code_agent/app.py | 7 +- .../claude_code_agent/tests/test_app.py | 25 +++ responses_api_agents/codex_agent/app.py | 37 +++- .../codex_agent/tests/test_app.py | 25 +++ responses_api_agents/harbor_agent/app.py | 189 +++++++++++++----- .../harbor_agent/tests/test_app.py | 121 ++++++++++- responses_api_agents/hermes_agent/app.py | 32 ++- .../hermes_agent/tests/test_app.py | 41 ++++ .../legal_agent_bench_agent/README.md | 11 +- .../legal_agent_bench_agent/app.py | 31 ++- .../legal_agent_bench_agent/tests/test_app.py | 93 ++++++++- .../legal_agent_bench_native_agent/app.py | 38 +++- .../tests/test_app.py | 9 +- 16 files changed, 698 insertions(+), 92 deletions(-) diff --git a/resources_servers/legal_agent_bench/README.md b/resources_servers/legal_agent_bench/README.md index f7e1d167eb..57ebeec918 100644 --- a/resources_servers/legal_agent_bench/README.md +++ b/resources_servers/legal_agent_bench/README.md @@ -280,8 +280,10 @@ jq . "$ARTIFACT_DIR/agent/trajectory.json" open "$ARTIFACT_DIR/verifier/report.html" # macOS; use xdg-open on Linux ``` -In `run_summary.json`, a reliable completed rollout has `mask_sample: false`, -all failure flags false, `judge_error_count: 0`, and `verifier_error: 0`. +In `run_summary.json`, a reliable scored rollout has `mask_sample: false`, all +failure flags false, `judge_error_count: 0`, and `verifier_error: 0`. This +includes incomplete max-turn or context-limit outcomes: their partial output is +still judged and saved in the main rollout JSONL. `output_files` lists the deliverables and `agent/trajectory.json` is the inner harness trace. A zero `full_task` reward can still be a valid rollout when one or more criteria fail; use `criteria_pass_rate` to see partial success. @@ -300,15 +302,19 @@ or more criteria fail; use `criteria_pass_rate` to see partial success. runner checks the policy endpoint from inside Docker. On Docker Desktop, derived loopback model URLs are automatically routed through `host.docker.internal`; `sandbox_model_base_url` is an explicit override. - Connectivity and harness failures are masked and skip judging. - Task-loading and harness-configuration failures are kept separate from Docker - sandbox failures. A zero reward without an infrastructure/judge flag is an - ordinary model/task result; a flagged result should be excluded from - model-quality comparisons. + Connectivity, harness, sandbox, and verifier failures are masked, skip + judging when applicable, and carry `_ng_failure_class`, which routes them to + the failure sidecar for bounded retry. Task-loading and harness-configuration + failures additionally carry `_ng_failure_terminal: true`, so they are not + retried. `mask_sample` is a training hint, not the routing signal. A zero + reward without an infrastructure/judge flag is an ordinary model/task result; + a flagged result should be excluded from model-quality comparisons. - Harbor agent, adapter, connection, and timeout failures preserve any partial trajectory but skip judging. Their result rows report `mask_sample`, `agent_failed`, `model_connection_failed`, `agent_timed_out`, and - `failure_reason`, and their reward is forced to zero. + `failure_reason`, carry `_ng_failure_class`, and have their reward forced to + zero. Harbor context-limit and max-turn stops are instead judged as valid + incomplete outcomes. - Hermes normally probes `/v1/models` and `/models` for optional pricing and context metadata. The LAB runner disables those lookups because Gym supplies the model explicitly and its internal policy proxy does not implement model diff --git a/resources_servers/legal_agent_bench/legal_harbor_agent.py b/resources_servers/legal_agent_bench/legal_harbor_agent.py index 08c4ff22b0..f2b516f838 100644 --- a/resources_servers/legal_agent_bench/legal_harbor_agent.py +++ b/resources_servers/legal_agent_bench/legal_harbor_agent.py @@ -414,13 +414,15 @@ async def _run_agent_async( except Exception as exc: err_msg = str(exc) _log_model_error(transcript_file, turn_count, exc) + if _is_context_overflow_error(err_msg): + # Context exhaustion is a valid incomplete model outcome. + # Preserve the partial artifacts and let Harbor verify them. + context_overflow = True + break model_error = err_msg model_error_type = type(exc).__name__ agent_timed_out = isinstance(exc, TimeoutError) model_connection_failed = isinstance(exc, (ConnectionError, OSError)) and not agent_timed_out - if _is_context_overflow_error(err_msg): - context_overflow = True - break break messages.append(response.message) diff --git a/resources_servers/legal_agent_bench/tests/test_harbor_integration.py b/resources_servers/legal_agent_bench/tests/test_harbor_integration.py index fe81c52e33..08fb8604ab 100644 --- a/resources_servers/legal_agent_bench/tests/test_harbor_integration.py +++ b/resources_servers/legal_agent_bench/tests/test_harbor_integration.py @@ -432,6 +432,107 @@ def get_metrics(self): assert result["model_connection_failed"] is False +@pytest.mark.asyncio +async def test_harbor_loop_preserves_context_limit_as_scoreable_incomplete_output(tmp_path) -> None: + class Adapter: + timeout_seconds = None + + def __init__(self): + self.calls = 0 + + def make_system_message(self, content): + return {"role": "system", "content": content} + + def make_user_message(self, content): + return {"role": "user", "content": content} + + def make_tool_result_messages(self, results): + return [{"role": "tool", "content": result} for _call_id, result in results] + + async def chat(self, _messages, _tools): + self.calls += 1 + if self.calls == 1: + return ModelResponse( + message={"role": "assistant", "content": "Partial work"}, + tool_calls=[SimpleNamespace(id="call-1", name="read", arguments='{"path":"input.docx"}')], + text="Partial work", + input_tokens=10, + output_tokens=2, + ) + raise RuntimeError("maximum context length exceeded") + + class ToolExecutor: + async def execute(self, _name, _arguments): + return "document text" + + def get_metrics(self): + return {} + + result = await _run_agent_async( + adapter=Adapter(), + system_prompt="system", + tool_executor=ToolExecutor(), + tools=[], + max_turns=3, + transcript_path=tmp_path / "transcript.jsonl", + ) + + assert result["context_overflow"] is True + assert result["finished_cleanly"] is False + assert result["model_error"] is None + assert result["model_connection_failed"] is False + assert result["agent_timed_out"] is False + assert result["output_tokens"] == 2 + + +@pytest.mark.asyncio +async def test_harbor_loop_preserves_max_turns_as_scoreable_incomplete_output(tmp_path) -> None: + class Adapter: + timeout_seconds = None + + def make_system_message(self, content): + return {"role": "system", "content": content} + + def make_user_message(self, content): + return {"role": "user", "content": content} + + def make_tool_result_messages(self, results): + return [{"role": "tool", "content": result} for _call_id, result in results] + + async def chat(self, _messages, _tools): + return ModelResponse( + message={"role": "assistant", "content": "Partial work"}, + tool_calls=[SimpleNamespace(id="call-1", name="read", arguments='{"path":"input.docx"}')], + text="Partial work", + input_tokens=10, + output_tokens=2, + ) + + class ToolExecutor: + async def execute(self, _name, _arguments): + return "document text" + + def get_metrics(self): + return {} + + result = await _run_agent_async( + adapter=Adapter(), + system_prompt="system", + tool_executor=ToolExecutor(), + tools=[], + max_turns=1, + transcript_path=tmp_path / "transcript.jsonl", + ) + + assert result["turn_count"] == 1 + assert result["finished_cleanly"] is False + assert result["context_overflow"] is False + assert result["model_error"] is None + assert result["model_connection_failed"] is False + assert result["agent_timed_out"] is False + assert result["output_tokens"] == 2 + + @pytest.mark.parametrize( "message, expected", [ diff --git a/responses_api_agents/claude_code_agent/app.py b/responses_api_agents/claude_code_agent/app.py index 3ce3de4be9..26268c1590 100644 --- a/responses_api_agents/claude_code_agent/app.py +++ b/responses_api_agents/claude_code_agent/app.py @@ -628,19 +628,22 @@ async def _create_response( input_tokens = run_metadata.get("input_tokens", 0) output_tokens = run_metadata.get("output_tokens", 0) run_status = str(run_metadata.get("status") or "incomplete") + limit_reached = run_status == "incomplete" and run_metadata.get("error_type") == "error_max_turns" failure_message = None - if run_status != "completed": + if run_status != "completed" and not limit_reached: failure_message = str(run_metadata.get("error_type") or run_status) return NeMoGymResponse( id=f"resp_{uuid4().hex}", created_at=int(time()), - status="failed" if failure_message else "completed", + status="incomplete" if limit_reached else ("failed" if failure_message else "completed"), error=( {"code": "server_error", "message": f"Claude Code failed: {failure_message}"} if failure_message else None ), + incomplete_details=({"reason": "max_output_tokens"} if limit_reached else None), + metadata=({"nemo_gym_stop_reason": "max_turns"} if limit_reached else None), model=model_name, object="response", output=output_items, diff --git a/responses_api_agents/claude_code_agent/tests/test_app.py b/responses_api_agents/claude_code_agent/tests/test_app.py index 972a7f9a07..435d279116 100644 --- a/responses_api_agents/claude_code_agent/tests/test_app.py +++ b/responses_api_agents/claude_code_agent/tests/test_app.py @@ -1056,3 +1056,28 @@ async def fake_run_claude_code(*args, **kwargs): assert response.status == "failed" assert response.error is not None assert "process_exit_1" in response.error.message + + +def test_claude_max_turn_output_is_a_scoreable_incomplete_outcome() -> None: + agent = _make_agent() + partial = NeMoGymResponseOutputMessage.model_validate( + { + "id": "msg-1", + "content": [{"type": "output_text", "text": "partial answer", "annotations": []}], + "role": "assistant", + "status": "completed", + "type": "message", + } + ) + + async def fake_run_claude_code(*args, **kwargs): + return [partial], "claude-test", {"status": "incomplete", "error_type": "error_max_turns"} + + object.__setattr__(agent, "_run_claude_code", fake_run_claude_code) + response = asyncio.run(agent._create_response(NeMoGymResponseCreateParamsNonStreaming(input="perform the task"))) + + assert response.output + assert response.status == "incomplete" + assert response.error is None + assert response.incomplete_details.reason == "max_output_tokens" + assert response.metadata == {"nemo_gym_stop_reason": "max_turns"} diff --git a/responses_api_agents/codex_agent/app.py b/responses_api_agents/codex_agent/app.py index 04f8032e6b..3065e4b6ed 100644 --- a/responses_api_agents/codex_agent/app.py +++ b/responses_api_agents/codex_agent/app.py @@ -240,6 +240,22 @@ def _add_tool_pair(item: dict[str, Any], name: str, arguments: dict[str, Any], o return output_items, metadata +def _is_context_limit_error(message: str) -> bool: + normalized = message.lower() + return any( + marker in normalized + for marker in ( + "context length", + "context window", + "context_length_exceeded", + "maximum context", + "max context", + "too many tokens", + "token limit", + ) + ) + + def _kill_process_group(proc: Any) -> None: """Kill the codex subprocess and every child in its process group. @@ -597,11 +613,11 @@ async def _create_response( if usage.get("errors"): LOG.warning("codex reported errors: %s", usage["errors"]) - run_metadata.update( - status="failed", - error_type="stream_error", - stderr="; ".join(str(error) for error in usage["errors"]), - ) + stream_error = "; ".join(str(error) for error in usage["errors"]) + if all(_is_context_limit_error(str(error)) for error in usage["errors"]): + run_metadata.update(status="incomplete", error_type="context_limit", stderr=stream_error) + else: + run_metadata.update(status="failed", error_type="stream_error", stderr=stream_error) if not any( getattr(item, "type", None) == "message" and getattr(item, "role", None) == "assistant" @@ -620,8 +636,13 @@ async def _create_response( input_tokens = usage.get("input_tokens", 0) output_tokens = usage.get("output_tokens", 0) + run_status = run_metadata.get("status") + run_error = " ".join( + str(value) for value in (run_metadata.get("error_type"), run_metadata.get("stderr")) if value + ) + limit_reached = run_status != "completed" and _is_context_limit_error(run_error) failure_message = None - if run_metadata.get("status") != "completed": + if run_status != "completed" and not limit_reached: failure_message = str(run_metadata.get("error_type") or "codex_failed") stderr = str(run_metadata.get("stderr") or "").strip() if stderr: @@ -630,10 +651,12 @@ async def _create_response( return NeMoGymResponse( id=f"resp_{uuid4().hex}", created_at=int(time()), - status="failed" if failure_message else "completed", + status="incomplete" if limit_reached else ("failed" if failure_message else "completed"), error=( {"code": "server_error", "message": f"Codex failed: {failure_message}"} if failure_message else None ), + incomplete_details=({"reason": "max_output_tokens"} if limit_reached else None), + metadata=({"nemo_gym_stop_reason": "context_limit"} if limit_reached else None), model=model_name, object="response", output=output_items, diff --git a/responses_api_agents/codex_agent/tests/test_app.py b/responses_api_agents/codex_agent/tests/test_app.py index 4dd596d3d7..264dceb789 100644 --- a/responses_api_agents/codex_agent/tests/test_app.py +++ b/responses_api_agents/codex_agent/tests/test_app.py @@ -886,3 +886,28 @@ async def fake_run_codex(*args, **kwargs): assert response.status == "failed" assert response.error is not None assert "stream_error" in response.error.message + + +@pytest.mark.parametrize("reported_in", ["stdout", "stderr"]) +def test_codex_context_limit_output_is_a_scoreable_incomplete_outcome(reported_in: str) -> None: + agent = _make_agent() + partial = _item_completed({"id": "msg-1", "type": "agent_message", "text": "partial answer"}) + context_error = "maximum context length exceeded" + stdout = partial + metadata = {"status": "failed", "error_type": "process_exit_1", "stderr": ""} + if reported_in == "stdout": + stdout += "\n" + json.dumps({"type": "turn.failed", "error": {"message": context_error}}) + else: + metadata["stderr"] = context_error + + async def fake_run_codex(*args, **kwargs): + return stdout, "codex-default", metadata + + object.__setattr__(agent, "_run_codex", fake_run_codex) + response = asyncio.run(agent._create_response(NeMoGymResponseCreateParamsNonStreaming(input="perform the task"))) + + assert response.output + assert response.status == "incomplete" + assert response.error is None + assert response.incomplete_details.reason == "max_output_tokens" + assert response.metadata == {"nemo_gym_stop_reason": "context_limit"} diff --git a/responses_api_agents/harbor_agent/app.py b/responses_api_agents/harbor_agent/app.py index 67e4aca48e..9ef748e9be 100644 --- a/responses_api_agents/harbor_agent/app.py +++ b/responses_api_agents/harbor_agent/app.py @@ -43,6 +43,7 @@ NeMoGymResponse, NeMoGymResponseCreateParamsNonStreaming, ) +from nemo_gym.rollout_collection import NG_FAILURE_CLASS_KEY, NG_TERMINAL_KEY from responses_api_agents.harbor_agent.utils import HarborAgentUtils @@ -250,17 +251,66 @@ def setup_webserver(self) -> FastAPI: async def responses(self, body: NeMoGymResponseCreateParamsNonStreaming = Body()) -> NeMoGymResponse: raise NotImplementedError + def _terminal_failure_response( + self, + body: HarborRunRequest, + *, + failure_class: str, + reason: str, + model_name: str, + ) -> HarborVerifyResponse: + response = HarborAgentUtils.get_default_response_object() + response["model"] = model_name + response["output"] = [] + return HarborVerifyResponse.model_validate( + { + "responses_create_params": body.responses_create_params.model_dump(mode="json"), + "reward": 0.0, + "response": response, + "instance_id": body.instance_id, + "mask_sample": True, + "task_failed": failure_class == "task_failed", + "configuration_failed": failure_class == "configuration_failed", + "failure_reason": reason, + NG_FAILURE_CLASS_KEY: failure_class, + NG_TERMINAL_KEY: True, + } + ) + async def run(self, body: HarborRunRequest) -> HarborVerifyResponse: async with self.sem: - global_config_dict = get_global_config_dict() - - policy_model_name = global_config_dict["policy_model_name"] - base_url = self._resolve_model_base_url(global_config_dict) + try: + global_config_dict = get_global_config_dict() + policy_model_name = global_config_dict["policy_model_name"] + base_url = self._resolve_model_base_url(global_config_dict) + except (KeyError, TypeError, ValueError) as exc: + return self._terminal_failure_response( + body, + failure_class="configuration_failed", + reason=f"{type(exc).__name__}: {exc}", + model_name=self.config.model_server.name, + ) run_timestamp = datetime.now(timezone.utc) run_id = self._build_run_id(run_timestamp) instance_id = body.instance_id - dataset_alias, task_name = self._parse_instance_id(instance_id) + try: + dataset_alias, task_name = self._parse_instance_id(instance_id) + except ValueError as exc: + return self._terminal_failure_response( + body, + failure_class="task_failed", + reason=f"{type(exc).__name__}: {exc}", + model_name=policy_model_name, + ) + if dataset_alias not in self.config.harbor_datasets: + available = ", ".join(sorted(self.config.harbor_datasets)) + return self._terminal_failure_response( + body, + failure_class="task_failed", + reason=f"Unknown dataset alias in instance_id: {dataset_alias!r}. Available aliases: [{available}]", + model_name=policy_model_name, + ) output_file_dir = self._get_results_output_dir(policy_model_name, dataset_alias, run_timestamp) jobs_dir = self._get_jobs_output_dir(policy_model_name, dataset_alias, run_timestamp) @@ -271,15 +321,23 @@ async def run(self, body: HarborRunRequest) -> HarborVerifyResponse: exclude_none=True, ) - job_config_dict = self._build_job_config( - dataset_alias, - task_name, - policy_model_name, - base_url, - job_name=job_name, - jobs_dir=jobs_dir, - responses_create_params=responses_create_params, - ) + try: + job_config_dict = self._build_job_config( + dataset_alias, + task_name, + policy_model_name, + base_url, + job_name=job_name, + jobs_dir=jobs_dir, + responses_create_params=responses_create_params, + ) + except (KeyError, TypeError, ValueError) as exc: + return self._terminal_failure_response( + body, + failure_class="configuration_failed", + reason=f"{type(exc).__name__}: {exc}", + model_name=policy_model_name, + ) try: params = dict( @@ -313,37 +371,71 @@ def _read_trial_files(): reward = HarborAgentUtils.extract_reward(verifier_result) failure_fields = {} - if self.config.harbor_skip_verification_on_agent_failure: - exception_info = trial_result.get("exception_info") or {} - agent_metadata = (trial_result.get("agent_result") or {}).get("metadata") or {} - failed_during_agent_phase = bool( - exception_info - and trial_result.get("agent_execution") is not None - and trial_result.get("verifier") is None - ) - agent_timed_out = bool( - exception_info.get("exception_type") == "AgentTimeoutError" - or agent_metadata.get("agent_timed_out", False) - ) - agent_failed = bool( - agent_metadata.get("agent_failed", False) or failed_during_agent_phase or agent_timed_out - ) - model_connection_failed = bool(agent_metadata.get("model_connection_failed", False)) - failure_reason = agent_metadata.get("model_error") - if agent_timed_out: - failure_reason = exception_info.get("exception_message") or failure_reason - elif agent_failed and not failure_reason: - failure_reason = exception_info.get("exception_message") - mask_sample = bool(agent_failed or model_connection_failed or agent_timed_out) + exception_info = trial_result.get("exception_info") or {} + agent_metadata = (trial_result.get("agent_result") or {}).get("metadata") or {} + context_limit_reached = bool(agent_error_flags.get("context_length_exceeded", False)) + memory_limit_reached = bool(agent_error_flags.get("memory_limit_exceeded", False)) + agent_started = trial_result.get("agent_execution") is not None + verifier_started = trial_result.get("verifier") is not None + failed_during_agent_phase = bool(exception_info and agent_started and not verifier_started) + agent_timed_out = bool( + exception_info.get("exception_type") == "AgentTimeoutError" + or agent_metadata.get("agent_timed_out", False) + ) + agent_failed = bool( + agent_metadata.get("agent_failed", False) or failed_during_agent_phase or agent_timed_out + ) + model_connection_failed = bool(agent_metadata.get("model_connection_failed", False)) + verifier_failed = bool( + (exception_info and verifier_started and not agent_failed) + or (not agent_failed and not context_limit_reached and verifier_result is None) + ) + sandbox_failed = bool( + memory_limit_reached or (exception_info and not agent_started and not verifier_started) + ) + + # Harbor exposes context exhaustion separately in + # context_length_exceeded_error. It is a scoreable incomplete + # outcome, not an operational failure. + if context_limit_reached: + agent_failed = False + model_connection_failed = False + agent_timed_out = False + + failure_reason = agent_metadata.get("model_error") + if agent_timed_out or verifier_failed or sandbox_failed: + failure_reason = exception_info.get("exception_message") or failure_reason + if memory_limit_reached and not failure_reason: + failure_reason = "Harbor sandbox exceeded its memory limit" + elif agent_failed and not failure_reason: + failure_reason = exception_info.get("exception_message") + + failure_class = None + if model_connection_failed: + failure_class = "model_connection_failed" + elif sandbox_failed: + failure_class = "sandbox_failed" + elif verifier_failed: + failure_class = "verifier_failed" + elif agent_timed_out: + failure_class = "agent_timed_out" + elif agent_failed: + failure_class = "agent_failed" + + mask_sample = failure_class is not None + if self.config.harbor_skip_verification_on_agent_failure or mask_sample: failure_fields = { "mask_sample": mask_sample, "agent_failed": agent_failed, "model_connection_failed": model_connection_failed, "agent_timed_out": agent_timed_out, + "verifier_failed": verifier_failed, + "sandbox_failed": sandbox_failed, "failure_reason": failure_reason, } - if mask_sample: - reward = 0.0 + if failure_class is not None: + failure_fields[NG_FAILURE_CLASS_KEY] = failure_class + reward = 0.0 # Convert Harbor outputs to NeMo Gym response items: # keep rich trajectory details, then overlay rollout token details when present. @@ -365,14 +457,16 @@ def _read_trial_files(): usage = None reward = 0.0 failure_fields = {} - if self.config.harbor_skip_verification_on_agent_failure: - failure_fields = { - "mask_sample": True, - "agent_failed": True, - "model_connection_failed": False, - "agent_timed_out": False, - "failure_reason": str(e), - } + failure_fields = { + "mask_sample": True, + "agent_failed": False, + "model_connection_failed": False, + "agent_timed_out": False, + "verifier_failed": False, + "sandbox_failed": True, + "failure_reason": str(e), + NG_FAILURE_CLASS_KEY: "sandbox_failed", + } response = HarborAgentUtils.get_default_response_object() response["model"] = policy_model_name @@ -396,7 +490,8 @@ def _read_trial_files(): context_length_exceeded_error=int(agent_error_flags.get("context_length_exceeded", False)), memory_limit_exceeded_error=int(agent_error_flags.get("memory_limit_exceeded", False)), agent_timeout_error=int( - ((trial_result or {}).get("exception_info") or {}).get("exception_type") == "AgentTimeoutError" + not agent_error_flags.get("context_length_exceeded", False) + and ((trial_result or {}).get("exception_info") or {}).get("exception_type") == "AgentTimeoutError" ), **failure_fields, ) diff --git a/responses_api_agents/harbor_agent/tests/test_app.py b/responses_api_agents/harbor_agent/tests/test_app.py index ff75083a26..79c4a057f5 100644 --- a/responses_api_agents/harbor_agent/tests/test_app.py +++ b/responses_api_agents/harbor_agent/tests/test_app.py @@ -275,6 +275,7 @@ def _make_run_request(instance_id="scientific::test_task_123", **kwargs) -> Harb def _harbor_run_mocks( trial_result: Optional[Dict[str, Any]] = None, trajectory: Optional[Dict[str, Any]] = None, + agent_error_flags: Optional[Dict[str, Any]] = None, side_effect: Optional[Exception] = None, ): """Patch external deps and wire up mocks for HarborAgent.run().""" @@ -298,10 +299,13 @@ async def _resolve_future(): else: trial_dir = tempfile.mkdtemp(prefix="harbor_trial_") (Path(trial_dir) / "result.json").write_text(json.dumps(trial_result or DEFAULT_TRIAL_RESULT)) - if trajectory is not None: + if trajectory is not None or agent_error_flags is not None: agent_dir = Path(trial_dir) / "agent" agent_dir.mkdir(parents=True, exist_ok=True) + if trajectory is not None: (agent_dir / "trajectory.json").write_text(json.dumps(trajectory)) + if agent_error_flags is not None: + (agent_dir / "agent_error_flags.json").write_text(json.dumps(agent_error_flags)) mock_ray.remote.side_effect = lambda *a, **k: _resolve_future() mock_ray.options.return_value.remote.side_effect = lambda *a, **k: _resolve_future() @@ -445,6 +449,10 @@ async def test_run_failed_execution(self): assert len(response.response.output) == 0 assert response.responses_create_params.temperature == 0.3 assert response.responses_create_params.input == [] + assert response.mask_sample is True + assert response.sandbox_failed is True + assert response.model_dump()["_ng_failure_class"] == "sandbox_failed" + assert "_ng_failure_terminal" not in response.model_dump() async def test_run_masks_partial_agent_failure_and_ignores_verifier_reward(self): server = _make_server(harbor_skip_verification_on_agent_failure=True) @@ -476,6 +484,8 @@ async def test_run_masks_partial_agent_failure_and_ignores_verifier_reward(self) assert response.model_connection_failed is True assert response.agent_timed_out is False assert response.failure_reason == "adapter failed after partial output" + assert response.model_dump()["_ng_failure_class"] == "model_connection_failed" + assert "_ng_failure_terminal" not in response.model_dump() params = mock_ray.remote.call_args.args[1] assert params["skip_verification_on_agent_failure"] is True @@ -509,8 +519,10 @@ async def test_run_masks_timeout_with_partial_output(self): assert response.agent_failed is True assert response.agent_timed_out is True assert response.failure_reason == "Agent execution timed out" + assert response.model_dump()["_ng_failure_class"] == "agent_timed_out" + assert "_ng_failure_terminal" not in response.model_dump() - async def test_default_harbor_timeout_contract_is_unchanged(self): + async def test_harbor_timeout_routes_to_retryable_failure_sidecar(self): server = _make_server() trial_result = { **DEFAULT_TRIAL_RESULT, @@ -524,10 +536,111 @@ async def test_default_harbor_timeout_contract_is_unchanged(self): with _harbor_run_mocks(trial_result=trial_result, trajectory=DEFAULT_TRAJECTORY): response = await server.run(_make_run_request()) - assert response.reward == 1.0 + assert response.reward == 0.0 assert response.agent_timeout_error == 1 + assert response.mask_sample is True + assert response.agent_failed is True + assert response.agent_timed_out is True + assert response.verifier_failed is False + assert response.model_dump()["_ng_failure_class"] == "agent_timed_out" + assert "_ng_failure_terminal" not in response.model_dump() + + async def test_harbor_verifier_failure_routes_to_retryable_failure_sidecar(self): + server = _make_server() + trial_result = { + **DEFAULT_TRIAL_RESULT, + "verifier_result": None, + "exception_info": { + "exception_type": "VerifierTimeoutError", + "exception_message": "Verifier execution timed out", + }, + "agent_execution": {"started_at": "2026-01-01T00:00:00Z"}, + "verifier": {"started_at": "2026-01-01T00:01:00Z"}, + } + with _harbor_run_mocks(trial_result=trial_result, trajectory=DEFAULT_TRAJECTORY): + response = await server.run(_make_run_request()) + + assert response.reward == 0.0 + assert response.mask_sample is True + assert response.agent_failed is False + assert response.verifier_failed is True + assert response.failure_reason == "Verifier execution timed out" + assert response.model_dump()["_ng_failure_class"] == "verifier_failed" + assert "_ng_failure_terminal" not in response.model_dump() + + async def test_harbor_context_limit_is_verified_and_scored(self): + server = _make_server() + trial_result = { + **DEFAULT_TRIAL_RESULT, + "exception_info": { + "exception_type": "AgentTimeoutError", + "exception_message": "maximum context length exceeded", + }, + "agent_execution": {"started_at": "2026-01-01T00:00:00Z"}, + "verifier": {"started_at": "2026-01-01T00:01:00Z"}, + } + with _harbor_run_mocks( + trial_result=trial_result, + trajectory=DEFAULT_TRAJECTORY, + agent_error_flags={"context_length_exceeded": True, "memory_limit_exceeded": False}, + ): + response = await server.run(_make_run_request()) + + assert response.reward == 1.0 + assert response.context_length_exceeded_error == 1 + assert response.agent_timeout_error == 0 + assert response.response.output assert "mask_sample" not in response.model_dump() - assert "agent_failed" not in response.model_dump() + assert "_ng_failure_class" not in response.model_dump() + assert "_ng_failure_terminal" not in response.model_dump() + + async def test_harbor_memory_limit_routes_as_retryable_sandbox_failure(self): + server = _make_server() + trial_result = { + **DEFAULT_TRIAL_RESULT, + "verifier_result": None, + "agent_execution": {"started_at": "2026-01-01T00:00:00Z"}, + "verifier": None, + } + with _harbor_run_mocks( + trial_result=trial_result, + trajectory=DEFAULT_TRAJECTORY, + agent_error_flags={"context_length_exceeded": False, "memory_limit_exceeded": True}, + ): + response = await server.run(_make_run_request()) + + assert response.reward == 0.0 + assert response.mask_sample is True + assert response.sandbox_failed is True + assert response.memory_limit_exceeded_error == 1 + assert response.failure_reason == "Harbor sandbox exceeded its memory limit" + assert response.model_dump()["_ng_failure_class"] == "sandbox_failed" + assert "_ng_failure_terminal" not in response.model_dump() + + async def test_invalid_instance_id_is_terminal(self): + server = _make_server() + with _harbor_run_mocks(): + response = await server.run(_make_run_request(instance_id="not-an-instance-id")) + + assert response.reward == 0.0 + assert response.mask_sample is True + assert response.task_failed is True + assert response.model_dump()["_ng_failure_class"] == "task_failed" + assert response.model_dump()["_ng_failure_terminal"] is True + + async def test_invalid_job_configuration_is_terminal(self): + server = _make_server() + with ( + patch("responses_api_agents.harbor_agent.app.get_global_config_dict", return_value=_GLOBAL_CONFIG), + patch.object(server, "_build_job_config", side_effect=ValueError("invalid dataset source")), + ): + response = await server.run(_make_run_request()) + + assert response.reward == 0.0 + assert response.mask_sample is True + assert response.configuration_failed is True + assert response.model_dump()["_ng_failure_class"] == "configuration_failed" + assert response.model_dump()["_ng_failure_terminal"] is True @pytest.mark.parametrize( "model_name, expected", diff --git a/responses_api_agents/hermes_agent/app.py b/responses_api_agents/hermes_agent/app.py index bf4994a309..03cebcec5e 100644 --- a/responses_api_agents/hermes_agent/app.py +++ b/responses_api_agents/hermes_agent/app.py @@ -57,6 +57,22 @@ from responses_api_agents.hermes_agent.observability import HermesAgentObserver +def _is_context_limit_error(message: str) -> bool: + normalized = message.lower() + return any( + marker in normalized + for marker in ( + "context length", + "context window", + "context_length_exceeded", + "maximum context", + "max context", + "too many tokens", + "token limit", + ) + ) + + def _usage_from_result(result: dict[str, Any]) -> NeMoGymResponseUsage: """Map Hermes' cumulative session counters to the Responses API usage schema.""" @@ -428,11 +444,23 @@ def _patched_build_api_kwargs(api_messages): ) agent_error = result.get("error") + context_limit_reached = bool(agent_error) and _is_context_limit_error(str(agent_error)) + max_turns_reached = bool( + not agent_error + and result.get("completed") is False + and int(result.get("api_calls") or 0) >= self.config.max_turns + ) + limit_reached = context_limit_reached or max_turns_reached + stop_reason = "context_limit" if context_limit_reached else "max_turns" return NeMoGymResponse( id=f"resp_{uuid4().hex}", created_at=int(time()), - status="failed" if agent_error else "completed", - error=({"code": "server_error", "message": str(agent_error)} if agent_error else None), + status="incomplete" if limit_reached else ("failed" if agent_error else "completed"), + error=( + {"code": "server_error", "message": str(agent_error)} if agent_error and not limit_reached else None + ), + incomplete_details=({"reason": "max_output_tokens"} if limit_reached else None), + metadata=({"nemo_gym_stop_reason": stop_reason} if limit_reached else None), model=model_name, object="response", output=output_items, diff --git a/responses_api_agents/hermes_agent/tests/test_app.py b/responses_api_agents/hermes_agent/tests/test_app.py index f0fcab673e..0ac820150c 100644 --- a/responses_api_agents/hermes_agent/tests/test_app.py +++ b/responses_api_agents/hermes_agent/tests/test_app.py @@ -397,6 +397,47 @@ def run_conversation(self, *args, **kwargs) -> dict: assert response.error is not None assert "stream disconnected" in response.error.message + @pytest.mark.parametrize( + ("result_update", "stop_reason"), + [ + ({"completed": False, "api_calls": 30}, "max_turns"), + ({"error": "maximum context length exceeded"}, "context_limit"), + ], + ) + def test_limit_output_is_a_scoreable_incomplete_outcome(self, monkeypatch, result_update, stop_reason) -> None: + import nemo_gym.base_responses_api_agent as base_agent + from nemo_gym.openai_utils import NeMoGymResponseCreateParamsNonStreaming + + monkeypatch.setattr(base_agent, "get_first_server_config_dict", lambda _gc, _name: {"host": "h", "port": 1}) + server_client = MagicMock(spec=ServerClient) + server_client.global_config_dict = {} + server_client._build_server_base_url = lambda _cfg: "http://h:1" + agent = HermesAgent(config=_config(max_turns=30), server_client=server_client) + monkeypatch.setattr(agent, "_ensure_sigterm_handler", lambda: None) + + class _StubAIAgent: + def __init__(self, **kwargs) -> None: + self._build_api_kwargs = lambda _messages: {} + self.compression_enabled = True + + def run_conversation(self, *args, **kwargs) -> dict: + return { + "messages": [{"role": "assistant", "content": "partial answer"}], + "prompt_tokens": 12, + "output_tokens": 3, + "total_tokens": 15, + **result_update, + } + + monkeypatch.setattr("run_agent.AIAgent", _StubAIAgent) + response = asyncio.run(agent.responses(request=None, body=NeMoGymResponseCreateParamsNonStreaming(input="hi"))) + + assert response.output + assert response.status == "incomplete" + assert response.error is None + assert response.incomplete_details.reason == "max_output_tokens" + assert response.metadata == {"nemo_gym_stop_reason": stop_reason} + class TestObservability: @pytest.mark.parametrize( diff --git a/responses_api_agents/legal_agent_bench_agent/README.md b/responses_api_agents/legal_agent_bench_agent/README.md index 52dc8cbf31..9cb00f8820 100644 --- a/responses_api_agents/legal_agent_bench_agent/README.md +++ b/responses_api_agents/legal_agent_bench_agent/README.md @@ -59,9 +59,14 @@ proxy as `LAB_POLICY_MODEL_URL`, keeping model credentials out of the sandbox. An explicit `sandbox_model_base_url` is used unchanged and disables that ECS tunnel. Before importing the selected harness, the in-container runner performs a bounded TCP connectivity check and writes -`runtime/runner_status.json`. Connectivity or harness failures set the -`mask_sample` flag on the row, skip the judge, and are distinct from an ordinary zero-reward -model result. For Hermes, the runner also disables its optional `/models` +`runtime/runner_status.json`. Connectivity, harness, sandbox, and verifier +failures set `mask_sample`, skip the judge when applicable, and carry +`_ng_failure_class` so rollout collection sends them to the failure sidecar for +bounded retry. Deterministic task and harness-configuration failures also carry +`_ng_failure_terminal: true` and are not retried. `mask_sample` is a training +hint, not the routing signal. Max-turn and context-limit stops are valid +incomplete outcomes: the runner verifies their partial deliverables and keeps +them in the main rollout JSONL. For Hermes, the runner also disables its optional `/models` pricing and context-metadata probes; Gym supplies the selected model explicitly, and actual model-call access logging is enabled. diff --git a/responses_api_agents/legal_agent_bench_agent/app.py b/responses_api_agents/legal_agent_bench_agent/app.py index 2aa152ddd1..bddae82d18 100644 --- a/responses_api_agents/legal_agent_bench_agent/app.py +++ b/responses_api_agents/legal_agent_bench_agent/app.py @@ -33,6 +33,7 @@ from nemo_gym.config_types import ModelServerRef, ResourcesServerRef from nemo_gym.global_config import get_first_server_config_dict from nemo_gym.openai_utils import NeMoGymResponse, NeMoGymResponseCreateParamsNonStreaming +from nemo_gym.rollout_collection import NG_FAILURE_CLASS_KEY, NG_TERMINAL_KEY from nemo_gym.sandbox import ( AsyncSandbox, SandboxResources, @@ -636,8 +637,11 @@ def agent_response_failure(response: NeMoGymResponse, agent_server_module: str) """Return a harness-failure reason without treating normal task-quality failures as infrastructure.""" if response.error is not None: return f"Agent returned an error response: {response.error}" + # Max-turn and context-limit stops are valid incomplete model outcomes. The + # harnesses represent those with ``incomplete_details`` so LAB can still + # verify and score whatever artifacts the agent produced. if response.incomplete_details is not None: - return f"Agent returned an incomplete response: {response.incomplete_details}" + return None if not response.output: return "Agent produced an empty trajectory" @@ -1379,6 +1383,30 @@ def _response( or verifier_error or judge_errors ) + failure_class: Optional[str] = None + failure_terminal = False + if task_failed: + failure_class = "task_failed" + failure_terminal = True + elif configuration_failed: + failure_class = "configuration_failed" + failure_terminal = True + elif model_connection_failed: + failure_class = "model_connection_failed" + elif sandbox_failed: + failure_class = "sandbox_failed" + elif verifier_failed or verifier_timed_out or verifier_error or judge_errors: + failure_class = "verifier_failed" + elif agent_timed_out: + failure_class = "agent_timed_out" + elif agent_failed: + failure_class = "agent_failed" + + routing: dict[str, Any] = {} + if failure_class is not None: + routing[NG_FAILURE_CLASS_KEY] = failure_class + if failure_terminal: + routing[NG_TERMINAL_KEY] = True return LegalAgentBenchAgentResponse( responses_create_params=params, response=response, @@ -1408,6 +1436,7 @@ def _response( else None ), output_dir=str(paths["output"]) if paths else None, + **routing, ) async def run(self, request: Request, body: LegalAgentBenchRunRequest) -> LegalAgentBenchAgentResponse: diff --git a/responses_api_agents/legal_agent_bench_agent/tests/test_app.py b/responses_api_agents/legal_agent_bench_agent/tests/test_app.py index 5ee076d4ca..0a9b4153b8 100644 --- a/responses_api_agents/legal_agent_bench_agent/tests/test_app.py +++ b/responses_api_agents/legal_agent_bench_agent/tests/test_app.py @@ -163,9 +163,7 @@ def test_dependency_runtime_cache_is_harness_and_recipe_specific(monkeypatch, tm def fake_run(args, **kwargs): if len(args) > 1 and args[1] == "run": - mounted_bundle = next( - Path(arg.split(":", 1)[0]) for arg in args if ":/nemo_gym_mount:ro" in arg - ) + mounted_bundle = next(Path(arg.split(":", 1)[0]) for arg in args if ":/nemo_gym_mount:ro" in arg) assert mounted_bundle != tmp_path assert not (mounted_bundle / "env.yaml").exists() calls.append((args, kwargs)) @@ -637,7 +635,7 @@ def test_stage_agent_source_copies_only_selected_runtime_package(monkeypatch, tm (package / "runtime_helpers").mkdir() (package / "app.py").write_text("VALUE = 1\n") (package / "runtime_helpers" / "tool.py").write_text("VALUE = 2\n") - (package / "tests" / "rubric.py").write_text("SECRET = True\n") + (package / "tests" / "rubric.py").write_text("SECRET = True\n") # pragma: allowlist secret (package / "data" / "example.jsonl").write_text("{}\n") (repository / "resources_servers" / "legal_agent_bench" / "data" / "runtime").mkdir(parents=True) (repository / "resources_servers" / "legal_agent_bench" / "data" / "runtime" / "rubric.json").write_text("{}") @@ -803,6 +801,44 @@ def test_response_masks_harness_and_verifier_failures() -> None: assert response.model_connection_failed is True assert response.verifier_failed is True assert response.mask_sample is True + assert response.model_dump()["_ng_failure_class"] == "model_connection_failed" + assert "_ng_failure_terminal" not in response.model_dump() + + +@pytest.mark.parametrize( + ("failure_flag", "expected_class", "terminal"), + [ + ("agent_failed", "agent_failed", False), + ("model_connection_failed", "model_connection_failed", False), + ("agent_timed_out", "agent_timed_out", False), + ("verifier_failed", "verifier_failed", False), + ("verifier_timed_out", "verifier_failed", False), + ("sandbox_failed", "sandbox_failed", False), + ("task_failed", "task_failed", True), + ("configuration_failed", "configuration_failed", True), + ], +) +def test_response_routes_every_failure_class(failure_flag, expected_class, terminal) -> None: + runner = app.LegalAgentBenchAgent.model_construct(config=_config()) + params = NeMoGymResponseCreateParamsNonStreaming(input=[]) + body = app.LegalAgentBenchRunRequest( + instance_id="legal_agent_bench::area__task", + responses_create_params=params, + ) + + response = runner._response( + body=body, + params=params, + response=app._empty_response("policy"), + reward_data={}, + paths=None, + **{failure_flag: True}, + ) + response_data = response.model_dump() + + assert response.mask_sample is True + assert response_data["_ng_failure_class"] == expected_class + assert response_data.get("_ng_failure_terminal", False) is terminal @pytest.mark.parametrize( @@ -834,6 +870,8 @@ def test_response_masks_malformed_verifier_metrics_instead_of_raising(reward_dat assert response.verifier_failed is True assert response.mask_sample is True assert response.failure_reason.startswith("Invalid verifier") + assert response.model_dump()["_ng_failure_class"] == "verifier_failed" + assert "_ng_failure_terminal" not in response.model_dump() @pytest.mark.asyncio @@ -850,6 +888,8 @@ async def test_run_classifies_invalid_task_without_sandbox_failure(monkeypatch) assert response.task_failed is True assert response.configuration_failed is False + assert response.model_dump()["_ng_failure_class"] == "task_failed" + assert response.model_dump()["_ng_failure_terminal"] is True assert response.sandbox_failed is False assert response.mask_sample is True assert "Unsafe Legal Agent Bench task name" in response.failure_reason @@ -887,6 +927,8 @@ async def reject_runtime(_image): assert response.task_failed is False assert response.sandbox_failed is False assert response.mask_sample is True + assert response.model_dump()["_ng_failure_class"] == "configuration_failed" + assert response.model_dump()["_ng_failure_terminal"] is True assert "missing configured CLI pin" in response.failure_reason @@ -1088,8 +1130,23 @@ def fake_sandbox(provider, spec): (b'{"reward": "invalid", "criteria_pass_rate": 1.0}', 0.0, True), ], ) -async def test_run_stops_agent_sandbox_before_starting_verifier_sandbox( - monkeypatch, tmp_path, capsys, reward_blob, expected_reward, expected_verifier_failed +@pytest.mark.parametrize( + "agent_module", + [ + "responses_api_agents.legal_agent_bench_native_agent.app", + "responses_api_agents.hermes_agent.app", + "responses_api_agents.claude_code_agent.app", + "responses_api_agents.codex_agent.app", + ], +) +async def test_incomplete_limit_outcomes_are_verified_for_every_harness( + monkeypatch, + tmp_path, + capsys, + reward_blob, + expected_reward, + expected_verifier_failed, + agent_module, ) -> None: _root, task = _task_tree(tmp_path) skills = _skills(tmp_path) @@ -1115,7 +1172,7 @@ async def stop(self): agent_sandbox = PhaseSandbox("agent") verifier_sandbox = PhaseSandbox("verifier") - runner = app.LegalAgentBenchAgent.model_construct(config=_config()) + runner = app.LegalAgentBenchAgent.model_construct(config=_config(agent_server_module=agent_module)) runner._sem = app.asyncio.Semaphore(1) runner._session_results_dir = tmp_path / "results" @@ -1126,7 +1183,15 @@ async def ensure_runtime(_image): return deps def write_runner(paths, params, model_url): - (paths["runtime"] / "response.json").write_text(_successful_response().model_dump_json()) + incomplete = app.NeMoGymResponse.model_validate( + _successful_response().model_dump(mode="json") + | { + "status": "incomplete", + "incomplete_details": {"reason": "max_output_tokens"}, + "metadata": {"nemo_gym_stop_reason": "max_turns"}, + } + ) + (paths["runtime"] / "response.json").write_text(incomplete.model_dump_json()) async def run_verifier(sandbox, task_dir, paths): assert sandbox is verifier_sandbox @@ -1142,7 +1207,7 @@ async def collect_agent(*args, **kwargs): monkeypatch.setattr(app, "resolve_task_dir", lambda runtime, instance: task) monkeypatch.setattr(app, "resolve_repo_path", lambda path: skills) - monkeypatch.setattr(app, "compose_agent_input", lambda task_dir, skills_dir, params: params) + monkeypatch.setattr(app, "compose_agent_input", lambda task_dir, skills_dir, params, **kwargs: params) monkeypatch.setattr(runner, "_ensure_image", ensure_image) monkeypatch.setattr(runner, "_ensure_runtime", ensure_runtime) monkeypatch.setattr(runner, "_stage_agent_source", lambda paths: None) @@ -1167,6 +1232,14 @@ async def collect_agent(*args, **kwargs): assert response.reward == expected_reward assert response.verifier_failed is expected_verifier_failed assert response.sandbox_failed is False + response_data = response.model_dump() + if expected_verifier_failed: + assert response_data["_ng_failure_class"] == "verifier_failed" + assert "_ng_failure_terminal" not in response_data + else: + assert "_ng_failure_class" not in response_data + assert "_ng_failure_terminal" not in response_data + assert response.mask_sample is False assert Path(response.artifact_dir).is_dir() terminal_output = capsys.readouterr().out assert "LAB rollout artifacts:" in terminal_output @@ -1251,6 +1324,8 @@ async def collect_agent(*args, **kwargs): assert response.model_connection_failed is True assert response.mask_sample is True assert response.verifier_failed is False + assert response.model_dump()["_ng_failure_class"] == "model_connection_failed" + assert "_ng_failure_terminal" not in response.model_dump() assert "unreachable" in response.failure_reason summary = json.loads(Path(response.run_summary_path).read_text()) assert summary["flags"]["model_connection_failed"] is True diff --git a/responses_api_agents/legal_agent_bench_native_agent/app.py b/responses_api_agents/legal_agent_bench_native_agent/app.py index 09c5d84be1..2629569fd7 100644 --- a/responses_api_agents/legal_agent_bench_native_agent/app.py +++ b/responses_api_agents/legal_agent_bench_native_agent/app.py @@ -179,6 +179,40 @@ def _failed_response( ) +def _limit_response( + *, + body: NeMoGymResponseCreateParamsNonStreaming, + response: Optional[NeMoGymResponse], + output: list[Any], + usage: Any, + stop_reason: str, +) -> NeMoGymResponse: + """Return a scoreable incomplete response for a normal agent-loop limit.""" + base = response or NeMoGymResponse( + id=f"resp_{uuid4().hex}", + created_at=0, + model=body.model or "policy_model", + object="response", + output=[], + parallel_tool_calls=body.parallel_tool_calls, + tool_choice=body.tool_choice, + tools=body.tools, + ) + metadata = dict(base.metadata or {}) + metadata["nemo_gym_stop_reason"] = stop_reason + return NeMoGymResponse.model_validate( + base.model_dump(mode="json") + | { + "status": "incomplete", + "error": None, + "incomplete_details": {"reason": "max_output_tokens"}, + "metadata": metadata, + "output": output, + "usage": usage, + } + ) + + class LegalAgentBenchNativeAgent(SimpleResponsesAPIAgent): """Run LAB's canonical tool loop through Gym's Responses API model server.""" @@ -291,12 +325,12 @@ async def responses( ) ) - return _failed_response( + return _limit_response( body=body, response=last_response, output=trajectory, usage=usage, - message=f"LAB agent reached the {self.config.max_turns}-turn limit", + stop_reason="max_turns", ) async def run(self, request: Request, body: LegalAgentBenchNativeRunRequest): diff --git a/responses_api_agents/legal_agent_bench_native_agent/tests/test_app.py b/responses_api_agents/legal_agent_bench_native_agent/tests/test_app.py index ca3fb7d9ea..3fdba86721 100644 --- a/responses_api_agents/legal_agent_bench_native_agent/tests/test_app.py +++ b/responses_api_agents/legal_agent_bench_native_agent/tests/test_app.py @@ -153,7 +153,7 @@ async def test_model_failure_preserves_partial_trajectory(monkeypatch) -> None: assert any(isinstance(item, NeMoGymFunctionCallOutput) for item in result.output) -async def test_turn_limit_is_an_explicit_failure(monkeypatch) -> None: +async def test_turn_limit_is_a_scoreable_incomplete_outcome(monkeypatch) -> None: agent = _agent(max_turns=1) agent.server_client.post = AsyncMock(return_value=_raw_response(_model_response([_function_call()]))) monkeypatch.setattr(app.LabToolExecutor, "execute", AsyncMock(return_value="file.txt")) @@ -163,9 +163,10 @@ async def test_turn_limit_is_an_explicit_failure(monkeypatch) -> None: NeMoGymResponseCreateParamsNonStreaming(input="Do the task"), ) - assert result.status == "failed" - assert result.error is not None - assert "1-turn limit" in result.error.message + assert result.status == "incomplete" + assert result.error is None + assert result.incomplete_details.reason == "max_output_tokens" + assert result.metadata == {"nemo_gym_stop_reason": "max_turns"} assert len(result.output) == 2 From a0e75a1efa5508ab2815a110a69cdb97cf5167fb Mon Sep 17 00:00:00 2001 From: Ryan Stewart Date: Thu, 6 Aug 2026 11:21:35 -0400 Subject: [PATCH 14/26] support reasoning effort for LAB judges Signed-off-by: Ryan Stewart --- resources_servers/legal_agent_bench/README.md | 1 + resources_servers/legal_agent_bench/app.py | 5 +++ .../configs/legal_agent_bench.yaml | 1 + .../configs/resources_only.yaml | 1 + .../legal_agent_bench/tests/test_app.py | 3 ++ .../tests/test_harbor_integration.py | 44 +++++++++++++++++++ .../vendor/harvey_labs/lab_harbor/judge.py | 5 +++ .../legal_agent_bench/verifier.py | 2 + 8 files changed, 62 insertions(+) diff --git a/resources_servers/legal_agent_bench/README.md b/resources_servers/legal_agent_bench/README.md index 57ebeec918..c9628e37b2 100644 --- a/resources_servers/legal_agent_bench/README.md +++ b/resources_servers/legal_agent_bench/README.md @@ -63,6 +63,7 @@ policy_model_name: your-policy-model judge_base_url: https://your-judge-endpoint.example/v1 judge_api_key: your-judge-key judge_model_name: your-judge-model +judge_reasoning_effort: medium # optional; only for judges that support it ``` The judge credentials are injected only into the regenerated, gitignored diff --git a/resources_servers/legal_agent_bench/app.py b/resources_servers/legal_agent_bench/app.py index 88ea1fbecd..53abef7b10 100644 --- a/resources_servers/legal_agent_bench/app.py +++ b/resources_servers/legal_agent_bench/app.py @@ -29,6 +29,7 @@ "judge_base_url": "LAB_JUDGE_BASE_URL", "judge_api_key": "LAB_JUDGE_API_KEY", # pragma: allowlist secret "judge_model_name": "LAB_JUDGE_MODEL", + "judge_reasoning_effort": "LAB_JUDGE_REASONING_EFFORT", "judge_temperature": "LAB_JUDGE_TEMPERATURE", "judge_request_timeout_seconds": "LAB_JUDGE_REQUEST_TIMEOUT_SECONDS", "judge_max_retries": "LAB_JUDGE_MAX_RETRIES", @@ -51,6 +52,10 @@ class LegalAgentBenchResourcesServerConfig(BaseResourcesServerConfig): judge_base_url: str | None = Field(default=None, description="OpenAI-compatible base URL for the LAB judge.") judge_api_key: str | None = Field(default=None, description="API key for the LAB judge endpoint.") judge_model_name: str | None = Field(default=None, description="Model identifier sent to the LAB judge endpoint.") + judge_reasoning_effort: Literal["minimal", "low", "medium", "high"] | None = Field( + default=None, + description="Optional reasoning effort sent to compatible LAB judge endpoints.", + ) judge_temperature: float | None = Field( default=None, ge=0, diff --git a/resources_servers/legal_agent_bench/configs/legal_agent_bench.yaml b/resources_servers/legal_agent_bench/configs/legal_agent_bench.yaml index 8211aa28a0..a484b5d7ce 100644 --- a/resources_servers/legal_agent_bench/configs/legal_agent_bench.yaml +++ b/resources_servers/legal_agent_bench/configs/legal_agent_bench.yaml @@ -14,6 +14,7 @@ legal_agent_bench: judge_base_url: ${oc.select:judge_base_url,null} judge_api_key: ${oc.select:judge_api_key,null} judge_model_name: ${oc.select:judge_model_name,null} + judge_reasoning_effort: ${oc.select:judge_reasoning_effort,null} judge_temperature: null judge_request_timeout_seconds: 90 judge_max_retries: 1 diff --git a/resources_servers/legal_agent_bench/configs/resources_only.yaml b/resources_servers/legal_agent_bench/configs/resources_only.yaml index 97d3814ee6..c0640067ec 100644 --- a/resources_servers/legal_agent_bench/configs/resources_only.yaml +++ b/resources_servers/legal_agent_bench/configs/resources_only.yaml @@ -14,6 +14,7 @@ legal_agent_bench: judge_base_url: ${oc.select:judge_base_url,null} judge_api_key: ${oc.select:judge_api_key,null} judge_model_name: ${oc.select:judge_model_name,null} + judge_reasoning_effort: ${oc.select:judge_reasoning_effort,null} judge_temperature: null judge_request_timeout_seconds: 90 judge_max_retries: 1 diff --git a/resources_servers/legal_agent_bench/tests/test_app.py b/resources_servers/legal_agent_bench/tests/test_app.py index 6e42f46a3b..a2ba5f2896 100644 --- a/resources_servers/legal_agent_bench/tests/test_app.py +++ b/resources_servers/legal_agent_bench/tests/test_app.py @@ -31,6 +31,7 @@ def test_defaults_use_full_task_and_auto_prepare(tmp_path) -> None: assert server.config.reward_mode == "full_task" assert server.config.auto_prepare_assets is True assert server.config.judge_request_timeout_seconds == 90 + assert server.config.judge_reasoning_effort is None assert server.config.judge_max_retries == 1 assert server.config.judge_structured_output is True assert server.config.judge_parse_repair_attempts == 1 @@ -45,12 +46,14 @@ def test_startup_prepares_assets_then_rebuilds_runtime(tmp_path) -> None: judge_base_url="https://judge.example/v1", judge_api_key="test-key", # pragma: allowlist secret judge_model_name="provider/model-name", + judge_reasoning_effort="medium", judge_temperature=0.2, ) judge_env = { "LAB_JUDGE_BASE_URL": "https://judge.example/v1", "LAB_JUDGE_API_KEY": "test-key", # pragma: allowlist secret "LAB_JUDGE_MODEL": "openai-compatible/provider/model-name", + "LAB_JUDGE_REASONING_EFFORT": "medium", "LAB_JUDGE_TEMPERATURE": "0.2", "LAB_JUDGE_REQUEST_TIMEOUT_SECONDS": "90", "LAB_JUDGE_MAX_RETRIES": "1", diff --git a/resources_servers/legal_agent_bench/tests/test_harbor_integration.py b/resources_servers/legal_agent_bench/tests/test_harbor_integration.py index 08fb8604ab..1b4c82e405 100644 --- a/resources_servers/legal_agent_bench/tests/test_harbor_integration.py +++ b/resources_servers/legal_agent_bench/tests/test_harbor_integration.py @@ -32,8 +32,10 @@ _marker, flatten_task_id, ) +from resources_servers.legal_agent_bench.vendor.harvey_labs.lab_harbor import judge as lab_judge from resources_servers.legal_agent_bench.vendor.harvey_labs.lab_harbor import scoring as lab_scoring from resources_servers.legal_agent_bench.vendor.harvey_labs.lab_harbor.judge import ( + OpenAICompatibleJudge, _extract_judge_message_text, ) from resources_servers.legal_agent_bench.verifier import ( @@ -550,6 +552,48 @@ def test_empty_judge_message_fails_clearly() -> None: _extract_judge_message_text({"content": "", "reasoning_content": None}) +def test_judge_sends_configured_reasoning_effort(monkeypatch) -> None: + observed = {} + + class Response: + def __enter__(self): + return self + + def __exit__(self, *_args): + return None + + def read(self): + return json.dumps({"choices": [{"message": {"content": '{"verdict":"pass","reasoning":"ok"}'}}]}).encode() + + def urlopen(request, *, timeout): + observed["payload"] = json.loads(request.data) + observed["timeout"] = timeout + return Response() + + monkeypatch.setattr(lab_judge.urllib.request, "urlopen", urlopen) + judge = OpenAICompatibleJudge( + model="openai-compatible/provider/judge", + base_url="https://judge.example/v1", + api_key="test-key", # pragma: allowlist secret + temperature=None, + timeout_seconds=30, + reasoning_effort="medium", + ) + + result = judge.evaluate( + { + "task_description": "task", + "agent_output": "output", + "criterion_title": "criterion", + "match_criteria": "match", + } + ) + + assert result == {"verdict": "pass", "reasoning": "ok"} + assert observed["payload"]["reasoning_effort"] == "medium" + assert observed["timeout"] == 30 + + def test_reward_mode_validation() -> None: assert _validate_reward_mode("full_task") == "full_task" assert _validate_reward_mode("criteria_pass_rate") == "criteria_pass_rate" diff --git a/resources_servers/legal_agent_bench/vendor/harvey_labs/lab_harbor/judge.py b/resources_servers/legal_agent_bench/vendor/harvey_labs/lab_harbor/judge.py index 69b6e9f2e5..54baeb651a 100644 --- a/resources_servers/legal_agent_bench/vendor/harvey_labs/lab_harbor/judge.py +++ b/resources_servers/legal_agent_bench/vendor/harvey_labs/lab_harbor/judge.py @@ -75,6 +75,7 @@ def __init__( parse_repair_attempts: int = 2, repair_max_tokens: int = 1024, transcript_path: Path | None = None, + reasoning_effort: str | None = None, ): self.model = model self.api_model = self._normalize_api_model(model) @@ -87,6 +88,7 @@ def __init__( self.parse_repair_attempts = max(0, parse_repair_attempts) self.repair_max_tokens = max(1, repair_max_tokens) self.transcript_path = transcript_path + self.reasoning_effort = reasoning_effort self.trace_context: dict[str, Any] = {} self.last_raw_response: str | None = None self.last_structured: bool | None = None @@ -320,6 +322,8 @@ def _chat_completion( } if self.temperature is not None: payload["temperature"] = self.temperature + if self.reasoning_effort is not None: + payload["reasoning_effort"] = self.reasoning_effort if structured: payload["response_format"] = { "type": "json_schema", @@ -350,6 +354,7 @@ def _chat_completion( "model": self.api_model, "structured": structured, "max_tokens": max_tokens, + "reasoning_effort": self.reasoning_effort, "prompt_chars": len(prompt), "payload_bytes": len(body), "request_timeout_seconds": self.timeout_seconds, diff --git a/resources_servers/legal_agent_bench/verifier.py b/resources_servers/legal_agent_bench/verifier.py index 7cafbcfebb..fa86386fa9 100644 --- a/resources_servers/legal_agent_bench/verifier.py +++ b/resources_servers/legal_agent_bench/verifier.py @@ -92,6 +92,7 @@ def run_verifier(args: argparse.Namespace) -> dict[str, Any]: "base_url": os.environ.get("LAB_JUDGE_BASE_URL") or args.judge_base_url, "api_key": os.environ.get("LAB_JUDGE_API_KEY") or args.judge_api_key, "temperature": _optional_float(os.environ.get("LAB_JUDGE_TEMPERATURE")), + "reasoning_effort": os.environ.get("LAB_JUDGE_REASONING_EFFORT") or None, "timeout_seconds": float( os.environ.get("LAB_JUDGE_REQUEST_TIMEOUT_SECONDS") or os.environ.get("LAB_JUDGE_TIMEOUT_SECONDS", "90") ), @@ -128,6 +129,7 @@ def judge_factory() -> OpenAICompatibleJudge: "task": config.get("task") or metrics.get("task") or task_config.get("title", ""), "judge_model": judge_model, "judge_model_base_url": judge.base_url, + "judge_reasoning_effort": judge.reasoning_effort, "judge_parallelism": judge_parallelism, "scored_at": datetime.now(timezone.utc).isoformat(), } From 78fa52956e06318c7da46bfc07079dd733080b1d Mon Sep 17 00:00:00 2001 From: Ryan Stewart Date: Thu, 6 Aug 2026 11:21:46 -0400 Subject: [PATCH 15/26] serialize LAB runtime cache replacement Signed-off-by: Ryan Stewart --- .../legal_agent_bench/prepare.py | 41 +++++++----- .../legal_agent_bench/tests/test_prepare.py | 63 +++++++++++++++++++ 2 files changed, 89 insertions(+), 15 deletions(-) diff --git a/resources_servers/legal_agent_bench/prepare.py b/resources_servers/legal_agent_bench/prepare.py index 8e72876df4..cd0952a028 100644 --- a/resources_servers/legal_agent_bench/prepare.py +++ b/resources_servers/legal_agent_bench/prepare.py @@ -5,6 +5,7 @@ from __future__ import annotations import argparse +import fcntl import hashlib import json import os @@ -657,21 +658,31 @@ def _hardlink_or_copy(source: str, destination: str) -> str: def _replace_directory(source: Path, target: Path) -> None: backup = target.with_name(f".{target.name}.backup") - if backup.exists(): - shutil.rmtree(backup) - if target.exists(): - target.rename(backup) - try: - source.rename(target) - except Exception: - if target.exists(): - shutil.rmtree(target) - if backup.exists(): - backup.rename(target) - raise - else: - if backup.exists(): - shutil.rmtree(backup) + lock_path = target.with_name(f".{target.name}.replace.lock") + lock_path.parent.mkdir(parents=True, exist_ok=True) + with lock_path.open("a+") as lock_file: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX) + try: + if backup.exists(): + if target.exists(): + shutil.rmtree(backup) + else: + backup.rename(target) + if target.exists(): + target.rename(backup) + try: + source.rename(target) + except Exception: + if target.exists(): + shutil.rmtree(target) + if backup.exists(): + backup.rename(target) + raise + else: + if backup.exists(): + shutil.rmtree(backup) + finally: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN) def _migrate_legacy_agent_index(tasks_dir: Path) -> bool: diff --git a/resources_servers/legal_agent_bench/tests/test_prepare.py b/resources_servers/legal_agent_bench/tests/test_prepare.py index c105e64b1a..a39c4d0ee6 100644 --- a/resources_servers/legal_agent_bench/tests/test_prepare.py +++ b/resources_servers/legal_agent_bench/tests/test_prepare.py @@ -7,6 +7,9 @@ import io import json import tarfile +import threading +import time +from concurrent.futures import ThreadPoolExecutor from pathlib import Path import pytest @@ -261,6 +264,66 @@ def test_runtime_hydration_can_reuse_a_validated_cache(monkeypatch, tmp_path) -> assert (runtime / "area__task-one" / "task.toml").is_file() +def test_directory_replacement_is_serialized_across_concurrent_publishers(monkeypatch, tmp_path) -> None: + target = tmp_path / "runtime" + target.mkdir() + (target / "value").write_text("old") + sources = [] + for value in ("first", "second"): + source = tmp_path / value + source.mkdir() + (source / "value").write_text(value) + sources.append(source) + + original_rename = Path.rename + active_renames = 0 + maximum_active_renames = 0 + counter_lock = threading.Lock() + + def slow_rename(path: Path, destination: Path) -> Path: + nonlocal active_renames, maximum_active_renames + with counter_lock: + active_renames += 1 + maximum_active_renames = max(maximum_active_renames, active_renames) + try: + time.sleep(0.02) + return original_rename(path, destination) + finally: + with counter_lock: + active_renames -= 1 + + monkeypatch.setattr(Path, "rename", slow_rename) + with ThreadPoolExecutor(max_workers=2) as executor: + list(executor.map(lambda source: prepare._replace_directory(source, target), sources)) + + assert maximum_active_renames == 1 + assert (target / "value").read_text() in {"first", "second"} + assert not target.with_name(".runtime.backup").exists() + + +def test_directory_replacement_restores_stale_backup_before_failed_publish(monkeypatch, tmp_path) -> None: + target = tmp_path / "runtime" + backup = tmp_path / ".runtime.backup" + source = tmp_path / "new" + backup.mkdir() + source.mkdir() + (backup / "value").write_text("preserved") + + original_rename = Path.rename + + def fail_new_publish(path: Path, destination: Path) -> Path: + if path == source: + raise OSError("publish failed") + return original_rename(path, destination) + + monkeypatch.setattr(Path, "rename", fail_new_publish) + with pytest.raises(OSError, match="publish failed"): + prepare._replace_directory(source, target) + + assert (target / "value").read_text() == "preserved" + assert not backup.exists() + + def test_missing_public_skill_fails_clearly(monkeypatch, tmp_path) -> None: _configure_small_snapshot(monkeypatch, tmp_path) source = _write_source(tmp_path / "source", missing_skill="pptx") From 9d31715666ec17cf34f7159e2526dfc432d76dd9 Mon Sep 17 00:00:00 2001 From: Ryan Stewart Date: Thu, 6 Aug 2026 11:23:27 -0400 Subject: [PATCH 16/26] pass LAB tool payloads over stdin Signed-off-by: Ryan Stewart --- .../tests/test_harbor_integration.py | 20 ++++++++++ .../lab_harbor/container_tool_runner.py | 13 ++++-- .../legal_agent_bench_native_agent/app.py | 15 +++++-- .../tests/test_app.py | 40 +++++++++++++++++-- 4 files changed, 77 insertions(+), 11 deletions(-) diff --git a/resources_servers/legal_agent_bench/tests/test_harbor_integration.py b/resources_servers/legal_agent_bench/tests/test_harbor_integration.py index 1b4c82e405..4d573aaafc 100644 --- a/resources_servers/legal_agent_bench/tests/test_harbor_integration.py +++ b/resources_servers/legal_agent_bench/tests/test_harbor_integration.py @@ -4,6 +4,7 @@ from __future__ import annotations import asyncio +import io import json import threading import time @@ -32,6 +33,7 @@ _marker, flatten_task_id, ) +from resources_servers.legal_agent_bench.vendor.harvey_labs.lab_harbor import container_tool_runner from resources_servers.legal_agent_bench.vendor.harvey_labs.lab_harbor import judge as lab_judge from resources_servers.legal_agent_bench.vendor.harvey_labs.lab_harbor import scoring as lab_scoring from resources_servers.legal_agent_bench.vendor.harvey_labs.lab_harbor.judge import ( @@ -50,6 +52,24 @@ BENCH_DIR = Path(__file__).resolve().parents[1] +@pytest.mark.parametrize("use_stdin", [False, True]) +def test_container_tool_runner_accepts_legacy_argv_and_stdin(monkeypatch, capsys, use_stdin: bool) -> None: + arguments = {"file_path": "memo.txt"} + raw_arguments = json.dumps(arguments) + argv = ["container_tool_runner.py", "preflight"] + if not use_stdin: + argv.append(raw_arguments) + monkeypatch.setattr(container_tool_runner.sys, "argv", argv) + monkeypatch.setattr(container_tool_runner.sys, "stdin", io.StringIO(raw_arguments if use_stdin else "")) + preflight = MagicMock(return_value="ready") + monkeypatch.setattr(container_tool_runner, "_preflight", preflight) + + container_tool_runner.main() + + assert json.loads(capsys.readouterr().out) == {"result": "ready", "metrics": {}} + preflight.assert_called_once_with() + + def _write_skills(skills_dir: Path) -> None: for name in REQUIRED_SKILLS: (skills_dir / name).mkdir(parents=True) diff --git a/resources_servers/legal_agent_bench/vendor/harvey_labs/lab_harbor/container_tool_runner.py b/resources_servers/legal_agent_bench/vendor/harvey_labs/lab_harbor/container_tool_runner.py index db3b7fc70e..619d1ba37c 100644 --- a/resources_servers/legal_agent_bench/vendor/harvey_labs/lab_harbor/container_tool_runner.py +++ b/resources_servers/legal_agent_bench/vendor/harvey_labs/lab_harbor/container_tool_runner.py @@ -4,8 +4,9 @@ """Container-side implementation for non-bash Harbor agent tools. LegalAgentBenchHarborAgent invokes this script inside the Harbor environment -with a tool name and a JSON argument payload. It returns one JSON object on -stdout: +with a tool name and a JSON argument payload. The native Gym agent passes the +payload over stdin so large document writes do not exceed the operating +system's argument-size limit. It returns one JSON object on stdout: {"result": "...", "metrics": {...}} @@ -36,13 +37,17 @@ def main() -> None: - if len(sys.argv) != 3: + if len(sys.argv) == 3: + raw_arguments = sys.argv[2] + elif len(sys.argv) == 2: + raw_arguments = sys.stdin.read() + else: _emit("Error: expected tool name and JSON arguments", {}) return tool_name = sys.argv[1] try: - arguments = json.loads(sys.argv[2]) + arguments = json.loads(raw_arguments) except json.JSONDecodeError as exc: _emit(f"Error: invalid JSON arguments: {exc}", {}) return diff --git a/responses_api_agents/legal_agent_bench_native_agent/app.py b/responses_api_agents/legal_agent_bench_native_agent/app.py index 2629569fd7..6c203d9ebf 100644 --- a/responses_api_agents/legal_agent_bench_native_agent/app.py +++ b/responses_api_agents/legal_agent_bench_native_agent/app.py @@ -79,23 +79,30 @@ async def execute(self, name: str, arguments: str | dict[str, Any]) -> str: command = parsed.get("command") if not isinstance(command, str) or not command: return "Error: command is required" - result = await self._run(["/bin/bash", "-lc", command]) + # Commands may contain generated documents or scripts large enough + # to exceed the host's argv limit. Preserve login-shell behavior, + # but stream the command over stdin so argv remains fixed-size. + result = await self._run(["/bin/bash", "-l", "-s"], stdin=command.encode()) else: - result = await self._run(["/usr/local/bin/python", str(CONTAINER_TOOL_RUNNER), name, json.dumps(parsed)]) + result = await self._run( + ["/usr/local/bin/python", str(CONTAINER_TOOL_RUNNER), name], + stdin=json.dumps(parsed).encode(), + ) full_read = name == "read" and "limit" in parsed and parsed.get("limit") in {0, None} return result if full_read else self._truncate(result) - async def _run(self, command: list[str]) -> str: + async def _run(self, command: list[str], *, stdin: bytes | None = None) -> str: process = await asyncio.create_subprocess_exec( *command, cwd="/workspace/output", env=os.environ.copy(), + stdin=asyncio.subprocess.PIPE if stdin is not None else None, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, start_new_session=True, ) try: - stdout, stderr = await asyncio.wait_for(process.communicate(), timeout=self.timeout_seconds) + stdout, stderr = await asyncio.wait_for(process.communicate(input=stdin), timeout=self.timeout_seconds) except asyncio.TimeoutError: try: os.killpg(process.pid, signal.SIGKILL) diff --git a/responses_api_agents/legal_agent_bench_native_agent/tests/test_app.py b/responses_api_agents/legal_agent_bench_native_agent/tests/test_app.py index 3fdba86721..3797e07fdc 100644 --- a/responses_api_agents/legal_agent_bench_native_agent/tests/test_app.py +++ b/responses_api_agents/legal_agent_bench_native_agent/tests/test_app.py @@ -190,7 +190,7 @@ async def test_tool_executor_validates_calls_and_truncates_output(monkeypatch) - assert await executor.execute("read", []) == "Error: arguments for read must be a JSON object" assert await executor.execute("bash", {}) == "Error: command is required" assert await executor.execute("bash", {"command": "pwd"}) == "compl\n[output truncated]" - run.assert_awaited_once_with(["/bin/bash", "-lc", "pwd"]) + run.assert_awaited_once_with(["/bin/bash", "-l", "-s"], stdin=b"pwd") run.reset_mock(return_value=True) run.return_value = "short" @@ -214,6 +214,33 @@ async def test_explicit_full_read_is_not_truncated(monkeypatch) -> None: assert result == "complete document" +async def test_tool_executor_passes_large_non_bash_payload_over_stdin(monkeypatch) -> None: + executor = app.LabToolExecutor(timeout_seconds=1, max_output_chars=2_000_000) + run = AsyncMock(return_value="written") + monkeypatch.setattr(executor, "_run", run) + content = "x" * 1_000_000 + + result = await executor.execute("write", {"file_path": "memo.txt", "content": content}) + + assert result == "written" + command = run.await_args.args[0] + assert command == ["/usr/local/bin/python", str(app.CONTAINER_TOOL_RUNNER), "write"] + assert json.loads(run.await_args.kwargs["stdin"]) == {"file_path": "memo.txt", "content": content} + + +async def test_tool_executor_passes_large_bash_command_over_stdin(monkeypatch) -> None: + executor = app.LabToolExecutor(timeout_seconds=1, max_output_chars=2_000_000) + run = AsyncMock(return_value="written") + monkeypatch.setattr(executor, "_run", run) + command = "printf %s " + "x" * 1_000_000 + + result = await executor.execute("bash", {"command": command}) + + assert result == "written" + assert run.await_args.args[0] == ["/bin/bash", "-l", "-s"] + assert run.await_args.kwargs["stdin"] == command.encode() + + async def test_process_runner_parses_container_result_and_stderr(monkeypatch) -> None: executor = app.LabToolExecutor(timeout_seconds=1, max_output_chars=100) process = _process(stdout=b'log line\n{"result":"document text"}\n', stderr=b"tool warning") @@ -221,12 +248,15 @@ async def test_process_runner_parses_container_result_and_stderr(monkeypatch) -> monkeypatch.setattr(app.asyncio, "create_subprocess_exec", create) result = await executor._run( - ["/usr/local/bin/python", str(app.CONTAINER_TOOL_RUNNER), "read", '{"file_path":"memo.docx"}'] + ["/usr/local/bin/python", str(app.CONTAINER_TOOL_RUNNER), "read"], + stdin=b'{"file_path":"memo.docx"}', ) assert result == "document text\nSTDERR:\ntool warning" assert create.await_args.kwargs["cwd"] == "/workspace/output" assert create.await_args.kwargs["start_new_session"] is True + assert create.await_args.kwargs["stdin"] is app.asyncio.subprocess.PIPE + process.communicate.assert_awaited_once_with(input=b'{"file_path":"memo.docx"}') @pytest.mark.parametrize( @@ -250,7 +280,10 @@ async def test_process_runner_handles_unstructured_and_failed_output( AsyncMock(return_value=_process(stdout=stdout, returncode=returncode)), ) - result = await executor._run(["/usr/local/bin/python", str(app.CONTAINER_TOOL_RUNNER), "preflight", "{}"]) + result = await executor._run( + ["/usr/local/bin/python", str(app.CONTAINER_TOOL_RUNNER), "preflight"], + stdin=b"{}", + ) assert result == expected @@ -275,6 +308,7 @@ async def time_out(awaitable, *, timeout): killpg.assert_called_once_with(123, app.signal.SIGKILL) assert process.communicate.call_count == 2 assert process.communicate.await_count == 1 + process.communicate.assert_any_call(input=None) def test_merge_usage_ignores_missing_current_usage() -> None: From 8a1cd59a10145b181844b05398af10158dab35d0 Mon Sep 17 00:00:00 2001 From: Ryan Stewart Date: Thu, 6 Aug 2026 11:23:37 -0400 Subject: [PATCH 17/26] exclude raw OOXML from LAB aggregate scoring Signed-off-by: Ryan Stewart --- .../tests/test_harbor_integration.py | 15 +++++++++++++++ .../vendor/harvey_labs/lab_harbor/scoring.py | 5 +++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/resources_servers/legal_agent_bench/tests/test_harbor_integration.py b/resources_servers/legal_agent_bench/tests/test_harbor_integration.py index 4d573aaafc..49ea4f2a0b 100644 --- a/resources_servers/legal_agent_bench/tests/test_harbor_integration.py +++ b/resources_servers/legal_agent_bench/tests/test_harbor_integration.py @@ -714,6 +714,21 @@ def test_deliverable_matching_ignores_thread_export() -> None: ) == {"contract": "final-contract.docx"} +def test_full_output_ignores_raw_ooxml_working_files(monkeypatch, tmp_path) -> None: + output_dir = tmp_path / "output" + (output_dir / "workdir" / "word" / "_rels").mkdir(parents=True) + (output_dir / "response.docx").write_bytes(b"placeholder") + (output_dir / "workdir" / "word" / "document.xml").write_text("raw document xml") + (output_dir / "workdir" / "word" / "_rels" / "document.xml.rels").write_text("raw relationships") + monkeypatch.setattr(lab_scoring, "_read_file_as_text", lambda path: f"content:{path.name}") + + content = lab_scoring._load_all_output(output_dir) + + assert "content:response.docx" in content + assert "document.xml" not in content + assert "document.xml.rels" not in content + + def test_parallel_judging_uses_isolated_judges_and_preserves_order(tmp_path) -> None: output_dir = tmp_path / "run" / "output" output_dir.mkdir(parents=True) diff --git a/resources_servers/legal_agent_bench/vendor/harvey_labs/lab_harbor/scoring.py b/resources_servers/legal_agent_bench/vendor/harvey_labs/lab_harbor/scoring.py index 1727d28ab8..acdcf727b2 100644 --- a/resources_servers/legal_agent_bench/vendor/harvey_labs/lab_harbor/scoring.py +++ b/resources_servers/legal_agent_bench/vendor/harvey_labs/lab_harbor/scoring.py @@ -10,7 +10,8 @@ - uses the configured OpenAI-compatible judge instead of provider SDKs; - creates isolated judges for parallel criteria; - records per-criterion transcripts and judge errors; -- fails missing or unreadable deliverables without calling the judge; and +- fails missing or unreadable deliverables without calling the judge; +- excludes raw OOXML working files from aggregate judge input; and - uses deterministic filename matching without Anthropic-specific LLM fallback. """ @@ -27,7 +28,7 @@ SKIP_DIRS = {"node_modules", ".npm", "__pycache__", ".git", "venv", ".venv"} -SKIP_EXTENSIONS = {".lock", ".map"} +SKIP_EXTENSIONS = {".lock", ".map", ".rels", ".xml"} SKIP_FILES = {"package-lock.json"} From 78e41725949c28bf7a1fc2d3ca9b0b33e48da9c0 Mon Sep 17 00:00:00 2001 From: Ryan Stewart Date: Thu, 6 Aug 2026 11:23:57 -0400 Subject: [PATCH 18/26] classify LAB native and verifier failures Signed-off-by: Ryan Stewart --- .../legal_agent_bench_agent/app.py | 24 +++ .../legal_agent_bench_agent/tests/test_app.py | 152 ++++++++++++++++++ .../legal_agent_bench_native_agent/app.py | 41 ++++- .../tests/test_app.py | 56 +++++++ 4 files changed, 272 insertions(+), 1 deletion(-) diff --git a/responses_api_agents/legal_agent_bench_agent/app.py b/responses_api_agents/legal_agent_bench_agent/app.py index bddae82d18..6dec016526 100644 --- a/responses_api_agents/legal_agent_bench_agent/app.py +++ b/responses_api_agents/legal_agent_bench_agent/app.py @@ -59,6 +59,8 @@ DATASET_ALIAS = "legal_agent_bench" INITIAL_USER_PROMPT = "Please begin working on the task described in the system prompt." NATIVE_AGENT_MODULE = "responses_api_agents.legal_agent_bench_native_agent.app" +AGENT_FAILURE_CLASS_METADATA_KEY = "nemo_gym_failure_class" +PROPAGATED_AGENT_FAILURE_CLASSES = frozenset({"agent_timed_out", "model_connection_failed"}) LAB_SYSTEM_PROMPT = ( PARENT_DIR / "resources_servers" / "legal_agent_bench" / "vendor" / "harvey_labs" / "harness" / "system-prompt.md" ).read_text(encoding="utf-8") @@ -665,6 +667,17 @@ def agent_response_failure(response: NeMoGymResponse, agent_server_module: str) return None +def agent_response_failure_flags(response: NeMoGymResponse, agent_server_module: str) -> tuple[bool, bool]: + """Return structured model-connection and timeout flags from a failed native response.""" + if agent_server_module != NATIVE_AGENT_MODULE or response.error is None: + return False, False + metadata = response.metadata or {} + failure_class = metadata.get(AGENT_FAILURE_CLASS_METADATA_KEY) + if failure_class not in PROPAGATED_AGENT_FAILURE_CLASSES: + return False, False + return failure_class == "model_connection_failed", failure_class == "agent_timed_out" + + def _task_name(instance_id: str) -> str: alias, separator, task_name = instance_id.partition("::") if separator != "::" or alias != DATASET_ALIAS or not task_name: @@ -1371,6 +1384,12 @@ def _response( failure_reason = failure_reason or str(exc) verifier_error = normalized_reward["verifier_error"] judge_errors = normalized_reward["judge_error_count"] + verifier_failed = verifier_failed or bool(verifier_error or judge_errors) + if failure_reason is None and judge_errors: + suffix = "error" if judge_errors == 1 else "errors" + failure_reason = f"Verifier reported {judge_errors} judge {suffix}" + elif failure_reason is None and verifier_error: + failure_reason = "Verifier reported an internal error" unreliable = bool( agent_failed or model_connection_failed @@ -1536,6 +1555,11 @@ async def run(self, request: Request, body: LegalAgentBenchRunRequest) -> LegalA else: agent_failed = True failure_reason = failure_reason or "Agent did not produce response.json" + response_model_connection_failed, response_agent_timed_out = agent_response_failure_flags( + response, self.config.agent_server_module + ) + model_connection_failed = model_connection_failed or response_model_connection_failed + agent_timed_out = agent_timed_out or response_agent_timed_out if not agent_failed: response_failure = agent_response_failure(response, self.config.agent_server_module) if response_failure: diff --git a/responses_api_agents/legal_agent_bench_agent/tests/test_app.py b/responses_api_agents/legal_agent_bench_agent/tests/test_app.py index 0a9b4153b8..f22d33f4c7 100644 --- a/responses_api_agents/legal_agent_bench_agent/tests/test_app.py +++ b/responses_api_agents/legal_agent_bench_agent/tests/test_app.py @@ -778,6 +778,34 @@ def test_partial_response_with_harness_error_is_an_agent_failure() -> None: assert "adapter failed after partial output" in failure +def test_native_timeout_failure_metadata_propagates_timeout_flag() -> None: + response = app.NeMoGymResponse.model_validate( + { + **_successful_response().model_dump(mode="json"), + "status": "failed", + "error": {"code": "server_error", "message": "LAB model call timed out after 1800s"}, + "metadata": {app.AGENT_FAILURE_CLASS_METADATA_KEY: "agent_timed_out"}, + } + ) + + assert app.agent_response_failure_flags(response, app.NATIVE_AGENT_MODULE) == (False, True) + assert app.agent_response_failure_flags(response, "responses_api_agents.codex_agent.app") == (False, False) + + +def test_native_model_connection_failure_metadata_propagates_connection_flag() -> None: + response = app.NeMoGymResponse.model_validate( + { + **_successful_response().model_dump(mode="json"), + "status": "failed", + "error": {"code": "server_error", "message": "LAB model call failed: HTTP 500"}, + "metadata": {app.AGENT_FAILURE_CLASS_METADATA_KEY: "model_connection_failed"}, + } + ) + + assert app.agent_response_failure_flags(response, app.NATIVE_AGENT_MODULE) == (True, False) + assert app.agent_response_failure_flags(response, "responses_api_agents.codex_agent.app") == (False, False) + + def test_response_masks_harness_and_verifier_failures() -> None: runner = app.LegalAgentBenchAgent.model_construct(config=_config()) params = NeMoGymResponseCreateParamsNonStreaming(input=[]) @@ -805,6 +833,36 @@ def test_response_masks_harness_and_verifier_failures() -> None: assert "_ng_failure_terminal" not in response.model_dump() +@pytest.mark.parametrize( + ("reward_data", "expected_reason"), + [ + ({"judge_error_count": 1}, "Verifier reported 1 judge error"), + ({"judge_error_count": 3}, "Verifier reported 3 judge errors"), + ({"verifier_error": 1}, "Verifier reported an internal error"), + ], +) +def test_response_synthesizes_failure_reason_for_unreliable_verifier_metrics(reward_data, expected_reason) -> None: + runner = app.LegalAgentBenchAgent.model_construct(config=_config()) + params = NeMoGymResponseCreateParamsNonStreaming(input=[]) + body = app.LegalAgentBenchRunRequest( + instance_id="legal_agent_bench::area__task", + responses_create_params=params, + ) + + response = runner._response( + body=body, + params=params, + response=app._empty_response("policy"), + reward_data=reward_data, + paths=None, + ) + + assert response.failure_reason == expected_reason + assert response.verifier_failed is True + assert response.mask_sample is True + assert response.model_dump()["_ng_failure_class"] == "verifier_failed" + + @pytest.mark.parametrize( ("failure_flag", "expected_class", "terminal"), [ @@ -1182,6 +1240,12 @@ async def ensure_image(_task): async def ensure_runtime(_image): return deps + async def stage_agent(*args, **kwargs): + return None + + async def collect_agent(*args, **kwargs): + return {} + def write_runner(paths, params, model_url): incomplete = app.NeMoGymResponse.model_validate( _successful_response().model_dump(mode="json") @@ -1335,6 +1399,94 @@ async def collect_agent(*args, **kwargs): assert "LAB rollout failed:" in terminal_output +@pytest.mark.asyncio +async def test_native_model_timeout_is_masked_routed_and_skips_verifier(monkeypatch, tmp_path) -> None: + _root, task = _task_tree(tmp_path) + skills = _skills(tmp_path) + deps = tmp_path / "deps" + deps.mkdir() + events = [] + + class AgentSandbox: + async def start(self): + events.append("agent:start") + + async def exec(self, *args, **kwargs): + events.append("agent:exec") + return SimpleNamespace(return_code=0, error_type=None, stdout="", stderr="") + + async def stop(self): + events.append("agent:stop") + + runner = app.LegalAgentBenchAgent.model_construct( + config=_config( + agent_server_module=app.NATIVE_AGENT_MODULE, + agent_server_class="LegalAgentBenchNativeAgent", + agent_config_class="LegalAgentBenchNativeAgentConfig", + ) + ) + runner._sem = app.asyncio.Semaphore(1) + runner._session_results_dir = tmp_path / "results" + + async def ensure_image(_task): + return "lab:image" + + async def ensure_runtime(_image): + return deps + + async def stage_agent(*args, **kwargs): + return None + + async def collect_agent(*args, **kwargs): + return {} + + def write_runner(paths, params, model_url): + failed = app.NeMoGymResponse.model_validate( + { + **_successful_response("Partial work").model_dump(mode="json"), + "status": "failed", + "error": {"code": "server_error", "message": "LAB model call timed out after 1800s"}, + "metadata": {app.AGENT_FAILURE_CLASS_METADATA_KEY: "agent_timed_out"}, + } + ) + (paths["runtime"] / "response.json").write_text(failed.model_dump_json()) + + monkeypatch.setattr(app, "resolve_task_dir", lambda runtime, instance: task) + monkeypatch.setattr(app, "resolve_repo_path", lambda path: skills) + monkeypatch.setattr(app, "compose_agent_input", lambda task_dir, skills_dir, params, **kwargs: params) + monkeypatch.setattr(runner, "_ensure_image", ensure_image) + monkeypatch.setattr(runner, "_ensure_runtime", ensure_runtime) + monkeypatch.setattr(runner, "_stage_agent_source", lambda paths: None) + monkeypatch.setattr(runner, "_write_runner_config", write_runner) + monkeypatch.setattr(runner, "_model_url", lambda body: "http://model") + monkeypatch.setattr(runner, "_model_name", lambda: "policy") + monkeypatch.setattr(runner, "_agent_sandbox", lambda **kwargs: AgentSandbox()) + monkeypatch.setattr(runner, "_stage_agent_sandbox", stage_agent) + monkeypatch.setattr(runner, "_collect_agent_sandbox", collect_agent) + monkeypatch.setattr(runner, "_materialize_agent_downloads", lambda *args, **kwargs: None) + monkeypatch.setattr( + runner, + "_verifier_sandbox", + lambda **kwargs: pytest.fail("verifier must not run after a native model timeout"), + ) + monkeypatch.setattr(runner, "_artifacts", lambda *args, **kwargs: None) + body = app.LegalAgentBenchRunRequest( + instance_id="legal_agent_bench::area__task", + responses_create_params=NeMoGymResponseCreateParamsNonStreaming(input=[]), + ) + + response = await runner.run(None, body) + + assert events == ["agent:start", "agent:exec", "agent:stop"] + assert response.agent_failed is True + assert response.agent_timed_out is True + assert response.model_connection_failed is False + assert response.mask_sample is True + assert response.model_dump()["_ng_failure_class"] == "agent_timed_out" + assert "timed out after 1800s" in response.failure_reason + assert response.response.output + + class _FakeSandbox: def __init__(self, reward: dict | None, *, timed_out: bool = False): self.reward = reward diff --git a/responses_api_agents/legal_agent_bench_native_agent/app.py b/responses_api_agents/legal_agent_bench_native_agent/app.py index 6c203d9ebf..b6e60a5b97 100644 --- a/responses_api_agents/legal_agent_bench_native_agent/app.py +++ b/responses_api_agents/legal_agent_bench_native_agent/app.py @@ -12,6 +12,7 @@ from typing import Any, Optional from uuid import uuid4 +import aiohttp from fastapi import Body, Request from pydantic import ConfigDict, Field @@ -35,6 +36,25 @@ ) CONTAINER_TOOL_RUNNER = Path("/opt/legal-agent-bench/container_tool_runner.py") SUPPORTED_TOOLS = frozenset({"bash", "read", "write", "write_docx", "edit", "glob", "grep"}) +AGENT_FAILURE_CLASS_METADATA_KEY = "nemo_gym_failure_class" +MODEL_CONNECTION_FAILURE_CLASS = "model_connection_failed" +RETRYABLE_MODEL_ERROR_CODES = frozenset({"connection_error", "rate_limit_error", "server_error"}) + + +def _is_model_connection_exception(exc: BaseException) -> bool: + """Return whether a model-call exception is an operational transport failure.""" + if isinstance(exc, aiohttp.ClientResponseError): + return exc.status in {408, 429} or 500 <= exc.status < 600 + return isinstance(exc, (aiohttp.ClientConnectionError, aiohttp.ClientPayloadError)) + + +def _with_model_failure_class(response: NeMoGymResponse) -> NeMoGymResponse: + """Annotate retryable Responses API errors without masking incomplete outcomes.""" + if response.error is None or response.error.code not in RETRYABLE_MODEL_ERROR_CODES: + return response + metadata = dict(response.metadata or {}) + metadata[AGENT_FAILURE_CLASS_METADATA_KEY] = MODEL_CONNECTION_FAILURE_CLASS + return response.model_copy(update={"metadata": metadata}) class LegalAgentBenchNativeAgentConfig(BaseResponsesAPIAgentConfig): @@ -164,6 +184,7 @@ def _failed_response( output: list[Any], usage: Any, message: str, + failure_class: Optional[str] = None, ) -> NeMoGymResponse: base = response or NeMoGymResponse( id=f"resp_{uuid4().hex}", @@ -175,11 +196,15 @@ def _failed_response( tool_choice=body.tool_choice, tools=body.tools, ) + metadata = dict(base.metadata or {}) + if failure_class is not None: + metadata[AGENT_FAILURE_CLASS_METADATA_KEY] = failure_class return NeMoGymResponse.model_validate( base.model_dump(mode="json") | { "status": "failed", "error": {"code": "server_error", "message": message[-2000:]}, + "metadata": metadata, "output": output, "usage": usage, } @@ -270,6 +295,15 @@ async def responses( await raise_for_status(raw_response) model_response = NeMoGymResponse.model_validate(await get_response_json(raw_response)) model_cookies = raw_response.cookies + except TimeoutError: + return _failed_response( + body=body, + response=last_response, + output=trajectory, + usage=usage, + message=f"LAB model call timed out after {self.config.model_timeout_seconds}s", + failure_class="agent_timed_out", + ) except Exception as exc: return _failed_response( body=body, @@ -277,12 +311,17 @@ async def responses( output=trajectory, usage=usage, message=f"LAB model call failed: {type(exc).__name__}: {exc}", + failure_class=(MODEL_CONNECTION_FAILURE_CLASS if _is_model_connection_exception(exc) else None), ) last_response = model_response usage = _merge_usage(usage, model_response.usage) trajectory.extend(model_response.output) - if model_response.error is not None or model_response.incomplete_details is not None: + if model_response.error is not None: + return _with_model_failure_class( + model_response.model_copy(update={"output": trajectory, "usage": usage}) + ) + if model_response.incomplete_details is not None: return model_response.model_copy(update={"output": trajectory, "usage": usage}) function_calls = [ diff --git a/responses_api_agents/legal_agent_bench_native_agent/tests/test_app.py b/responses_api_agents/legal_agent_bench_native_agent/tests/test_app.py index 3797e07fdc..69d8470d77 100644 --- a/responses_api_agents/legal_agent_bench_native_agent/tests/test_app.py +++ b/responses_api_agents/legal_agent_bench_native_agent/tests/test_app.py @@ -153,6 +153,59 @@ async def test_model_failure_preserves_partial_trajectory(monkeypatch) -> None: assert any(isinstance(item, NeMoGymFunctionCallOutput) for item in result.output) +async def test_model_timeout_sets_structured_failure_metadata(monkeypatch) -> None: + agent = _agent() + agent.server_client.post = AsyncMock( + side_effect=[ + _raw_response(_model_response([_function_call()])), + TimeoutError(), + ] + ) + monkeypatch.setattr(app.LabToolExecutor, "execute", AsyncMock(return_value="file.txt")) + + result = await agent.responses( + SimpleNamespace(path_params={}), + NeMoGymResponseCreateParamsNonStreaming(input="Do the task"), + ) + + assert result.status == "failed" + assert result.error is not None + assert result.error.message == "LAB model call timed out after 1800s" + assert result.metadata == {app.AGENT_FAILURE_CLASS_METADATA_KEY: "agent_timed_out"} + assert any(isinstance(item, NeMoGymResponseFunctionToolCall) for item in result.output) + assert any(isinstance(item, NeMoGymFunctionCallOutput) for item in result.output) + + +@pytest.mark.parametrize( + ("status", "expected_failure_class"), + [ + (429, app.MODEL_CONNECTION_FAILURE_CLASS), + (500, app.MODEL_CONNECTION_FAILURE_CLASS), + (400, None), + ], +) +async def test_model_http_failures_classify_only_retryable_statuses(status, expected_failure_class) -> None: + agent = _agent() + agent.server_client.post = AsyncMock( + side_effect=app.aiohttp.ClientResponseError( + request_info=MagicMock(real_url="http://policy/v1/responses"), + history=(), + status=status, + message="model request failed", + ) + ) + + result = await agent.responses( + SimpleNamespace(path_params={}), + NeMoGymResponseCreateParamsNonStreaming(input="Do the task"), + ) + + assert result.status == "failed" + assert result.error is not None + assert f"ClientResponseError: {status}" in result.error.message + assert (result.metadata or {}).get(app.AGENT_FAILURE_CLASS_METADATA_KEY) == expected_failure_class + + async def test_turn_limit_is_a_scoreable_incomplete_outcome(monkeypatch) -> None: agent = _agent(max_turns=1) agent.server_client.post = AsyncMock(return_value=_raw_response(_model_response([_function_call()]))) @@ -336,6 +389,9 @@ async def test_model_error_response_is_returned_with_trajectory() -> None: assert result.status == "failed" assert result.error is not None assert result.error.message == "policy backend failed" + assert result.metadata == { + app.AGENT_FAILURE_CLASS_METADATA_KEY: app.MODEL_CONNECTION_FAILURE_CLASS, + } assert any(isinstance(item, NeMoGymResponseOutputMessage) for item in result.output) From 702c956e2c50609bb50229495cbde48f0b4890c1 Mon Sep 17 00:00:00 2001 From: Ryan Stewart Date: Thu, 6 Aug 2026 14:23:51 -0400 Subject: [PATCH 19/26] document LAB output and timeout guidance Signed-off-by: Ryan Stewart --- benchmarks/legal_agent_bench/README.md | 86 +++++++++++++++++++ resources_servers/legal_agent_bench/README.md | 23 +++++ 2 files changed, 109 insertions(+) diff --git a/benchmarks/legal_agent_bench/README.md b/benchmarks/legal_agent_bench/README.md index 32ff087724..9cabe3fc77 100644 --- a/benchmarks/legal_agent_bench/README.md +++ b/benchmarks/legal_agent_bench/README.md @@ -226,6 +226,92 @@ Harbor retains its established response format and additionally reports `failure_reason` for agent-phase failures. It preserves any partial trajectory, but skips judging and forces reward to zero when one of those failures occurs. +## Choose output, context, and timeout limits + +LAB does not prescribe one model-independent output-token limit. Its +[upstream adapters](https://github.com/harveyai/harvey-labs/tree/main/harness/adapters) +choose provider- and model-specific per-call limits, generally using the +model's large output capacity. For a locally hosted model, `64,000` is a +sensible starting point when the endpoint and hardware support it: + +```bash +++responses_create_params.max_output_tokens=64000 +``` + +Append that override to the native or configurable-harness command. Set the +limit as high as the model, server, total context window, and available KV +cache can sustain. A larger output reservation can reduce the input space +available within a fixed context window, so it is not always safe to use the +largest numerical value accepted by the API. If a higher limit exceeds local +memory capacity, reduce rollout and agent-server concurrency together before +lowering the limit. + +An unreasonably low output limit can stop a long tool call or reasoning turn +before the agent finishes its deliverables. A context window that is too small +can have the same effect later in a multi-turn task. LAB scores those incomplete +outcomes, so either constraint can skew model-quality results downward. Record +both limits with published results and compare models using settings that do +not prematurely truncate otherwise supported work. + +### Timeouts and turn limits + +LAB does not define one canonical wall-clock timeout for a task. Its current +upstream [runner CLI](https://github.com/harveyai/harvey-labs/blob/main/harness/run.py) +exposes a turn budget and a per-command shell timeout, but the +[agent loop](https://github.com/harveyai/harvey-labs/blob/main/harness/agent_loop.py) +has no overall deadline. Upstream +[model adapters](https://github.com/harveyai/harvey-labs/tree/main/harness/adapters) +also do not share a model-request timeout, and the +[evaluator CLI](https://github.com/harveyai/harvey-labs/blob/main/evaluation/run_eval.py) +does not expose a uniform judge timeout. Provider SDK defaults therefore +differ. + +Gym adds layered operational deadlines so a stalled endpoint or sandbox cannot +hold a rollout worker indefinitely. The checked-in defaults are the recommended +starting values for full evaluations: + +| Layer | Default | Guidance | +| --- | ---: | --- | +| Complete agent phase | 10,800 seconds (3 hours) | Use for all harnesses. Claude Code and Codex also receive this as their inner harness timeout. | +| One policy-model request | 1,800 seconds (30 minutes) | Used by the native and Harbor loops. This is intentionally generous for slow local reasoning models. | +| Shell command | 60 seconds | Used by the native and Harbor loops. Hermes uses a 180-second terminal timeout. A shell timeout is returned to the agent as a tool error. | +| Complete verifier phase | 3,600 seconds (1 hour) | Covers output staging, all criterion calls, and artifact collection. | +| One judge request | 90 seconds, with one retry | Increase only when the judge endpoint is healthy but consistently needs longer than 90 seconds. | + +Start with these values. For a slow locally hosted policy model, reduce both +rollout and agent-server concurrency before changing timeouts. If successful +model generations genuinely take longer than 30 minutes, raise the policy-call +timeout and the enclosing agent-phase timeout together. Keep the outer agent +deadline comfortably above a single model call, and keep the verifier deadline +above the judge request timeout plus retries. Raising a timeout does not reserve +GPU memory; increasing output or context limits can. + +The default native and Harbor configurations use a 60-turn budget, while +Hermes uses 90 turns and the CLI harnesses use their own configured stopping +behavior. A turn limit is a model-behavior constraint, not a timeout. Current +upstream LAB `main` uses 200 turns, but this Gym integration is pinned to an +earlier LAB revision and does not silently adopt later harness changes. Record +the turn budget as well as output, context, and timeout settings with published +results. + +The common defaults are configured in +`responses_api_agents/legal_agent_bench_agent/configs/` and +`resources_servers/legal_agent_bench/configs/legal_agent_bench.yaml`. For the +default native benchmark, the relevant override paths are: + +```text ++legal_agent_bench_benchmark_native_agent.responses_api_agents.legal_agent_bench_agent.agent_timeout_seconds= ++legal_agent_bench_benchmark_native_agent.responses_api_agents.legal_agent_bench_agent.verifier_timeout_seconds= ++legal_agent_bench_benchmark_native_agent.responses_api_agents.legal_agent_bench_agent.agent_kwargs.model_timeout_seconds= ++legal_agent_bench_benchmark_native_agent.responses_api_agents.legal_agent_bench_agent.agent_kwargs.shell_timeout= ++legal_agent_bench_benchmark_resources_server.resources_servers.legal_agent_bench.judge_request_timeout_seconds= +``` + +Substitute the selected variant's agent and resource-server prefixes when +running Harbor, Hermes, Claude Code, or Codex. A whole-agent, policy-connection, +sandbox, or verifier timeout is an operational failure and is routed through +Gym's failure handling; it is not a completed zero-reward model outcome. + ## Run a larger evaluation Remove `--limit 1` from the desired smoke command. Choose a new output filename diff --git a/resources_servers/legal_agent_bench/README.md b/resources_servers/legal_agent_bench/README.md index c9628e37b2..6dd95629b0 100644 --- a/resources_servers/legal_agent_bench/README.md +++ b/resources_servers/legal_agent_bench/README.md @@ -229,6 +229,29 @@ This setting is forwarded to each task verifier as The task container requires network access because the policy agent and verifier call configured endpoints. +### Output, context, and timeout limits + +LAB does not define one model-independent output-token limit. Upstream harness +adapters use provider- and model-specific per-call caps, typically near each +model's supported output capacity. For locally hosted policy models, start with +`++responses_create_params.max_output_tokens=64000` when the endpoint, total +context window, and available KV cache support it. See +[Choose output, context, and timeout limits](../../benchmarks/legal_agent_bench/README.md#choose-output-context-and-timeout-limits) +for the full guidance and concurrency tradeoff. + +Avoid an unnecessarily low output cap or context window: either can truncate a +valid long-running agent trajectory, and LAB will score that incomplete result. +Record the limits used when reporting benchmark scores. + +LAB does not specify one whole-task wall-clock timeout. Gym's recommended +defaults are a 3-hour agent phase, a 30-minute policy request for the native and +Harbor loops, a 1-hour verifier phase, and a 90-second judge request with one +retry. Native and Harbor shell commands use 60 seconds; Hermes terminal calls +use 180 seconds. These are operational safety limits rather than LAB scoring +parameters. See +[Timeouts and turn limits](../../benchmarks/legal_agent_bench/README.md#timeouts-and-turn-limits) +for the complete table, override paths, and guidance for slow local models. + ## Caches and outputs The default paths are: From b07289074fdf512fcf1763f028b732be275fd0e8 Mon Sep 17 00:00:00 2001 From: Ryan Stewart Date: Thu, 13 Aug 2026 13:10:21 -0400 Subject: [PATCH 20/26] make the LAB runner sandbox-provider agnostic Signed-off-by: Ryan Stewart --- benchmarks/legal_agent_bench/README.md | 206 ++-- .../legal_agent_bench/tests/test_prepare.py | 44 + .../apptainer/configs/apptainer.yaml | 18 + nemo_gym/server_utils.py | 5 + resources_servers/legal_agent_bench/README.md | 51 +- .../legal_agent_bench_agent/README.md | 114 ++- .../legal_agent_bench_agent/app.py | 787 ++++++++++----- .../legal_agent_bench_claude_code.yaml | 12 +- .../configs/legal_agent_bench_codex.yaml | 14 +- .../configs/legal_agent_bench_hermes.yaml | 12 +- .../configs/legal_agent_bench_native.yaml | 13 +- .../scripts/smoke_provider.py | 113 +++ .../legal_agent_bench_agent/tests/test_app.py | 907 +++++++++++++----- tests/unit_tests/test_server_utils.py | 17 + 14 files changed, 1724 insertions(+), 589 deletions(-) create mode 100644 nemo_gym/sandbox/providers/apptainer/configs/apptainer.yaml create mode 100644 responses_api_agents/legal_agent_bench_agent/scripts/smoke_provider.py diff --git a/benchmarks/legal_agent_bench/README.md b/benchmarks/legal_agent_bench/README.md index 9cabe3fc77..01d3b0c95c 100644 --- a/benchmarks/legal_agent_bench/README.md +++ b/benchmarks/legal_agent_bench/README.md @@ -25,10 +25,11 @@ agent during dataset collation. ## Requirements -- Python 3.13.13 and the repository environment installed with `uv` -- Docker with a running daemon for the default local backend and portable - harness provisioning -- The `sandbox` dependency extra when using ECS Fargate +- Python 3.13.14 and the repository environment installed with `uv` +- One supported Gym sandbox provider: Docker, ECS Fargate, Enroot, Apptainer, + OpenSandbox, Daytona, or OpenShell +- The provider's local CLI, service, credentials, and Gym dependency extra, as + documented under `nemo_gym/sandbox/providers//` - Authorized OpenAI-compatible policy and judge endpoints in the root `env.yaml` - At least 10 GB of free working space @@ -38,6 +39,8 @@ Not required: - Separate Harbor, Hermes, Claude Code, or Codex installations - Anthropic or OpenAI vendor subscriptions or CLI logins for the Claude Code and Codex harnesses +- Docker when a non-Docker provider is selected and a compatible LAB image is + already available Gym provisions the pinned harness dependencies automatically. Every harness uses the configured policy model endpoint. Access to your configured policy and @@ -46,21 +49,26 @@ judge endpoints is still required and may itself be metered or paid. See the [resource-server README](../../resources_servers/legal_agent_bench/README.md) for endpoint configuration, source and license details, cache locations, and troubleshooting. The initial source download is several hundred MiB, and the -first rollout builds a document-tooling Docker image that can take several -minutes. +first rollout provisions the selected harness inside a temporary sandbox. The +default Docker backend also builds the document-tooling image on first use. ## Set up From the repository root: ```bash -uv venv --python 3.12 +uv venv --python 3.13.14 source .venv/bin/activate uv sync --extra dev +``` + +For Docker, verify the default local backend: + +```bash docker info >/dev/null ``` -For a remote sandbox provider, also install its dependencies with: +For SDK-backed providers, install the sandbox dependencies as well: ```bash uv sync --extra dev --extra sandbox @@ -274,7 +282,9 @@ starting values for full evaluations: | --- | ---: | --- | | Complete agent phase | 10,800 seconds (3 hours) | Use for all harnesses. Claude Code and Codex also receive this as their inner harness timeout. | | One policy-model request | 1,800 seconds (30 minutes) | Used by the native and Harbor loops. This is intentionally generous for slow local reasoning models. | +| Native tool preflight | 120 seconds | Allows document tooling to initialize on a cold or CPU-throttled sandbox. This does not increase the timeout for normal tool calls. | | Shell command | 60 seconds | Used by the native and Harbor loops. Hermes uses a 180-second terminal timeout. A shell timeout is returned to the agent as a tool error. | +| Sandbox staging/collection | 900 seconds (15 minutes) | Covers extraction and collection of portable runtimes and task artifacts. Remote providers should give file-transfer API requests at least the same budget. Reduce concurrency if large parallel transfers saturate the provider. | | Complete verifier phase | 3,600 seconds (1 hour) | Covers output staging, all criterion calls, and artifact collection. | | One judge request | 90 seconds, with one retry | Increase only when the judge endpoint is healthy but consistently needs longer than 90 seconds. | @@ -301,8 +311,10 @@ default native benchmark, the relevant override paths are: ```text +legal_agent_bench_benchmark_native_agent.responses_api_agents.legal_agent_bench_agent.agent_timeout_seconds= ++legal_agent_bench_benchmark_native_agent.responses_api_agents.legal_agent_bench_agent.sandbox_staging_timeout_seconds= +legal_agent_bench_benchmark_native_agent.responses_api_agents.legal_agent_bench_agent.verifier_timeout_seconds= +legal_agent_bench_benchmark_native_agent.responses_api_agents.legal_agent_bench_agent.agent_kwargs.model_timeout_seconds= ++legal_agent_bench_benchmark_native_agent.responses_api_agents.legal_agent_bench_agent.agent_kwargs.preflight_timeout_seconds= +legal_agent_bench_benchmark_native_agent.responses_api_agents.legal_agent_bench_agent.agent_kwargs.shell_timeout= +legal_agent_bench_benchmark_resources_server.resources_servers.legal_agent_bench.judge_request_timeout_seconds= ``` @@ -335,76 +347,134 @@ the corresponding `legal_agent_bench_benchmark_claude_code_agent` or agent-server default stays at `1`; changing only `--concurrency` leaves the server-side semaphore serial. -## Use ECS Fargate - -Gym's native `ecs_fargate` provider can run the native, Hermes, Claude Code, -or Codex LAB variants without changing their agent loops. The reference ECS -infrastructure is discovered from `//ecs-sandbox/config` in SSM; -`ssm_project` defaults to `harbor`. Configure an AWS profile locally and export -the profile and region before starting Gym: +## Choose a sandbox provider + +The native, Hermes, Claude Code, and Codex variants use Gym's shared sandbox +API. Their runtime builder, agent phase, and isolated verifier phase all use the +same selected provider. The Harbor compatibility variant uses Harbor's own +container orchestration instead and is Docker-only. + +| Provider | LAB image | Policy-proxy routing | Provider setup | +| --- | --- | --- | --- | +| Docker (default) | Automatically built locally, or `sandbox_image` | Derived loopback URLs are translated for Docker Desktop or Linux bridge networking | Running Docker daemon | +| ECS Fargate | OCI registry image; prefer an immutable digest | Automatic Gym SSH reverse tunnel | AWS/SSM/ECR/S3 infrastructure and TCP access to the task SSH sidecar | +| Enroot | Registry/Docker URI or local `.sqsh` | Shares the orchestrator host network | `enroot` CLI | +| Apptainer | Registry/Docker URI or local `.sif` | Shares the orchestrator host network | `apptainer` CLI | +| OpenSandbox | Provider-accessible OCI image | Set a reachable proxy URL when Gym's proxy is host-local | OpenSandbox service and credentials | +| Daytona | Provider-accessible OCI image or provider-supported snapshot configuration | Set a reachable proxy URL when Gym's proxy is host-local | Daytona service and credentials | +| OpenShell | Provider-accessible OCI image | Set a reachable proxy URL when Gym's proxy is host-local | OpenShell gateway and credentials when required | + +For the checked-in ECS Fargate configuration, export `AWS_PROFILE`, +`AWS_REGION`, and `AWS_DEFAULT_REGION`. Gym discovers the reference +infrastructure from `//ecs-sandbox/config` in SSM; +`ssm_project` defaults to `harbor`. That configuration must identify the ECS +cluster, subnets, security groups, task roles, ECR repository, S3 staging +bucket, and SSH-sidecar key material. The orchestrator must be able to reach +the task SSH sidecar on TCP port `52222`. See each provider YAML and README for +its current environment variables and service-specific options. + +Non-Docker providers do not invoke the Docker CLI. Supply a provider-compatible +image through `NEMO_GYM_LAB_SANDBOX_IMAGE` or the `sandbox_image` override. The +image must contain LAB's document tooling plus `bash`, `curl`, and `tar`, and +must permit writes under `/sandbox`. The first rollout starts a short-lived runtime +builder sandbox with outbound package-download access. The resulting portable, +content-addressed runtime is reused by later rollouts. + +OpenShell's Docker compute driver additionally requires `iproute2`, a +restricted `sandbox` user and group, and a work directory writable by that +identity. Images generated by LAB preparation include these requirements. + +For OpenSandbox, LAB treats each task's declared CPU, memory, disk, and GPU as +its limits. The checked-in agent configs set `opensandbox_request_fraction: +0.25`, which requests 25% of the CPU and memory limits while retaining the full +disk and GPU requests. This request/limit split permits deliberate +oversubscription when many LAB sandboxes run concurrently. It matches the +[Mini SWE Agent 2 example](https://github.com/NVIDIA-NeMo/Gym/blob/main/responses_api_agents/mini_swe_agent_2/configs/mini_swe_agent_2.yaml), +which requests 0.5 of a 2 CPU limit and 2 GiB of an 8 GiB memory limit. It +applies to the runtime builder, agent, and verifier. Raise the fraction toward +`1.0` if your cluster needs firmer CPU or memory reservations, or set it to +`null` to disable the split. For example, add +`+legal_agent_bench_benchmark_native_agent.responses_api_agents.legal_agent_bench_agent.opensandbox_request_fraction=0.5` +to the native `gym eval run` command below. + +The standard provider YAMLs define a top-level sandbox named `sandbox`. Select +one by adding its config and pointing the LAB agent at that name. This native +example works for Docker, ECS Fargate, Enroot, Apptainer, OpenSandbox, Daytona, +or OpenShell by replacing `` and ``: ```bash -export AWS_PROFILE=gym-ecs -export AWS_REGION=us-east-1 -export AWS_DEFAULT_REGION="$AWS_REGION" +gym eval run \ + --model-type vllm_model \ + --benchmark legal_agent_bench \ + --config nemo_gym/sandbox/providers//configs/.yaml \ + --split benchmark \ + --output results/legal_agent_bench_native__smoke.jsonl \ + --concurrency 1 \ + --limit 1 \ + +legal_agent_bench_benchmark_native_agent.responses_api_agents.legal_agent_bench_agent.sandbox_provider=sandbox \ + +legal_agent_bench_benchmark_native_agent.responses_api_agents.legal_agent_bench_agent.sandbox_image= ``` -The orchestrator must be able to reach the ECS task SSH sidecar on TCP port -`52222`. The SSM configuration must provide the cluster, subnets, security -groups, task roles, ECR repository, S3 staging bucket, and SSH-sidecar key -ARNs. Install the `sandbox` dependency extra as shown above. - -Build the shared LAB environment for `linux/amd64`, push it to ECR, and use its -immutable digest. The prepared LAB tasks currently share this environment: +For Hermes, Claude Code, or Codex, replace the benchmark and the native agent +prefix with the corresponding variant prefix. `sandbox_image` is passed through +unchanged, so use the native reference expected by the selected provider rather +than converting it in LAB. + +ECS Fargate automatically tunnels the rollout-scoped Gym policy proxy. Enroot +and Apptainer can reach a host-local proxy through their shared host network. +OpenSandbox, Daytona, and OpenShell cannot reach the orchestrator's loopback +address. Prefer exposing the credential-free Gym policy proxy on a route +reachable from the sandbox, then set `NEMO_GYM_SANDBOX_MODEL_BASE_URL` or the +`sandbox_model_base_url` override. Do not put credentials in that URL. If only +a directly authenticated model endpoint is reachable, set +`NEMO_GYM_LAB_SANDBOX_MODEL_API_KEY_ENV` to the name of a launcher environment +variable containing a narrowly scoped, short-lived model key. LAB copies that +key only into the agent sandbox as `LAB_POLICY_API_KEY`; it is not serialized +into the runner configuration or supplied to the runtime builder or verifier. +The evaluated agent can read its own environment, so this fallback accepts key +exposure to untrusted agent code. Use a dedicated key with the minimum required +permissions and rotate it after the run. Every provider's verifier sandbox must +also be able to reach the configured judge endpoint. + +OpenShell should use three separate policies through +`runtime_builder_provider_options`, `agent_sandbox_provider_options`, and +`verifier_sandbox_provider_options`. Give the builder only dependency-registry +egress, the agent only policy-proxy egress, and the verifier only judge egress. +The filesystem policy must allow the image runtime and +`/opt/legal-agent-bench` read-only and `/sandbox` writable. OpenShell injects a +policy-enforcing HTTP(S) proxy; the LAB runner opts Gym's inner HTTP client into +that proxy only for this provider. In the builder policy, configure the +`registry.npmjs.org` endpoint with `protocol: rest`, `access: read-only`, +`enforcement: enforce`, and `allow_encoded_slash: true`; npm uses encoded +slashes when resolving scoped packages such as the Claude Code dependency. +See OpenShell's +[policy schema](https://docs.nvidia.com/openshell/latest/reference/policy-schema#endpoint-object). + +Benchmark variants inherit their agent configuration, so supply the complete +phase-option maps through the decoded environment settings rather than adding +nested map keys on the command line: ```bash -LAB_ENV_DIR=resources_servers/legal_agent_bench/data/runtime/harbor_tasks/legal_agent_bench/antitrust-competition__analyze-antitrust-hsr-strategy/environment -ECR_REPOSITORY=.dkr.ecr..amazonaws.com/ -ECR_REGISTRY=${ECR_REPOSITORY%%/*} -ECR_REPOSITORY_NAME=${ECR_REPOSITORY#*/} -LAB_IMAGE_TAG=legal-agent-bench-smoke - -aws ecr get-login-password --region "$AWS_REGION" | - docker login --username AWS --password-stdin "$ECR_REGISTRY" -docker buildx build \ - --platform linux/amd64 \ - --tag "$ECR_REPOSITORY:$LAB_IMAGE_TAG" \ - --push \ - "$LAB_ENV_DIR" -LAB_IMAGE_DIGEST=$(aws ecr describe-images \ - --region "$AWS_REGION" \ - --repository-name "$ECR_REPOSITORY_NAME" \ - --image-ids "imageTag=$LAB_IMAGE_TAG" \ - --query 'imageDetails[0].imageDigest' \ - --output text) -export LAB_ECS_IMAGE="$ECR_REPOSITORY@$LAB_IMAGE_DIGEST" -docker pull --platform linux/amd64 "$LAB_ECS_IMAGE" +export NEMO_GYM_LAB_RUNTIME_BUILDER_PROVIDER_OPTIONS='{policy: /path/to/builder-policy.yaml}' +export NEMO_GYM_LAB_AGENT_SANDBOX_PROVIDER_OPTIONS='{policy: /path/to/agent-policy.yaml}' +export NEMO_GYM_LAB_VERIFIER_SANDBOX_PROVIDER_OPTIONS='{policy: /path/to/verifier-policy.yaml}' ``` -Run one native smoke through ECS: +The values are OmegaConf mappings and default to `{}`, so they have no effect +on providers that do not need phase-specific options. + +Before spending model tokens, exercise create, upload, execute, download, and +cleanup through the same public API. For a checked-in provider YAML: ```bash -gym eval run \ - --model-type vllm_model \ - --benchmark legal_agent_bench \ - --config nemo_gym/sandbox/providers/ecs_fargate/configs/ecs_fargate.yaml \ - --split benchmark \ - --output results/legal_agent_bench_native_ecs_smoke.jsonl \ - --concurrency 1 \ - --limit 1 \ - +legal_agent_bench_benchmark_native_agent.responses_api_agents.legal_agent_bench_agent.sandbox_provider=sandbox \ - "+legal_agent_bench_benchmark_native_agent.responses_api_agents.legal_agent_bench_agent.sandbox_image=${LAB_ECS_IMAGE}" \ - +legal_agent_bench_benchmark_native_agent.responses_api_agents.legal_agent_bench_agent.runtime_docker_platform=linux/amd64 +python responses_api_agents/legal_agent_bench_agent/scripts/smoke_provider.py \ + --config nemo_gym/sandbox/providers//configs/.yaml \ + --image ``` -ECS tunnels the rollout-scoped Gym policy-model URL into the agent task, so -model credentials remain in Gym and `sandbox_model_base_url` is unnecessary. -An explicit `sandbox_model_base_url` still bypasses the tunnel when the endpoint -is already reachable from the task. The verifier contacts the configured judge -endpoint directly, so that endpoint must be reachable from the Fargate VPC. -Each completed rollout creates one agent task followed by one verifier task; -both are stopped during cleanup. Artifacts use the same local paths documented -above. +You can also use `--provider ` instead of `--config ...` to smoke a +provider's constructor defaults. A passing lifecycle smoke does not test the +policy or judge endpoints; follow it with the one-task `gym eval run` above. ## Manage servers separately @@ -469,6 +539,10 @@ task. The default roots are `results/legal_agent_bench/native_jobs`, `results/legal_agent_bench/claude_code_jobs`, and `results/legal_agent_bench/codex_jobs`. Harbor uses `results/legal_agent_bench/harbor_jobs`. +Set `NEMO_GYM_LAB_RESULTS_DIR` before `gym eval run` to redirect the native and +configurable artifact root, for example to VM-native storage when Gym runs in +a Linux VM over a macOS-shared checkout. The rollout JSONL paths remain the +ones supplied with `--output`. Each configurable output row includes `artifact_dir`, `run_summary_path`, `agent_trace_path`, `agent_stdout_path`, `agent_stderr_path`, diff --git a/benchmarks/legal_agent_bench/tests/test_prepare.py b/benchmarks/legal_agent_bench/tests/test_prepare.py index 962c140096..ebb7ed4a64 100644 --- a/benchmarks/legal_agent_bench/tests/test_prepare.py +++ b/benchmarks/legal_agent_bench/tests/test_prepare.py @@ -139,6 +139,9 @@ def test_benchmark_config_is_isolated_and_resolves_shared_cache_paths() -> None: assert agent.agent_server_module == "responses_api_agents.legal_agent_bench_native_agent.app" assert agent.runtime_tasks_dir == resource.harbor_tasks_dir assert agent.skills_dir == resource.harness_skills_dir + assert agent.runtime_builder_provider_options == {} + assert agent.agent_sandbox_provider_options == {} + assert agent.verifier_sandbox_provider_options == {} assert agent.agent_kwargs.max_turns == 60 assert len(agent.datasets) == 1 assert agent.datasets[0].type == "benchmark" @@ -203,8 +206,49 @@ def test_configurable_benchmark_variants_resolve(filename, expected_agent, expec assert agent.agent_kwargs.claude_code_version == "2.1.211" elif expected_module == "responses_api_agents.codex_agent.app": assert agent.agent_kwargs.codex_version == "0.144.4" + assert agent.agent_kwargs.cwd == "/sandbox/nemo-gym-legal-agent-bench/workspace/output" assert agent.datasets[0].type == "benchmark" assert agent.runtime_tasks_dir.endswith("data/runtime/harbor_tasks/legal_agent_bench") + assert agent.runtime_builder_provider_options == {} + assert agent.agent_sandbox_provider_options == {} + assert agent.verifier_sandbox_provider_options == {} + + +@pytest.mark.parametrize( + ("filename", "expected_agent"), + [ + ("config.yaml", "legal_agent_bench_benchmark_native_agent"), + ("config_hermes.yaml", "legal_agent_bench_benchmark_hermes_agent"), + ("config_claude_code.yaml", "legal_agent_bench_benchmark_claude_code_agent"), + ("config_codex.yaml", "legal_agent_bench_benchmark_codex_agent"), + ], +) +def test_configurable_variants_decode_phase_provider_options_from_environment( + monkeypatch, filename, expected_agent +) -> None: + monkeypatch.setenv( + "NEMO_GYM_LAB_RUNTIME_BUILDER_PROVIDER_OPTIONS", + "{policy: /tmp/lab-builder-policy.yaml}", + ) + monkeypatch.setenv( + "NEMO_GYM_LAB_AGENT_SANDBOX_PROVIDER_OPTIONS", + "{policy: /tmp/lab-agent-policy.yaml}", + ) + monkeypatch.setenv( + "NEMO_GYM_LAB_VERIFIER_SANDBOX_PROVIDER_OPTIONS", + "{policy: /tmp/lab-verifier-policy.yaml}", + ) + initial_config = OmegaConf.merge( + OmegaConf.load(BENCHMARK_DIR / filename), + GlobalConfigDictParserConfig.NO_MODEL_GLOBAL_CONFIG_DICT, + ) + + resolved = GlobalConfigDictParser().parse_no_environment(initial_global_config_dict=initial_config) + agent = resolved[expected_agent].responses_api_agents.legal_agent_bench_agent + + assert agent.runtime_builder_provider_options == {"policy": "/tmp/lab-builder-policy.yaml"} + assert agent.agent_sandbox_provider_options == {"policy": "/tmp/lab-agent-policy.yaml"} + assert agent.verifier_sandbox_provider_options == {"policy": "/tmp/lab-verifier-policy.yaml"} @pytest.mark.parametrize( diff --git a/nemo_gym/sandbox/providers/apptainer/configs/apptainer.yaml b/nemo_gym/sandbox/providers/apptainer/configs/apptainer.yaml new file mode 100644 index 0000000000..e23b5e1431 --- /dev/null +++ b/nemo_gym/sandbox/providers/apptainer/configs/apptainer.yaml @@ -0,0 +1,18 @@ +# Apptainer sandbox provider config. +# +# `sandbox` is the instance name an agent references via `sandbox_provider: +# sandbox`; the child key `apptainer` selects the provider class. Provider +# defaults use a private host staging directory mounted at `/sandbox` and run +# each sandbox as a persistent Apptainer instance. +# +# Requires the `apptainer` CLI on PATH. See ../README.md for image formats, +# fakeroot, cgroup, bind, and writable-tmpfs options. +sandbox: + default_metadata: + sandbox-api: apptainer + apptainer: + create: + # Apptainer otherwise bind-mounts the host's /tmp and home into every + # instance. Containment gives each sandbox private writable scratch + # directories and prevents state from leaking between concurrent tasks. + extra_start_args: ["--containall"] diff --git a/nemo_gym/server_utils.py b/nemo_gym/server_utils.py index d9a91392de..75126a030e 100644 --- a/nemo_gym/server_utils.py +++ b/nemo_gym/server_utils.py @@ -85,6 +85,10 @@ class GlobalAIOHTTPAsyncClientConfig(BaseModel): global_aiohttp_connector_limit_per_host: int = 1024 global_aiohttp_client_request_debug: bool = False + # Disabled by default so ordinary Gym traffic cannot be silently redirected by + # ambient proxy variables. Sandboxed runtimes whose provider deliberately + # exposes policy-enforced egress through HTTP(S)_PROXY may opt in. + global_aiohttp_trust_env: bool = False global_aiohttp_tcp_keepalive_idle_seconds: int = Field( default=60, @@ -159,6 +163,7 @@ def set_global_aiohttp_client(cfg: GlobalAIOHTTPAsyncClientConfig) -> ClientSess ), timeout=ClientTimeout(), cookie_jar=DummyCookieJar(), + trust_env=cfg.global_aiohttp_trust_env, ) global _GLOBAL_AIOHTTP_CLIENT diff --git a/resources_servers/legal_agent_bench/README.md b/resources_servers/legal_agent_bench/README.md index 6dd95629b0..2b5b14eb68 100644 --- a/resources_servers/legal_agent_bench/README.md +++ b/resources_servers/legal_agent_bench/README.md @@ -17,10 +17,11 @@ selects the native loop; Harbor is available explicitly as ## Requirements -- Python 3.13.13 and [uv](https://docs.astral.sh/uv/) -- Docker with a running daemon for local execution and portable runtime - provisioning. ECS Fargate execution is also supported with an immutable - registry image; see the benchmark README. +- Python 3.13.14 and [uv](https://docs.astral.sh/uv/) +- One Gym sandbox backend for the native/configurable runner: Docker, ECS + Fargate, Enroot, Apptainer, OpenSandbox, Daytona, or OpenShell. Docker is the + zero-configuration default; the provider matrix is in the benchmark README. +- Docker specifically when using the separate Harbor compatibility variant - An OpenAI-compatible policy endpoint and judge endpoint - At least 10 GB of free working space for preparation and the first Docker build @@ -35,9 +36,9 @@ uses the configured policy model endpoint. Access to the configured policy and judge endpoints is still required and may itself be metered or paid. The pinned source download is about 579 MiB; allow a few GiB of free working -space during preparation. The first task also builds a -document-tooling Docker image and can take several minutes. Later tasks reuse -Docker layers. +space during preparation. With the default Docker provider, the first task also +builds a document-tooling image and can take several minutes. Other providers +reuse a supplied compatible image. From a fresh clone, create the repository environment: @@ -45,9 +46,12 @@ From a fresh clone, create the repository environment: uv venv --python 3.13.14 source .venv/bin/activate uv sync --extra dev -docker info >/dev/null ``` +Run `docker info` when using Docker or Harbor. Install the repository's +`sandbox` extra and complete the provider-specific setup when using an +SDK-backed provider. + Keep this environment activated when running LAB commands. Invoke `gym` directly rather than `uv run gym` for workflows that start servers: Ray starts components from their own working directories, which can conflict with uv's @@ -140,7 +144,7 @@ The commands below describe the lower-level resource-server and standalone agent workflow. The resource-server discovery config predates the benchmark wrapper and starts -the Harbor compatibility agent: +the Docker-only Harbor compatibility agent: ```bash gym env start \ @@ -226,8 +230,9 @@ gym env start \ This setting is forwarded to each task verifier as `LAB_JUDGE_PARALLELISM`. -The task container requires network access because the policy agent and -verifier call configured endpoints. +The selected agent sandbox requires access to Gym's policy proxy, and the +separate verifier sandbox requires access to the configured judge endpoint. +See the benchmark README for provider-specific proxy routing. ### Output, context, and timeout limits @@ -289,10 +294,13 @@ are browsable and safe under concurrent execution. `` is the configured normalized, and the run ID is an eight-character unique suffix. Starting or validating an agent server does not create an empty session directory; the directory is created by its first rollout. -The Docker image is content-addressed from the task environment and reused -across harnesses. Agent and verifier phases use separate containers: the agent +Docker can build a content-addressed image from the task environment +automatically. Other providers use the configured image reference unchanged. +The selected provider is also used to build the portable harness runtime and is +reused for the agent and verifier phases. Agent and verifier phases use +separate sandboxes: the agent sees only its selected Gym package and public task inputs, while the verifier -starts after that container is destroyed and receives the completed LAB run +starts after the agent sandbox is destroyed and receives the completed LAB run read-only. To inspect a native or configurable smoke run: @@ -323,9 +331,14 @@ or more criteria fail; use `criteria_pass_rate` to see partial success. `model_connection_failed`, `agent_timed_out`, `verifier_failed`, `verifier_timed_out`, `sandbox_failed`, `task_failed`, `configuration_failed`, and `mask_sample`. Before the harness starts, the - runner checks the policy endpoint from inside Docker. On Docker Desktop, - derived loopback model URLs are automatically routed through - `host.docker.internal`; `sandbox_model_base_url` is an explicit override. + runner checks the policy endpoint from inside the selected sandbox. Docker + translates derived loopback URLs when needed, ECS Fargate creates a reverse + tunnel, Enroot and Apptainer share the host network, and remote providers need + an explicitly reachable `sandbox_model_base_url` when Gym's proxy is + host-local. Prefer a credential-free reachable proxy. If a direct endpoint is + the only option, the configurable runner supports an agent-only key through + `sandbox_model_api_key_env`; use a narrowly scoped, short-lived key because + the evaluated agent can read its own environment. Connectivity, harness, sandbox, and verifier failures are masked, skip judging when applicable, and carry `_ng_failure_class`, which routes them to the failure sidecar for bounded retry. Task-loading and harness-configuration @@ -344,8 +357,8 @@ or more criteria fail; use `criteria_pass_rate` to see partial success. the model explicitly and its internal policy proxy does not implement model discovery. This does not suppress access logs for actual chat-completion requests. -- If Docker appears idle on the first rollout, inspect `docker ps` and the - `gym env start` terminal; Harbor is normally building the task image. +- If Docker appears idle on the first Harbor rollout, inspect `docker ps` and + the `gym env start` terminal; Harbor is normally building the task image. - Do not copy or publish `data/runtime/`: it can contain local judge credentials. - Results are revision-specific and should not be compared directly with runs that use a different task snapshot or skill set. diff --git a/responses_api_agents/legal_agent_bench_agent/README.md b/responses_api_agents/legal_agent_bench_agent/README.md index 9cb00f8820..f4e3a19362 100644 --- a/responses_api_agents/legal_agent_bench_agent/README.md +++ b/responses_api_agents/legal_agent_bench_agent/README.md @@ -13,22 +13,27 @@ configurations select Gym's Hermes, Claude Code, and Codex agents through: - `agent_kwargs` The runner also accepts the policy `model_server`, LAB runtime task and skill -paths, concurrency and timeout settings, `sandbox_provider`, an optional -`sandbox_image`, Docker platform, network and model URL overrides, and a result -directory. See `configs/` for complete examples. +paths, concurrency and layered timeout settings (including +`sandbox_staging_timeout_seconds` for large archive extraction and collection), `sandbox_provider`, an optional +`sandbox_image`, a model URL override, Docker-only network settings, +`opensandbox_request_fraction`, and a result directory. See `configs/` for +complete examples. Those configurations include the LAB `resources_only.yaml` definition so Gym starts one resource server and only the selected agent; the combined compatibility config is reserved for Harbor. ## Execution boundary -For each rollout, the runner validates `instance_id`, builds or reuses the -content-addressed LAB image, provisions a portable Linux runtime for the -selected harness, and starts an agent-only sandbox. Runtimes are immutable and -content-addressed by their setup script, requirements, configured CLI pin, LAB -image identity and architecture, Gym packaging metadata, and installed -`nemo_gym` source. An interprocess lock and atomic publication prevent -concurrent evaluations from rewriting a runtime that is being built or used. +For each rollout, the runner validates `instance_id`, resolves a +provider-compatible LAB image, provisions a portable Linux runtime for the +selected harness through that same sandbox provider, and starts an agent-only +sandbox. Docker can build the LAB image automatically as a local convenience; +every other provider receives the configured image reference unchanged. +Runtimes are immutable and content-addressed by their setup script, +requirements, configured CLI pin, LAB image and provider identity, Gym +packaging metadata, and installed `nemo_gym` source. An interprocess lock and +atomic publication prevent concurrent evaluations from rewriting a runtime +that is being built or used. Dependency provisioning exposes an explicit allowlist containing only the installer and required Gym package files. It never mounts the repository, @@ -49,16 +54,35 @@ tree, rubric tests, and verifier-only judge credentials. The verifier sees deliverables at `lab-run/output`, preserving LAB's contract without sharing a process namespace or writable filesystem with the agent. -The default provider is Docker. The runner translates derived host-loopback -model URLs to `host.docker.internal` on Docker Desktop and registers the -equivalent host gateway for Linux bridge networking. Gym's ECS Fargate provider -can be selected through `sandbox_provider`; non-Docker providers require an -immutable registry `sandbox_image`. ECS uses -Gym's provider-native SSH reverse tunnel to expose the rollout-scoped policy -proxy as `LAB_POLICY_MODEL_URL`, keeping model credentials out of the sandbox. -An explicit `sandbox_model_base_url` is used unchanged and disables that ECS -tunnel. Before importing the selected harness, the in-container runner -performs a bounded TCP connectivity check and writes +The default provider is Docker, but the common path uses only Gym's +`AsyncSandbox` operations: start, upload, execute, download, and stop. Docker, +ECS Fargate, Enroot, Apptainer, OpenSandbox, Daytona, and OpenShell can all be +selected with `sandbox_provider` without changing LAB code. The runtime +builder, agent, and verifier use the same selected provider. No common command +requests root, mounts a host path, invokes a container CLI, interprets the +provider's image format, or assumes the image's configured user. Transfer +archives are made readable to non-root image users, and all large runtime, +scratch, and transfer writes stay under the sandbox's writable `/sandbox` +tree rather than a provider's potentially bounded `/tmp`. +LAB's generated image also includes OpenShell's Docker-driver prerequisites: +`iproute2`, a high-UID `sandbox` identity, and a writable work directory. + +Docker's automatic image build and host-network translation are optional +provider conveniences. ECS uses Gym's provider-native SSH reverse tunnel to +expose the rollout-scoped policy proxy as `LAB_POLICY_MODEL_URL`, keeping model +credentials out of the sandbox. Enroot and Apptainer share the orchestrator's +host network. OpenSandbox, Daytona, and OpenShell need a +`sandbox_model_base_url` that is reachable from their remote sandbox whenever +Gym's derived policy-proxy URL is host-local. An explicit +`sandbox_model_base_url` is always used unchanged. A reachable credential-free +Gym proxy is preferred. For a directly authenticated endpoint, +`sandbox_model_api_key_env` may name a launcher environment variable whose +value is copied into the agent sandbox as `LAB_POLICY_API_KEY`. It is not +serialized into runner metadata or supplied to the runtime builder or verifier, +but the evaluated agent can read its own environment. Use this fallback only +with a narrowly scoped, short-lived key and rotate it after the run. Before +importing the selected harness, the in-sandbox runner +performs a bounded, proxy-aware HTTP connectivity check and writes `runtime/runner_status.json`. Connectivity, harness, sandbox, and verifier failures set `mask_sample`, skip the judge when applicable, and carry `_ng_failure_class` so rollout collection sends them to the failure sidecar for @@ -70,6 +94,44 @@ them in the main rollout JSONL. For Hermes, the runner also disables its optiona pricing and context-metadata probes; Gym supplies the selected model explicitly, and actual model-call access logging is enabled. +For OpenSandbox, each LAB task's declared resources are the burst limits. By +default, `opensandbox_request_fraction: 0.25` sets CPU and memory scheduling +requests to 25% of those limits so concurrent evaluations can pack densely. +Disk and any requested GPU are not oversubscribed. The same split is applied to +the runtime builder, agent, and verifier sandboxes. This is the same ratio used +by Gym's +[Mini SWE Agent 2 config](../mini_swe_agent_2/configs/mini_swe_agent_2.yaml), +where 0.5 CPU and 2 GiB are requested against limits of 2 CPU and 8 GiB. Set +the value closer to `1.0` when the cluster needs stronger reservations, or to +`null` to let the OpenSandbox server use one resource map for both requests +and limits. + +Codex currently has the largest portable runtime archive. On a remote +OpenSandbox deployment, keep the provider's +`connection.request_timeout_s` at least as high as the runner's 900-second +staging timeout and reduce concurrency if parallel file transfers saturate the +OpenSandbox file API. These deadlines cover infrastructure transfer and +staging; they do not change the model's agent-phase deadline. + +Providers that enforce different network access by phase can use +`runtime_builder_provider_options`, `agent_sandbox_provider_options`, and +`verifier_sandbox_provider_options`. For OpenShell, use separate least-privilege +policies: allow only package registries in the runtime builder, only the Gym +policy proxy in the agent, and only the judge endpoint in the verifier. The LAB +image's global Node document libraries are exposed through an explicit +`NODE_PATH`; OpenShell's injected HTTP(S) proxy is trusted only inside its inner +runner so policy-model traffic passes through the gateway policy engine. The +builder's `registry.npmjs.org` endpoint must use `protocol: rest`, `access: +read-only`, `enforcement: enforce`, and `allow_encoded_slash: true`, because npm +uses encoded slashes for scoped packages. + +The checked-in configs decode the whole phase-option maps from +`NEMO_GYM_LAB_RUNTIME_BUILDER_PROVIDER_OPTIONS`, +`NEMO_GYM_LAB_AGENT_SANDBOX_PROVIDER_OPTIONS`, and +`NEMO_GYM_LAB_VERIFIER_SANDBOX_PROVIDER_OPTIONS`. This avoids inherited-config +merge restrictions when a benchmark variant needs to add provider-specific +keys such as `policy`; each setting defaults to `{}`. + Artifacts are written below `results/legal_agent_bench/_jobs///_/`. For example, the default native loop uses @@ -84,6 +146,10 @@ top-level `run_summary.json`. The rollout response also includes direct paths for the summary, trajectory, stdout, stderr, output directory, and verifier report. +Set `NEMO_GYM_LAB_RESULTS_DIR` to redirect the artifact root. This is useful +when Gym runs inside a Linux VM: use the VM's native filesystem instead of a +macOS-shared mount to avoid cross-OS permissions and ownership behavior. + For setup, five copy-paste smoke commands, and result inspection, see the [benchmark README](../../benchmarks/legal_agent_bench/README.md#test-the-various-harnesses). Run `gym` from an activated repository environment rather than prefixing @@ -94,7 +160,7 @@ server-starting commands with `uv run`. A harness can be selected without changing this runner when it: 1. is implemented as a Gym Responses API agent; -2. can run inside the LAB Linux container; +2. can run inside the LAB Linux sandbox image; 3. has a portable dependency script at `responses_api_agents//scripts/_deps.sh`; and 4. can use the configured OpenAI-compatible policy endpoint. @@ -104,5 +170,7 @@ that inherits it. CLI versions belong in the harness configuration's `agent_kwargs`; provisioning derives its exact package specification from that pin and rejects unpinned Claude Code or Codex configurations. -Docker remains the zero-configuration local backend. See the benchmark README -for ECS Fargate requirements and provider overrides. +Docker is the zero-configuration local backend. See the benchmark README for +the provider matrix, image and networking requirements, lifecycle smoke test, +and one-task override examples. The separate Harbor compatibility variant does +not use this runner and is Docker-only. diff --git a/responses_api_agents/legal_agent_bench_agent/app.py b/responses_api_agents/legal_agent_bench_agent/app.py index 6dec016526..430643c96f 100644 --- a/responses_api_agents/legal_agent_bench_agent/app.py +++ b/responses_api_agents/legal_agent_bench_agent/app.py @@ -13,7 +13,6 @@ import re import shutil import stat -import subprocess import sys import tarfile import tempfile @@ -24,10 +23,19 @@ from urllib.parse import urlsplit, urlunsplit from uuid import uuid4 + +# Server processes may run from a component-local working directory with an +# installed ``nemo_gym`` ahead of the checkout on sys.path. Derive the source +# root from this file and put it first so LAB's vendored runtime assets are +# resolved from the same checkout as the agent implementation on every host. +PACKAGE_DIR = Path(__file__).resolve().parent +PARENT_DIR = PACKAGE_DIR.parents[1] +if str(PARENT_DIR) not in sys.path: + sys.path.insert(0, str(PARENT_DIR)) + from fastapi import Body, Request from pydantic import ConfigDict, Field, PrivateAttr -from nemo_gym import PARENT_DIR from nemo_gym.base_resources_server import BaseRunRequest, BaseVerifyResponse from nemo_gym.base_responses_api_agent import BaseResponsesAPIAgentConfig, SimpleResponsesAPIAgent from nemo_gym.config_types import ModelServerRef, ResourcesServerRef @@ -54,7 +62,6 @@ ) -PACKAGE_DIR = Path(__file__).resolve().parent PORTABLE_PYTHON_SH = PACKAGE_DIR / "setup_scripts" / "_portable_python.sh" DATASET_ALIAS = "legal_agent_bench" INITIAL_USER_PROMPT = "Please begin working on the task described in the system prompt." @@ -73,8 +80,20 @@ r"(?:-[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?" r"(?:\+[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?$" ) -IMMUTABLE_IMAGE = re.compile(r"^[^@\s]+@sha256:[0-9a-f]{64}$") -CODEX_MODEL_CATALOG_PATH = "/trajectories_mount/codex_model_catalog.json" +SANDBOX_ROOT = "/sandbox/nemo-gym-legal-agent-bench" +SANDBOX_AGENT_SOURCE = f"{SANDBOX_ROOT}/agent_source" +SANDBOX_AGENT_DEPS = f"{SANDBOX_ROOT}/agent_deps" +SANDBOX_RUNTIME = f"{SANDBOX_ROOT}/runtime" +SANDBOX_WORKSPACE = f"{SANDBOX_ROOT}/workspace" +SANDBOX_VDR = f"{SANDBOX_WORKSPACE}/vdr" +SANDBOX_OUTPUT = f"{SANDBOX_WORKSPACE}/output" +SANDBOX_SCRATCH = f"{SANDBOX_WORKSPACE}/scratch" +SANDBOX_SKILLS = f"{SANDBOX_WORKSPACE}/skills" +SANDBOX_VERIFIER = f"{SANDBOX_ROOT}/verifier" +SANDBOX_TESTS = f"{SANDBOX_VERIFIER}/tests" +SANDBOX_LOGS = f"{SANDBOX_VERIFIER}/logs" +LAB_SANDBOX_ENV = {"NODE_PATH": "/usr/local/lib/node_modules"} +CODEX_MODEL_CATALOG_PATH = f"{SANDBOX_RUNTIME}/codex_model_catalog.json" CODEX_MODEL_CATALOG = { "models": [ { @@ -102,15 +121,33 @@ ] } + +def codex_model_catalog(model_name: str) -> dict[str, Any]: + """Return Codex metadata for both proxied and direct policy routing. + + Codex indexes the catalog by the configured model slug. Gym's in-process + proxy uses ``gym-policy-model``, while a direct remote endpoint requires the + provider's real model name. Keep the proxy alias and add the selected model + as a second entry so one portable runtime works for both routes. + """ + catalog = json.loads(json.dumps(CODEX_MODEL_CATALOG)) + if model_name != catalog["models"][0]["slug"]: + direct_model = json.loads(json.dumps(catalog["models"][0])) + direct_model["slug"] = model_name + direct_model["display_name"] = model_name + catalog["models"].append(direct_model) + return catalog + + GENERIC_HARNESS_PREAMBLE = """\ You are an AI agent running in an automated Legal Agent Bench evaluation. ## Workspace layout -- Source documents are under `/workspace/vdr` and `$VDR_DIR`. Treat them as read-only. -- Write every final deliverable under `/workspace/output` and `$OUTPUT_DIR`. -- Use `/workspace/workspace` for scratch files. -- Skill manuals and their supporting assets are under `/workspace/skills`. +- Source documents are under `$VDR_DIR`. Treat them as read-only. +- Write every final deliverable under `$OUTPUT_DIR`. +- Use `$WORKSPACE_DIR` for scratch files. +- Skill manuals and their supporting assets are under `$SKILLS_DIR`. - Do not search for task configuration, rubric, verifier, test, or judge files. They are intentionally unavailable while you work. @@ -124,35 +161,50 @@ import inspect import json import os -import socket import sys from pathlib import Path from types import SimpleNamespace -from urllib.parse import urlsplit - -sys.path.insert(0, "/agent_source_mount") -os.environ["PATH"] = os.environ.get("PATH", "") + os.pathsep + "/agent_deps_mount/bin" -os.environ["VDR_DIR"] = "/workspace/vdr" -os.environ["OUTPUT_DIR"] = "/workspace/output" -os.environ["WORKSPACE_DIR"] = "/workspace/workspace" -os.environ["SKILLS_DIR"] = "/workspace/skills" -os.environ["HOME"] = "/workspace/workspace" +from urllib.error import HTTPError +from urllib.parse import urlsplit, urlunsplit +from urllib.request import Request, urlopen + +SANDBOX_ROOT = "/sandbox/nemo-gym-legal-agent-bench" +AGENT_SOURCE = f"{SANDBOX_ROOT}/agent_source" +AGENT_DEPS = f"{SANDBOX_ROOT}/agent_deps" +RUNTIME = f"{SANDBOX_ROOT}/runtime" +WORKSPACE = f"{SANDBOX_ROOT}/workspace" + +sys.path.insert(0, AGENT_SOURCE) +os.environ["PATH"] = os.environ.get("PATH", "") + os.pathsep + f"{AGENT_DEPS}/bin" +os.environ["VDR_DIR"] = f"{WORKSPACE}/vdr" +os.environ["OUTPUT_DIR"] = f"{WORKSPACE}/output" +os.environ["WORKSPACE_DIR"] = f"{WORKSPACE}/scratch" +os.environ["SKILLS_DIR"] = f"{WORKSPACE}/skills" +os.environ["HOME"] = f"{WORKSPACE}/scratch" +os.environ["TMPDIR"] = f"{WORKSPACE}/scratch" # The inner runner receives its complete configuration below. Prevent Gym's # shared aiohttp client from invoking Hydra's CLI config loader in the output # directory when the native agent makes its first model request. os.environ.setdefault("NEMO_GYM_CONFIG_DICT", "{}") -os.chdir("/workspace/output") +os.chdir(os.environ["OUTPUT_DIR"]) from nemo_gym.config_types import ModelServerRef, ResourcesServerRef from nemo_gym.openai_utils import NeMoGymResponseCreateParamsNonStreaming -from nemo_gym.server_utils import ServerClient +from nemo_gym.server_utils import ( + GlobalAIOHTTPAsyncClientConfig, + ServerClient, + is_global_aiohttp_client_setup, + set_global_aiohttp_client, +) -runner = json.loads(Path("/trajectories_mount/runner.json").read_text()) +runner = json.loads(Path(f"{RUNTIME}/runner.json").read_text()) # ECS Fargate injects this value after resolving the host-side policy URL through # its SSH reverse tunnel. Other providers use the URL persisted in runner.json. model_url = os.environ.get("LAB_POLICY_MODEL_URL", runner["model_url"]).rstrip("/") +model_api_key = os.environ.get("LAB_POLICY_API_KEY") +model_url_root = model_url.removesuffix("/v1").rstrip("/") model_url_v1 = model_url if model_url.endswith("/v1") else model_url + "/v1" -status_path = Path("/trajectories_mount/runner_status.json") +status_path = Path(f"{RUNTIME}/runner_status.json") def write_status(*, ok, phase, error=None): @@ -163,12 +215,27 @@ def write_status(*, ok, phase, error=None): parsed_model_url = urlsplit(model_url) if not parsed_model_url.hostname: raise ValueError(f"Policy model URL has no hostname: {model_url!r}") - model_port = parsed_model_url.port or (443 if parsed_model_url.scheme == "https" else 80) - with socket.create_connection( - (parsed_model_url.hostname, model_port), - timeout=runner.get("model_connect_timeout_seconds", 10), - ): - pass + # Use a proxy-aware HTTP request rather than a raw TCP socket. Policy-enforcing + # providers such as OpenShell expose permitted egress through HTTP(S)_PROXY and + # intentionally block direct sockets. Strip only the terminal API-version segment + # so Gym policy proxies can answer their quiet 200 liveness route while preserving + # any rollout-scoped path prefix. + probe_path = parsed_model_url.path.removesuffix("/v1") or "/" + if not probe_path.endswith("/"): + probe_path += "/" + probe_url = urlunsplit(parsed_model_url._replace(path=probe_path, query="", fragment="")) + try: + probe_headers = {"Authorization": f"Bearer {model_api_key}"} if model_api_key else {} + with urlopen( + Request(probe_url, headers=probe_headers, method="GET"), + timeout=runner.get("model_connect_timeout_seconds", 10), + ): + pass + except HTTPError as exc: + # Any application response proves transport connectivity, but authentication + # or policy denials mean the agent cannot use the endpoint. + if exc.code in {401, 403}: + raise except Exception as exc: message = f"Policy model is unreachable from the LAB sandbox: {model_url} ({type(exc).__name__}: {exc})" write_status(ok=False, phase="model_connectivity", error=message) @@ -193,10 +260,17 @@ def empty_endpoint_model_metadata(*args, **kwargs): return {} hermes_model_metadata.fetch_endpoint_model_metadata = empty_endpoint_model_metadata + # Current Hermes releases perform a second, optional local-server + # context probe when endpoint metadata is empty. Gym's rollout proxy is + # neither Ollama, LM Studio, llama.cpp, nor a public model catalog, so + # probing those routes only emits misleading 404s. Preserve Hermes's + # normal fallback context behavior without issuing discovery requests. + hermes_model_metadata._query_local_context_length = empty_endpoint_model_metadata hermes_usage_pricing.fetch_endpoint_model_metadata = empty_endpoint_model_metadata client = ServerClient.model_construct( global_config_dict={ + "global_aiohttp_trust_env": runner.get("http_proxy_from_environment", False), "policy_model": { "responses_api_models": { "policy_model": {}, @@ -204,7 +278,10 @@ def empty_endpoint_model_metadata(*args, **kwargs): } } ) - client._build_server_base_url = lambda _cfg: model_url + # ServerClient appends protocol paths such as /v1/responses itself, while + # SDK-style harnesses consume the versioned base URL below. Accept either a + # root or /v1 sandbox override without duplicating the API version. + client._build_server_base_url = lambda _cfg: model_url_root base = { "host": "0.0.0.0", @@ -216,22 +293,82 @@ def empty_endpoint_model_metadata(*args, **kwargs): } kwargs = {key: value for key, value in base.items() if key in config_class.model_fields} kwargs.update(runner.get("agent_kwargs") or {}) + if model_api_key and "model" in config_class.model_fields and not kwargs.get("model"): + # Gym model proxies substitute their configured model when a harness + # omits it or uses the proxy-local placeholder. A direct provider + # endpoint cannot do that, so give SDK/CLI harnesses the selected model. + kwargs["model"] = runner["model_name"] config = config_class(**kwargs) agent = agent_class(config=config, server_client=client) + if model_api_key: + # Direct remote endpoints need the configured provider credential. The + # secret enters only through the agent sandbox environment: it is not + # serialized into runner.json or staged into the verifier sandbox. + original_post = client.post + + async def authenticated_post(*args, **kwargs): + headers = dict(kwargs.pop("headers", {}) or {}) + headers.setdefault("Authorization", f"Bearer {model_api_key}") + return await original_post(*args, headers=headers, **kwargs) + + object.__setattr__(client, "post", authenticated_post) + + if "anthropic_api_key" in config_class.model_fields: + object.__setattr__(config, "anthropic_api_key", model_api_key) + if "openai_api_key" in config_class.model_fields: + object.__setattr__(config, "openai_api_key", model_api_key) + if runner.get("disable_endpoint_metadata_probe"): + # Hermes constructs its OpenAI client inside responses() with a + # local-proxy sentinel. Replace only that constructor argument for + # this inner process so direct endpoints receive the real key. + import run_agent as hermes_run_agent + + original_ai_agent = hermes_run_agent.AIAgent + + class AuthenticatedAIAgent(original_ai_agent): + def __init__(self, *args, **kwargs): + kwargs["api_key"] = model_api_key + super().__init__(*args, **kwargs) + + hermes_run_agent.AIAgent = AuthenticatedAIAgent + if hasattr(agent, "resolve_model_base_url"): object.__setattr__(agent, "resolve_model_base_url", lambda *args, **kwargs: model_url_v1) if hasattr(agent, "_resolve_model_base_url"): object.__setattr__(agent, "_resolve_model_base_url", lambda *args, **kwargs: model_url_v1) if hasattr(agent, "_resolve_base_url"): - object.__setattr__(agent, "_resolve_base_url", lambda *args, **kwargs: model_url) + # Claude Code appends /v1/messages to ANTHROPIC_BASE_URL itself. Give + # its private root resolver the unversioned URL even when the provider + # override was supplied with a terminal /v1 segment. + object.__setattr__(agent, "_resolve_base_url", lambda *args, **kwargs: model_url_root) body = NeMoGymResponseCreateParamsNonStreaming.model_validate(runner["responses_create_params"]) + if model_api_key and body.model is None: + body = body.model_copy(update={"model": runner["model_name"]}) response_kwargs = {"body": body} if "request" in inspect.signature(agent.responses).parameters: response_kwargs["request"] = SimpleNamespace(path_params={}) - response = asyncio.run(agent.responses(**response_kwargs)) - Path("/trajectories_mount/response.json").write_text(response.model_dump_json()) + async def invoke_agent(): + # This runner calls the selected agent directly rather than starting its + # Gym webserver, so initialize the shared HTTP client that webserver + # startup would normally create. OpenShell injects policy-enforced proxy + # variables; other providers retain aiohttp's direct-connection default. + http_client = None + if not is_global_aiohttp_client_setup(): + http_client = set_global_aiohttp_client( + GlobalAIOHTTPAsyncClientConfig( + global_aiohttp_trust_env=runner.get("http_proxy_from_environment", False), + ) + ) + try: + return await agent.responses(**response_kwargs) + finally: + if http_client is not None: + await http_client.close() + + response = asyncio.run(invoke_agent()) + Path(f"{RUNTIME}/response.json").write_text(response.model_dump_json()) except Exception as exc: write_status(ok=False, phase="agent_execution", error=f"{type(exc).__name__}: {exc}") raise @@ -255,13 +392,18 @@ class LegalAgentBenchAgentConfig(BaseResponsesAPIAgentConfig): model_connect_timeout_seconds: int = Field(default=10, ge=1) verifier_timeout_seconds: int = Field(default=3600, ge=1) runtime_build_timeout_seconds: int = Field(default=3600, ge=1) + sandbox_staging_timeout_seconds: int = Field(default=900, ge=1) image_build_timeout_seconds: int = Field(default=3600, ge=1) sandbox_ttl_seconds: int = Field(default=14400, ge=1) sandbox_provider: str | dict[str, Any] = Field(default_factory=lambda: {"docker": {}}) sandbox_image: Optional[str] = None - runtime_docker_platform: Optional[str] = None + runtime_builder_provider_options: dict[str, Any] = Field(default_factory=dict) + agent_sandbox_provider_options: dict[str, Any] = Field(default_factory=dict) + verifier_sandbox_provider_options: dict[str, Any] = Field(default_factory=dict) + opensandbox_request_fraction: Optional[float] = Field(default=0.25, gt=0, le=1) docker_network: Optional[str] = "host" sandbox_model_base_url: Optional[str] = None + sandbox_model_api_key_env: Optional[str] = None image_repository: str = "nemo-gym-legal-agent-bench" results_dir: str = "results/legal_agent_bench" @@ -326,6 +468,18 @@ def _create_archive(destination: Path, entries: list[tuple[Path, str]]) -> None: archive.add(source, arcname=arcname, recursive=True) +def _prepare_sandbox_upload(path: Path) -> None: + """Make a non-secret transfer archive readable by a non-root sandbox user. + + Some providers copy the host file's mode while creating the destination as + root. Temporary files start as mode 0600, which would make the shared + upload API unusable with an image whose default user is non-root. All LAB + transfer archives are created beneath private temporary/cache directories + and contain no model or judge credentials. + """ + path.chmod(0o644) + + def _validate_archive_member(member: tarfile.TarInfo) -> None: if member.name in {".", "./"} and member.isdir(): return @@ -358,32 +512,58 @@ def _copy_downloaded_file(source: Path, destination: Path) -> None: shutil.copyfile(source, destination) -def ensure_runtime_archive(deps_dir: Path) -> Path: - """Create one immutable, process-safe transport archive for a cached runtime.""" - archive_path = deps_dir.parent / f"{deps_dir.name}.tar.gz" - if archive_path.is_file(): - return archive_path - lock_path = deps_dir.parent / f".{deps_dir.name}.archive.lock" - with lock_path.open("a+") as lock_file: - fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX) +def _validate_runtime_archive(archive_path: Path) -> None: + """Validate the trusted-installer output before another sandbox extracts it.""" + root = PurePosixPath("agent_deps") + with tarfile.open(archive_path, "r:gz") as archive: + members = archive.getmembers() + if not members: + raise LegalAgentBenchArtifactError("LAB runtime builder returned an empty archive") + for member in members: + path = PurePosixPath(member.name) + if path.is_absolute() or not path.parts or any(part in {"", ".", ".."} for part in path.parts): + raise LegalAgentBenchArtifactError(f"Unsafe LAB runtime archive path: {member.name!r}") + if path != root and root not in path.parents: + raise LegalAgentBenchArtifactError(f"LAB runtime archive entry is outside {root}: {member.name!r}") + if member.issym() or member.islnk(): + target = PurePosixPath(member.linkname) + if target.is_absolute(): + raise LegalAgentBenchArtifactError( + f"LAB runtime archive contains an absolute link: {member.name!r}" + ) + # Tar symbolic links are relative to the link's parent, while + # hard-link targets are archive-root-relative member names. + resolved = target if member.islnk() else path.parent.joinpath(target) + normalized: list[str] = [] + for part in resolved.parts: + if part == "..": + if not normalized: + raise LegalAgentBenchArtifactError( + f"LAB runtime archive link escapes its root: {member.name!r}" + ) + normalized.pop() + elif part not in {"", "."}: + normalized.append(part) + normalized_path = PurePosixPath(*normalized) + if normalized_path != root and root not in normalized_path.parents: + raise LegalAgentBenchArtifactError(f"LAB runtime archive link escapes its root: {member.name!r}") + elif not (member.isdir() or member.isreg()): + raise LegalAgentBenchArtifactError( + "LAB runtime archives may contain only directories, regular files, and internal links: " + f"{member.name!r}" + ) + + +async def _acquire_file_lock(path: Path) -> Any: + """Acquire an interprocess lock without blocking the async server loop.""" + path.parent.mkdir(parents=True, exist_ok=True) + lock_file = path.open("a+") + while True: try: - if archive_path.is_file(): - return archive_path - with tempfile.NamedTemporaryFile( - suffix=".tar.gz", - prefix=f".{deps_dir.name}-", - dir=deps_dir.parent, - delete=False, - ) as temporary: - temporary_path = Path(temporary.name) - try: - _create_archive(temporary_path, [(deps_dir, "agent_deps_mount")]) - temporary_path.replace(archive_path) - finally: - temporary_path.unlink(missing_ok=True) - finally: - fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN) - return archive_path + fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + return lock_file + except BlockingIOError: + await asyncio.sleep(0.1) def agent_key(agent_server_module: str) -> str: @@ -463,35 +643,17 @@ def agent_runtime_env(agent_server_module: str, agent_kwargs: dict[str, Any]) -> return {environment_variable: f"{package}@{normalized_version}"} -def ensure_agent_runtime( +def _runtime_recipe( agent_server_module: str, *, agent_kwargs: dict[str, Any], image: str, - docker_network: Optional[str], - timeout_seconds: int, - docker_platform: Optional[str] = None, -) -> Path: + provider: Mapping[str, Any], +) -> tuple[str, Path, dict[str, str]]: key = agent_key(agent_server_module) script = resolve_agent_setup_script(agent_server_module) requirements = PARENT_DIR / "responses_api_agents" / key / "requirements.txt" runtime_env = agent_runtime_env(agent_server_module, agent_kwargs) - docker = shutil.which("docker") - if not docker: - raise FileNotFoundError("Docker CLI is required to provision Legal Agent Bench agent dependencies") - if docker_platform: - subprocess.run( - [docker, "pull", "--platform", docker_platform, image], - check=True, - timeout=timeout_seconds, - ) - image_info = subprocess.run( - [docker, "image", "inspect", "--format", "{{.Id}}:{{.Os}}/{{.Architecture}}", image], - check=True, - capture_output=True, - text=True, - timeout=60, - ).stdout.strip() recipe = _recipe_hash( [ PORTABLE_PYTHON_SH, @@ -501,81 +663,32 @@ def ensure_agent_runtime( PARENT_DIR / "README.md", PARENT_DIR / "nemo_gym", ], - values=[image_info, json.dumps(runtime_env, sort_keys=True)], + values=[ + image, + json.dumps(runtime_env, sort_keys=True), + json.dumps(provider, sort_keys=True, default=str), + ], + ) + return recipe, script, runtime_env + + +def _create_runtime_builder_input(destination: Path, agent_server_module: str, script: Path) -> None: + key = agent_key(agent_server_module) + requirements = PARENT_DIR / "responses_api_agents" / key / "requirements.txt" + _create_archive( + destination, + [ + (PARENT_DIR / "pyproject.toml", "nemo_gym_mount/pyproject.toml"), + (PARENT_DIR / "README.md", "nemo_gym_mount/README.md"), + (PARENT_DIR / "nemo_gym", "nemo_gym_mount/nemo_gym"), + (requirements, f"nemo_gym_mount/responses_api_agents/{key}/requirements.txt"), + (script, f"nemo_gym_mount/responses_api_agents/{key}/scripts/{script.name}"), + ( + PORTABLE_PYTHON_SH, + "nemo_gym_mount/responses_api_agents/legal_agent_bench_agent/setup_scripts/_portable_python.sh", + ), + ], ) - cache_root = PACKAGE_DIR / ".deps" / key - deps_dir = cache_root / recipe - sentinel = deps_dir / ".installed" - if sentinel.is_file() and sentinel.read_text().strip() == recipe: - return deps_dir - - cache_root.mkdir(parents=True, exist_ok=True) - lock_root = PACKAGE_DIR / ".deps" / ".locks" - lock_root.mkdir(parents=True, exist_ok=True) - lock_path = lock_root / f"{key}-{recipe}.lock" - with lock_path.open("a+") as lock_file: - fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX) - try: - if sentinel.is_file() and sentinel.read_text().strip() == recipe: - return deps_dir - - with tempfile.TemporaryDirectory(prefix=f".{recipe}-bundle-", dir=cache_root) as bundle_raw: - bundle = Path(bundle_raw) - shutil.copy2(PARENT_DIR / "pyproject.toml", bundle / "pyproject.toml") - shutil.copy2(PARENT_DIR / "README.md", bundle / "README.md") - shutil.copytree(PARENT_DIR / "nemo_gym", bundle / "nemo_gym") - staged_agent = bundle / "responses_api_agents" / key - (staged_agent / "scripts").mkdir(parents=True) - shutil.copy2(requirements, staged_agent / "requirements.txt") - shutil.copy2(script, staged_agent / "scripts" / script.name) - staged_setup = bundle / "responses_api_agents" / "legal_agent_bench_agent" / "setup_scripts" - staged_setup.mkdir(parents=True) - shutil.copy2(PORTABLE_PYTHON_SH, staged_setup / PORTABLE_PYTHON_SH.name) - - build_dir = Path(tempfile.mkdtemp(prefix=f".{recipe}-build-", dir=cache_root)) - try: - env = { - "PORTABLE_PYTHON_SH": ( - "/nemo_gym_mount/responses_api_agents/legal_agent_bench_agent/" - "setup_scripts/_portable_python.sh" - ), - "DEPS_DIR": "/agent_deps", - "NEMO_GYM_ROOT": "/nemo_gym_mount", - "HOME": "/tmp", - **runtime_env, - } - command = [ - docker, - "run", - "--rm", - "--user", - f"{os.getuid()}:{os.getgid()}", - "--volume", - f"{bundle}:/nemo_gym_mount:ro", - "--volume", - f"{build_dir}:/agent_deps", - ] - if docker_platform: - command.extend(["--platform", docker_platform]) - if docker_network: - command.extend(["--network", docker_network]) - for name, value in env.items(): - command.extend(["--env", f"{name}={value}"]) - command.extend( - [image, "bash", f"/nemo_gym_mount/responses_api_agents/{key}/scripts/{script.name}"] - ) - subprocess.run(command, check=True, timeout=timeout_seconds) - (build_dir / ".installed").write_text(recipe) - if deps_dir.exists(): - shutil.rmtree(build_dir) - else: - build_dir.replace(deps_dir) - finally: - if build_dir.exists(): - shutil.rmtree(build_dir) - finally: - fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN) - return deps_dir def _empty_response(model_name: str) -> NeMoGymResponse: @@ -614,7 +727,7 @@ def sandbox_model_url( def host_tunnel_model_url(model_url: str) -> str: - """Translate wildcard listeners into a concrete host endpoint for an ECS reverse tunnel.""" + """Translate wildcard listeners into an address usable on a shared host network or tunnel.""" parsed = urlsplit(model_url) if (parsed.hostname or "").lower() not in {"0.0.0.0", "::"}: return model_url @@ -816,6 +929,22 @@ def _sandbox_resources(task_dir: Path) -> SandboxResources: ) +def _fractional_resource_requests(resources: SandboxResources, fraction: float) -> dict[str, Any]: + """Keep Kubernetes requests below LAB limits while preserving non-shareable resources.""" + requests: dict[str, Any] = {} + if resources.cpu is not None: + requests["cpu"] = resources.cpu * fraction + if resources.memory_mib is not None: + requests["memory_mib"] = max(1, math.ceil(resources.memory_mib * fraction)) + if resources.disk_gib is not None: + requests["disk_gib"] = resources.disk_gib + if resources.gpu: + requests["gpu"] = resources.gpu + if resources.gpu and resources.gpu_type is not None: + requests["gpu_type"] = resources.gpu_type + return requests + + def _environment_hash(environment_dir: Path) -> str: digest = hashlib.sha256() for path in sorted(item for item in environment_dir.rglob("*") if item.is_file()): @@ -848,7 +977,7 @@ class LegalAgentBenchAgent(SimpleResponsesAPIAgent): _image_lock: asyncio.Lock = PrivateAttr() _runtime_lock: asyncio.Lock = PrivateAttr() _session_results_dir: Path = PrivateAttr() - _deps_dir: Optional[Path] = PrivateAttr(default=None) + _runtime_archives: dict[str, Path] = PrivateAttr(default_factory=dict) def model_post_init(self, context: Any) -> None: self._sem = asyncio.Semaphore(self.config.concurrency) @@ -873,37 +1002,133 @@ async def responses( async def _ensure_runtime(self, image: str) -> Path: async with self._runtime_lock: - if self._deps_dir is None: - self._deps_dir = await asyncio.to_thread( - ensure_agent_runtime, - self.config.agent_server_module, - agent_kwargs=self.config.agent_kwargs, - image=image, - docker_network=self.config.docker_network, - docker_platform=self.config.runtime_docker_platform, - timeout_seconds=self.config.runtime_build_timeout_seconds, - ) - return self._deps_dir + provider = self._provider_config() + recipe, script, runtime_env = _runtime_recipe( + self.config.agent_server_module, + agent_kwargs=self.config.agent_kwargs, + image=image, + provider=provider, + ) + if recipe in self._runtime_archives: + return self._runtime_archives[recipe] + + key = agent_key(self.config.agent_server_module) + cache_root = PACKAGE_DIR / ".deps" / key + cache_root.mkdir(parents=True, exist_ok=True) + archive_path = cache_root / f"{recipe}.tar.gz" + if archive_path.is_file(): + _validate_runtime_archive(archive_path) + self._runtime_archives[recipe] = archive_path + return archive_path + + lock_path = PACKAGE_DIR / ".deps" / ".locks" / f"{key}-{recipe}.lock" + lock_file = await _acquire_file_lock(lock_path) + try: + if archive_path.is_file(): + _validate_runtime_archive(archive_path) + self._runtime_archives[recipe] = archive_path + return archive_path + + with tempfile.NamedTemporaryFile(suffix=".tar.gz", delete=False) as temporary: + builder_input = Path(temporary.name) + with tempfile.NamedTemporaryFile( + suffix=".tar.gz", + prefix=f".{recipe}-", + dir=cache_root, + delete=False, + ) as temporary: + downloaded_runtime = Path(temporary.name) + builder_sandbox: Optional[AsyncSandbox] = None + try: + _create_runtime_builder_input( + builder_input, + self.config.agent_server_module, + script, + ) + _prepare_sandbox_upload(builder_input) + builder_resources = SandboxResources(cpu=1, memory_mib=4096, disk_gib=10) + builder_options = self._resource_provider_options(provider, builder_resources) + builder_options.update(self.config.runtime_builder_provider_options) + builder_sandbox = AsyncSandbox( + provider, + SandboxSpec( + image=image, + ttl_s=self.config.sandbox_ttl_seconds, + workdir="/tmp", + env=LAB_SANDBOX_ENV, + resources=builder_resources, + metadata=self._sandbox_metadata(), + provider_options=builder_options, + ), + ) + await builder_sandbox.start() + root_result = await builder_sandbox.exec(f"mkdir -p {SANDBOX_ROOT}", timeout_s=300) + if root_result.return_code != 0 or root_result.error_type is not None: + raise RuntimeError( + root_result.stderr or root_result.stdout or "Failed to create LAB sandbox root" + ) + await builder_sandbox.upload(builder_input, f"{SANDBOX_ROOT}/runtime-builder-input.tar.gz") + build_root = f"{SANDBOX_ROOT}/runtime-builder" + nemo_gym_root = f"{build_root}/nemo_gym_mount" + deps_root = f"{build_root}/agent_deps" + environment = { + "PORTABLE_PYTHON_SH": ( + f"{nemo_gym_root}/responses_api_agents/legal_agent_bench_agent/" + "setup_scripts/_portable_python.sh" + ), + "DEPS_DIR": deps_root, + "NEMO_GYM_ROOT": nemo_gym_root, + "TMPDIR": f"{build_root}/tmp", + **runtime_env, + } + result = await builder_sandbox.exec( + f"rm -rf {build_root} && mkdir -p {build_root}/home {build_root}/tmp {deps_root} && " + f"tar -xzf {SANDBOX_ROOT}/runtime-builder-input.tar.gz -C {build_root} && " + f"export HOME={build_root}/home && " + f"bash {nemo_gym_root}/responses_api_agents/{key}/scripts/{script.name} && " + f"tar -czf {SANDBOX_ROOT}/runtime-builder-output.tar.gz -C {build_root} agent_deps", + cwd="/tmp", + env=environment, + timeout_s=self.config.runtime_build_timeout_seconds, + ) + if result.return_code != 0: + detail = result.stderr or result.stdout or "LAB runtime builder failed" + raise RuntimeError(detail[-4000:]) + await builder_sandbox.download( + f"{SANDBOX_ROOT}/runtime-builder-output.tar.gz", + downloaded_runtime, + ) + _validate_runtime_archive(downloaded_runtime) + _prepare_sandbox_upload(downloaded_runtime) + downloaded_runtime.replace(archive_path) + finally: + if builder_sandbox is not None: + await builder_sandbox.stop() + builder_input.unlink(missing_ok=True) + downloaded_runtime.unlink(missing_ok=True) + finally: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN) + lock_file.close() + + self._runtime_archives[recipe] = archive_path + return archive_path async def _ensure_image(self, task_dir: Path) -> str: if self.config.sandbox_image: - if _provider_name(self._provider_config()) != "docker" and not IMMUTABLE_IMAGE.fullmatch( - self.config.sandbox_image - ): - raise LegalAgentBenchConfigurationError( - "Non-Docker LAB sandbox_image must use a complete immutable @sha256 digest" - ) return self.config.sandbox_image if _provider_name(self._provider_config()) != "docker": raise LegalAgentBenchConfigurationError( - "Non-Docker LAB sandboxes require sandbox_image to reference an immutable registry image" + "Non-Docker LAB sandboxes require a provider-compatible sandbox_image" ) environment_dir = task_dir / "environment" image = f"{self.config.image_repository}:{_environment_hash(environment_dir)[:16]}" async with self._image_lock: docker = shutil.which("docker") if not docker: - raise FileNotFoundError("Docker CLI is required for Legal Agent Bench") + raise FileNotFoundError( + "Docker CLI is required to auto-build the default Legal Agent Bench image; " + "set sandbox_image to use an existing image" + ) inspect, _stdout, _stderr = await _run_process( [docker, "image", "inspect", image], cwd=environment_dir, @@ -924,10 +1149,6 @@ def _model_url(self, body: LegalAgentBenchRunRequest) -> str: if self.config.sandbox_model_base_url: return self.config.sandbox_model_base_url provider_name = _provider_name(self._provider_config()) - if provider_name not in {"docker", "ecs_fargate"}: - raise LegalAgentBenchConfigurationError( - "Non-Docker LAB sandboxes require sandbox_model_base_url reachable from the provider" - ) try: model_config = get_first_server_config_dict( self.server_client.global_config_dict, @@ -942,20 +1163,48 @@ def _model_url(self, body: LegalAgentBenchRunRequest) -> str: prefixed_url = apply_rollout_prefix(base_url, rollout_id) if rollout_id else base_url if provider_name == "docker": return sandbox_model_url(prefixed_url, docker_network=self.config.docker_network) - return host_tunnel_model_url(prefixed_url) + if provider_name == "ecs_fargate": + return host_tunnel_model_url(prefixed_url) + if provider_name in {"apptainer", "enroot"}: + return host_tunnel_model_url(prefixed_url) - def _agent_provider_options(self, model_url: str) -> dict[str, Any]: - """Return provider-specific routing without exposing policy credentials to the sandbox.""" - if _provider_name(self._provider_config()) != "ecs_fargate" or self.config.sandbox_model_base_url: + hostname = (urlsplit(prefixed_url).hostname or "").lower() + if hostname in {"localhost", "127.0.0.1", "0.0.0.0", "::1", "::"}: + raise LegalAgentBenchConfigurationError( + f"LAB sandbox provider {provider_name!r} cannot use the host-local policy URL; " + "set sandbox_model_base_url to a policy proxy reachable from that provider" + ) + return prefixed_url + + def _resource_provider_options( + self, + provider: Mapping[str, Any], + resources: SandboxResources, + ) -> dict[str, Any]: + fraction = self.config.opensandbox_request_fraction + if _provider_name(provider) != "opensandbox" or fraction is None: return {} - return { - "outside_endpoints": [ + return {"resource_requests": _fractional_resource_requests(resources, fraction)} + + def _agent_provider_options( + self, + provider: Mapping[str, Any], + model_url: str, + resources: SandboxResources, + ) -> dict[str, Any]: + """Return provider-specific routing without exposing policy credentials to the sandbox.""" + options = self._resource_provider_options(provider, resources) + options.update(self.config.agent_sandbox_provider_options) + if _provider_name(provider) == "ecs_fargate" and not self.config.sandbox_model_base_url: + outside_endpoints = list(options.get("outside_endpoints") or []) + outside_endpoints.append( { "url": model_url, "env_var": "LAB_POLICY_MODEL_URL", } - ] - } + ) + options["outside_endpoints"] = outside_endpoints + return options def _model_name(self) -> str: global_config = getattr(getattr(self, "server_client", None), "global_config_dict", None) @@ -1030,7 +1279,8 @@ def _write_runner_config( if agent_key(self.config.agent_server_module) == "codex_agent": extra_config = dict(agent_kwargs.get("extra_config") or {}) if "model_catalog_json" not in extra_config: - (paths["runtime"] / "codex_model_catalog.json").write_text(json.dumps(CODEX_MODEL_CATALOG, indent=2)) + catalog = codex_model_catalog(self._model_name()) + (paths["runtime"] / "codex_model_catalog.json").write_text(json.dumps(catalog, indent=2)) extra_config["model_catalog_json"] = CODEX_MODEL_CATALOG_PATH agent_kwargs["extra_config"] = extra_config runner = { @@ -1038,8 +1288,10 @@ def _write_runner_config( "agent_server_class": self.config.agent_server_class, "agent_config_class": self.config.agent_config_class, "agent_kwargs": agent_kwargs, + "model_name": self._model_name(), "model_url": model_url, "model_connect_timeout_seconds": self.config.model_connect_timeout_seconds, + "http_proxy_from_environment": _provider_name(self._provider_config()) == "openshell", "disable_endpoint_metadata_probe": agent_key(self.config.agent_server_module) == "hermes_agent", "responses_create_params": params.model_dump(mode="json", exclude_none=True), } @@ -1047,7 +1299,10 @@ def _write_runner_config( def _provider_config(self) -> dict[str, Any]: global_config = getattr(getattr(self, "server_client", None), "global_config_dict", None) - provider = resolve_provider_config(self.config.sandbox_provider, global_config) + try: + provider = resolve_provider_config(self.config.sandbox_provider, global_config) + except (TypeError, ValueError) as exc: + raise LegalAgentBenchConfigurationError(f"Invalid sandbox provider configuration: {exc}") from exc if _provider_name(provider) != "docker": return provider @@ -1069,27 +1324,36 @@ def _provider_config(self) -> dict[str, Any]: def _sandbox_metadata(self) -> dict[str, Any]: global_config = getattr(getattr(self, "server_client", None), "global_config_dict", None) - return resolve_provider_metadata(self.config.sandbox_provider, global_config) + try: + return resolve_provider_metadata(self.config.sandbox_provider, global_config) + except (TypeError, ValueError) as exc: + raise LegalAgentBenchConfigurationError(f"Invalid sandbox provider metadata: {exc}") from exc def _agent_sandbox( self, *, image: str, task_dir: Path, - skills_dir: Path, - deps_dir: Path, - paths: dict[str, Path], model_url: str, ) -> AsyncSandbox: + provider = self._provider_config() + resources = _sandbox_resources(task_dir) + sandbox_env = dict(LAB_SANDBOX_ENV) + if self.config.sandbox_model_api_key_env: + model_api_key = os.environ.get(self.config.sandbox_model_api_key_env) + if not model_api_key: + raise LegalAgentBenchConfigurationError("Configured sandbox_model_api_key_env is unset or empty") + sandbox_env["LAB_POLICY_API_KEY"] = model_api_key return AsyncSandbox( - self._provider_config(), + provider, SandboxSpec( image=image, ttl_s=self.config.sandbox_ttl_seconds, - workdir="/workspace/output", - resources=_sandbox_resources(task_dir), + workdir="/tmp", + env=sandbox_env, + resources=resources, metadata=self._sandbox_metadata(), - provider_options=self._agent_provider_options(model_url), + provider_options=self._agent_provider_options(provider, model_url, resources), ), ) @@ -1098,16 +1362,21 @@ def _verifier_sandbox( *, image: str, task_dir: Path, - paths: dict[str, Path], ) -> AsyncSandbox: + provider = self._provider_config() + resources = _sandbox_resources(task_dir) + provider_options = self._resource_provider_options(provider, resources) + provider_options.update(self.config.verifier_sandbox_provider_options) return AsyncSandbox( - self._provider_config(), + provider, SandboxSpec( image=image, ttl_s=self.config.sandbox_ttl_seconds, - workdir="/logs/agent/artifacts/lab-run/output", - resources=_sandbox_resources(task_dir), + workdir="/tmp", + env=LAB_SANDBOX_ENV, + resources=resources, metadata=self._sandbox_metadata(), + provider_options=provider_options, ), ) @@ -1117,7 +1386,7 @@ async def _stage_agent_sandbox( *, task_dir: Path, skills_dir: Path, - deps_dir: Path, + runtime_archive: Path, paths: dict[str, Path], ) -> None: with tempfile.NamedTemporaryFile(suffix=".tar.gz", delete=False) as temporary: @@ -1126,26 +1395,31 @@ async def _stage_agent_sandbox( _create_archive( archive_path, [ - (paths["agent_source"], "agent_source_mount"), + (paths["agent_source"], "agent_source"), (task_dir / "documents", "workspace/vdr"), (skills_dir, "workspace/skills"), - (paths["runtime"], "trajectories_mount"), + (paths["runtime"], "runtime"), ], ) - await sandbox.upload(archive_path, "/tmp/legal-agent-bench-agent-input.tar.gz") - await sandbox.upload(ensure_runtime_archive(deps_dir), "/tmp/legal-agent-bench-runtime.tar.gz") + _prepare_sandbox_upload(archive_path) + _prepare_sandbox_upload(runtime_archive) + root_result = await sandbox.exec( + f"mkdir -p {SANDBOX_ROOT}", timeout_s=self.config.sandbox_staging_timeout_seconds + ) + if root_result.return_code != 0 or root_result.error_type is not None: + raise RuntimeError(root_result.stderr or root_result.stdout or "Failed to create LAB sandbox root") + await sandbox.upload(archive_path, f"{SANDBOX_ROOT}/agent-input.tar.gz") + await sandbox.upload(runtime_archive, f"{SANDBOX_ROOT}/runtime.tar.gz") finally: archive_path.unlink(missing_ok=True) command = ( - "mkdir -p /agent_source_mount /agent_deps_mount /trajectories_mount " - "/workspace/vdr /workspace/skills /workspace/output /workspace/workspace && " - "tar -xzf /tmp/legal-agent-bench-agent-input.tar.gz -C / && " - "tar -xzf /tmp/legal-agent-bench-runtime.tar.gz -C / && " - "chmod -R a+rX,a-w /agent_source_mount /agent_deps_mount /workspace/vdr /workspace/skills && " - "chown -R nobody:nogroup /trajectories_mount /workspace/output /workspace/workspace" + f"mkdir -p {SANDBOX_ROOT} {SANDBOX_OUTPUT} {SANDBOX_SCRATCH} && " + f"tar -xzf {SANDBOX_ROOT}/agent-input.tar.gz -C {SANDBOX_ROOT} && " + f"tar -xzf {SANDBOX_ROOT}/runtime.tar.gz -C {SANDBOX_ROOT} && " + f"chmod -R a+rX,a-w {SANDBOX_AGENT_SOURCE} {SANDBOX_AGENT_DEPS} {SANDBOX_VDR} {SANDBOX_SKILLS}" ) - result = await sandbox.exec(command, timeout_s=300, user="root") + result = await sandbox.exec(command, timeout_s=self.config.sandbox_staging_timeout_seconds) if result.return_code != 0: raise RuntimeError(result.stderr or result.stdout or "Failed to stage LAB agent sandbox") @@ -1154,20 +1428,19 @@ async def _collect_agent_sandbox(self, sandbox: AsyncSandbox, download_dir: Path for filename in ("response.json", "runner_status.json"): destination = download_dir / filename try: - await sandbox.download(f"/trajectories_mount/{filename}", destination) + await sandbox.download(f"{SANDBOX_RUNTIME}/{filename}", destination) except Exception: continue downloads[filename] = destination archive_path = download_dir / "output.tar.gz" archive_result = await sandbox.exec( - "tar -czf /tmp/legal-agent-bench-output.tar.gz -C /workspace/output .", - timeout_s=300, - user="root", + f"tar -czf {SANDBOX_ROOT}/output.tar.gz -C {SANDBOX_OUTPUT} .", + timeout_s=self.config.sandbox_staging_timeout_seconds, ) if archive_result.return_code != 0: raise RuntimeError(archive_result.stderr or archive_result.stdout or "Failed to collect LAB output") - await sandbox.download("/tmp/legal-agent-bench-output.tar.gz", archive_path) + await sandbox.download(f"{SANDBOX_ROOT}/output.tar.gz", archive_path) downloads["output.tar.gz"] = archive_path return downloads @@ -1206,33 +1479,65 @@ async def _stage_and_run_verifier( (task_dir / "tests", "tests"), ], ) - await sandbox.upload(archive_path, "/tmp/legal-agent-bench-verifier-input.tar.gz") + _prepare_sandbox_upload(archive_path) + root_result = await sandbox.exec( + f"mkdir -p {SANDBOX_ROOT}", timeout_s=self.config.sandbox_staging_timeout_seconds + ) + if root_result.return_code != 0 or root_result.error_type is not None: + reason = root_result.stderr or root_result.stdout or "Failed to create LAB verifier sandbox root" + return {}, root_result.error_type == "timeout", reason[-2000:] + await sandbox.upload(archive_path, f"{SANDBOX_ROOT}/verifier-input.tar.gz") finally: archive_path.unlink(missing_ok=True) - command = ( - "rm -rf /tests /logs/agent /logs/verifier && mkdir -p /logs/verifier && " - "tar -xzf /tmp/legal-agent-bench-verifier-input.tar.gz -C / && " - "chmod -R a-w /logs/agent && " - "bash /tests/test.sh" + stage_command = ( + f"rm -rf {SANDBOX_VERIFIER} && mkdir -p {SANDBOX_VERIFIER} && " + f"tar -xzf {SANDBOX_ROOT}/verifier-input.tar.gz -C {SANDBOX_VERIFIER} && " + f"mkdir -p {SANDBOX_LOGS}/verifier && " + f"chmod -R a-w {SANDBOX_LOGS}/agent" + ) + stage_result = await sandbox.exec( + stage_command, + cwd="/tmp", + timeout_s=self.config.sandbox_staging_timeout_seconds, ) + if stage_result.return_code != 0 or stage_result.error_type is not None: + timed_out = stage_result.error_type == "timeout" + reason = stage_result.stderr or stage_result.stdout or "Failed to stage LAB verifier sandbox" + return {}, timed_out, reason[-2000:] + + verifier_env = { + **_verifier_env(task_dir), + "LAB_TESTS_DIR": SANDBOX_TESTS, + "LAB_LOGS_DIR": SANDBOX_LOGS, + } result = await sandbox.exec( - command, - cwd="/workspace/output", - env=_verifier_env(task_dir), + f"bash {SANDBOX_TESTS}/test.sh", + cwd=f"{SANDBOX_LOGS}/agent/artifacts/lab-run/output", + env=verifier_env, timeout_s=self.config.verifier_timeout_seconds, - user="root", ) + verifier_filenames = ("reward.json", "scores.json", "transcript.jsonl", "report.html", "error.json") + manifest_command = ( + "for filename in " + + " ".join(verifier_filenames) + + f'; do test -f "{SANDBOX_LOGS}/verifier/$filename" && printf "%s\\n" "$filename"; done; true' + ) + manifest_result = await sandbox.exec(manifest_command, cwd="/tmp", timeout_s=60) + if manifest_result.return_code != 0 or manifest_result.error_type is not None: + reason = manifest_result.stderr or manifest_result.stdout or "Failed to list LAB verifier artifacts" + return {}, manifest_result.error_type == "timeout", reason[-2000:] + available = set(manifest_result.stdout.splitlines()) & set(verifier_filenames) + downloaded: dict[str, bytes] = {} with tempfile.TemporaryDirectory(prefix="legal-agent-bench-verifier-") as temporary_dir: temporary_path = Path(temporary_dir) - for filename in ("reward.json", "scores.json", "transcript.jsonl", "report.html", "error.json"): - destination = temporary_path / filename - try: - await sandbox.download(f"/logs/verifier/{filename}", destination) - except Exception: + for filename in verifier_filenames: + if filename not in available: continue + destination = temporary_path / filename + await sandbox.download(f"{SANDBOX_LOGS}/verifier/{filename}", destination) if not stat.S_ISREG(destination.lstat().st_mode): return {}, result.error_type == "timeout", f"Verifier returned unsafe {filename}" downloaded[filename] = destination.read_bytes() @@ -1291,7 +1596,7 @@ def _artifacts( "model": model_name, "task": task_name, "run_id": paths["root"].name, - "tool_runtime": "gym-agent-in-docker", + "tool_runtime": "gym-agent-in-sandbox", "agent_server_module": self.config.agent_server_module, "agent_server_class": self.config.agent_server_class, "skills": list(REQUIRED_SKILLS), @@ -1488,7 +1793,7 @@ async def run(self, request: Request, body: LegalAgentBenchRunRequest) -> LegalA staging_temp = tempfile.TemporaryDirectory(prefix="legal-agent-bench-stage-") staged_paths = self._paths_for_root(Path(staging_temp.name), create=True) image = await self._ensure_image(task_dir) - deps_dir = await self._ensure_runtime(image) + runtime_archive = await self._ensure_runtime(image) self._stage_agent_source(staged_paths) model_url = self._model_url(body) self._write_runner_config(staged_paths, params, model_url) @@ -1496,9 +1801,6 @@ async def run(self, request: Request, body: LegalAgentBenchRunRequest) -> LegalA sandbox = self._agent_sandbox( image=image, task_dir=task_dir, - skills_dir=skills_dir, - deps_dir=deps_dir, - paths=staged_paths, model_url=model_url, ) await sandbox.start() @@ -1506,16 +1808,15 @@ async def run(self, request: Request, body: LegalAgentBenchRunRequest) -> LegalA sandbox, task_dir=task_dir, skills_dir=skills_dir, - deps_dir=deps_dir, + runtime_archive=runtime_archive, paths=staged_paths, ) with tempfile.TemporaryDirectory(prefix="legal-agent-bench-agent-") as download_raw: started = time.time() agent_result = await sandbox.exec( - "/agent_deps_mount/bin/python /trajectories_mount/agent_runner.py", - cwd="/workspace/output", + f"{SANDBOX_AGENT_DEPS}/bin/python {SANDBOX_RUNTIME}/agent_runner.py", + cwd=SANDBOX_OUTPUT, timeout_s=self.config.agent_timeout_seconds, - user="nobody", ) agent_elapsed = time.time() - started downloads = await self._collect_agent_sandbox(sandbox, Path(download_raw)) @@ -1575,7 +1876,7 @@ async def run(self, request: Request, body: LegalAgentBenchRunRequest) -> LegalA ) if not agent_failed: - verifier_sandbox = self._verifier_sandbox(image=image, task_dir=task_dir, paths=paths) + verifier_sandbox = self._verifier_sandbox(image=image, task_dir=task_dir) await verifier_sandbox.start() verifier_downloads, verifier_timed_out, verifier_failure = await self._stage_and_run_verifier( verifier_sandbox, task_dir, paths diff --git a/responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_claude_code.yaml b/responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_claude_code.yaml index 5c498b5113..25ba73a8ba 100644 --- a/responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_claude_code.yaml +++ b/responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_claude_code.yaml @@ -19,11 +19,17 @@ legal_agent_bench_claude_code_agent: concurrency: 1 agent_timeout_seconds: 10800 verifier_timeout_seconds: 3600 + sandbox_staging_timeout_seconds: 900 + results_dir: ${oc.env:NEMO_GYM_LAB_RESULTS_DIR,results/legal_agent_bench} sandbox_provider: docker: {} - sandbox_image: null - runtime_docker_platform: null - sandbox_model_base_url: null + sandbox_image: ${oc.env:NEMO_GYM_LAB_SANDBOX_IMAGE,null} + runtime_builder_provider_options: ${oc.decode:${oc.env:NEMO_GYM_LAB_RUNTIME_BUILDER_PROVIDER_OPTIONS,{}}} + agent_sandbox_provider_options: ${oc.decode:${oc.env:NEMO_GYM_LAB_AGENT_SANDBOX_PROVIDER_OPTIONS,{}}} + verifier_sandbox_provider_options: ${oc.decode:${oc.env:NEMO_GYM_LAB_VERIFIER_SANDBOX_PROVIDER_OPTIONS,{}}} + opensandbox_request_fraction: 0.25 + sandbox_model_base_url: ${oc.env:NEMO_GYM_SANDBOX_MODEL_BASE_URL,null} + sandbox_model_api_key_env: ${oc.env:NEMO_GYM_LAB_SANDBOX_MODEL_API_KEY_ENV,null} agent_server_module: responses_api_agents.claude_code_agent.app agent_server_class: ClaudeCodeAgent agent_config_class: ClaudeCodeAgentConfig diff --git a/responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_codex.yaml b/responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_codex.yaml index 5e8861a56e..2b5a7bd545 100644 --- a/responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_codex.yaml +++ b/responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_codex.yaml @@ -19,11 +19,17 @@ legal_agent_bench_codex_agent: concurrency: 1 agent_timeout_seconds: 10800 verifier_timeout_seconds: 3600 + sandbox_staging_timeout_seconds: 900 + results_dir: ${oc.env:NEMO_GYM_LAB_RESULTS_DIR,results/legal_agent_bench} sandbox_provider: docker: {} - sandbox_image: null - runtime_docker_platform: null - sandbox_model_base_url: null + sandbox_image: ${oc.env:NEMO_GYM_LAB_SANDBOX_IMAGE,null} + runtime_builder_provider_options: ${oc.decode:${oc.env:NEMO_GYM_LAB_RUNTIME_BUILDER_PROVIDER_OPTIONS,{}}} + agent_sandbox_provider_options: ${oc.decode:${oc.env:NEMO_GYM_LAB_AGENT_SANDBOX_PROVIDER_OPTIONS,{}}} + verifier_sandbox_provider_options: ${oc.decode:${oc.env:NEMO_GYM_LAB_VERIFIER_SANDBOX_PROVIDER_OPTIONS,{}}} + opensandbox_request_fraction: 0.25 + sandbox_model_base_url: ${oc.env:NEMO_GYM_SANDBOX_MODEL_BASE_URL,null} + sandbox_model_api_key_env: ${oc.env:NEMO_GYM_LAB_SANDBOX_MODEL_API_KEY_ENV,null} agent_server_module: responses_api_agents.codex_agent.app agent_server_class: CodexAgent agent_config_class: CodexAgentConfig @@ -32,7 +38,7 @@ legal_agent_bench_codex_agent: sandbox_mode: danger-full-access timeout: 10800 codex_version: 0.144.4 - cwd: /workspace/output + cwd: /sandbox/nemo-gym-legal-agent-bench/workspace/output datasets: - name: example type: example diff --git a/responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_hermes.yaml b/responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_hermes.yaml index abd3a74b7d..851c18a6ed 100644 --- a/responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_hermes.yaml +++ b/responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_hermes.yaml @@ -19,11 +19,17 @@ legal_agent_bench_hermes_agent: concurrency: 1 agent_timeout_seconds: 10800 verifier_timeout_seconds: 3600 + sandbox_staging_timeout_seconds: 900 + results_dir: ${oc.env:NEMO_GYM_LAB_RESULTS_DIR,results/legal_agent_bench} sandbox_provider: docker: {} - sandbox_image: null - runtime_docker_platform: null - sandbox_model_base_url: null + sandbox_image: ${oc.env:NEMO_GYM_LAB_SANDBOX_IMAGE,null} + runtime_builder_provider_options: ${oc.decode:${oc.env:NEMO_GYM_LAB_RUNTIME_BUILDER_PROVIDER_OPTIONS,{}}} + agent_sandbox_provider_options: ${oc.decode:${oc.env:NEMO_GYM_LAB_AGENT_SANDBOX_PROVIDER_OPTIONS,{}}} + verifier_sandbox_provider_options: ${oc.decode:${oc.env:NEMO_GYM_LAB_VERIFIER_SANDBOX_PROVIDER_OPTIONS,{}}} + opensandbox_request_fraction: 0.25 + sandbox_model_base_url: ${oc.env:NEMO_GYM_SANDBOX_MODEL_BASE_URL,null} + sandbox_model_api_key_env: ${oc.env:NEMO_GYM_LAB_SANDBOX_MODEL_API_KEY_ENV,null} agent_server_module: responses_api_agents.hermes_agent.app agent_server_class: HermesAgent agent_config_class: HermesAgentConfig diff --git a/responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_native.yaml b/responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_native.yaml index 94cb62e3a5..f75bba0ed6 100644 --- a/responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_native.yaml +++ b/responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_native.yaml @@ -19,17 +19,24 @@ legal_agent_bench_native_agent: concurrency: 1 agent_timeout_seconds: 10800 verifier_timeout_seconds: 3600 + sandbox_staging_timeout_seconds: 900 + results_dir: ${oc.env:NEMO_GYM_LAB_RESULTS_DIR,results/legal_agent_bench} sandbox_provider: docker: {} - sandbox_image: null - runtime_docker_platform: null - sandbox_model_base_url: null + sandbox_image: ${oc.env:NEMO_GYM_LAB_SANDBOX_IMAGE,null} + runtime_builder_provider_options: ${oc.decode:${oc.env:NEMO_GYM_LAB_RUNTIME_BUILDER_PROVIDER_OPTIONS,{}}} + agent_sandbox_provider_options: ${oc.decode:${oc.env:NEMO_GYM_LAB_AGENT_SANDBOX_PROVIDER_OPTIONS,{}}} + verifier_sandbox_provider_options: ${oc.decode:${oc.env:NEMO_GYM_LAB_VERIFIER_SANDBOX_PROVIDER_OPTIONS,{}}} + opensandbox_request_fraction: 0.25 + sandbox_model_base_url: ${oc.env:NEMO_GYM_SANDBOX_MODEL_BASE_URL,null} + sandbox_model_api_key_env: ${oc.env:NEMO_GYM_LAB_SANDBOX_MODEL_API_KEY_ENV,null} agent_server_module: responses_api_agents.legal_agent_bench_native_agent.app agent_server_class: LegalAgentBenchNativeAgent agent_config_class: LegalAgentBenchNativeAgentConfig agent_kwargs: max_turns: 60 shell_timeout: 60 + preflight_timeout_seconds: 120 model_timeout_seconds: 1800 max_output_chars: 16384 datasets: diff --git a/responses_api_agents/legal_agent_bench_agent/scripts/smoke_provider.py b/responses_api_agents/legal_agent_bench_agent/scripts/smoke_provider.py new file mode 100644 index 0000000000..d915eaa6a1 --- /dev/null +++ b/responses_api_agents/legal_agent_bench_agent/scripts/smoke_provider.py @@ -0,0 +1,113 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Exercise the shared sandbox contract used by Legal Agent Bench.""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import shlex +import tempfile +from pathlib import Path +from uuid import uuid4 + +from omegaconf import OmegaConf + +from nemo_gym.sandbox import AsyncSandbox, SandboxSpec +from nemo_gym.sandbox.config import resolve_provider_config + + +def _provider(args: argparse.Namespace) -> dict: + if args.config is None: + return {args.provider: {}} + config = OmegaConf.load(args.config) + resolved = OmegaConf.to_container(config, resolve=True) + if not isinstance(resolved, dict): + raise ValueError(f"Sandbox config must resolve to a mapping: {args.config}") + return resolve_provider_config(args.sandbox_name, resolved) + + +async def _smoke(args: argparse.Namespace) -> dict[str, object]: + provider = _provider(args) + provider_name = next(iter(provider)) + token = uuid4().hex + remote_root = f"{args.workdir.rstrip('/')}/nemo-gym-lab-provider-smoke-{token[:8]}" + input_path = f"{remote_root}/input.txt" + output_path = f"{remote_root}/output.txt" + sandbox = AsyncSandbox( + provider, + SandboxSpec( + image=args.image, + ttl_s=args.ttl, + ready_timeout_s=args.ready_timeout, + workdir=args.workdir, + metadata={"benchmark": "legal-agent-bench", "purpose": "provider-smoke"}, + ), + ) + with tempfile.TemporaryDirectory(prefix="legal-agent-bench-provider-smoke-") as temporary: + temporary_root = Path(temporary) + source = temporary_root / "input.txt" + downloaded = temporary_root / "output.txt" + source.write_text(token) + try: + await sandbox.start() + await sandbox.upload(source, input_path) + command = ( + f'test "$(cat {shlex.quote(input_path)})" = {shlex.quote(token)} && ' + f"printf %s {shlex.quote(token)} > {shlex.quote(output_path)}" + ) + result = await sandbox.exec(command, timeout_s=args.timeout) + if result.return_code != 0 or result.error_type is not None: + raise RuntimeError( + f"sandbox exec failed: return_code={result.return_code}, " + f"error_type={result.error_type!r}, stderr={result.stderr!r}" + ) + await sandbox.download(output_path, downloaded) + if downloaded.read_text() != token: + raise RuntimeError("sandbox upload/exec/download round trip changed the payload") + finally: + await sandbox.stop() + return { + "provider": provider_name, + "image": args.image, + "start": "passed", + "exec": "passed", + "upload_download": "passed", + "cleanup": "passed", + } + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + selection = parser.add_mutually_exclusive_group(required=True) + selection.add_argument("--provider", help="Provider name with its default configuration") + selection.add_argument("--config", type=Path, help="Gym provider YAML containing a named sandbox block") + parser.add_argument("--sandbox-name", default="sandbox", help="Named sandbox block in --config") + parser.add_argument("--image", required=True, help="Provider-compatible LAB image reference") + parser.add_argument("--workdir", default="/tmp", help="Writable directory in the sandbox image") + parser.add_argument("--timeout", type=float, default=300, help="Command timeout in seconds") + parser.add_argument("--ready-timeout", type=float, default=900, help="Sandbox readiness timeout in seconds") + parser.add_argument("--ttl", type=float, default=1200, help="Sandbox lifetime in seconds") + return parser + + +def main() -> None: + result = asyncio.run(_smoke(_parser().parse_args())) + print(json.dumps(result, indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/responses_api_agents/legal_agent_bench_agent/tests/test_app.py b/responses_api_agents/legal_agent_bench_agent/tests/test_app.py index f22d33f4c7..53462b1fd1 100644 --- a/responses_api_agents/legal_agent_bench_agent/tests/test_app.py +++ b/responses_api_agents/legal_agent_bench_agent/tests/test_app.py @@ -4,8 +4,8 @@ from __future__ import annotations +import io import json -import multiprocessing import os import sys import tarfile @@ -18,11 +18,48 @@ from nemo_gym.config_types import ModelServerRef, ResourcesServerRef from nemo_gym.openai_utils import NeMoGymResponseCreateParamsNonStreaming from responses_api_agents.legal_agent_bench_agent import app +from responses_api_agents.legal_agent_bench_agent.scripts import smoke_provider -def _runtime_archive_worker(deps_dir: str) -> tuple[str, int]: - archive = app.ensure_runtime_archive(Path(deps_dir)) - return str(archive), archive.stat().st_size +class _RuntimeBuilderSandbox: + instances = [] + + def __init__(self, provider, spec): + self.provider = provider + self.spec = spec + self.upload_members = set() + self.exec_calls = [] + self.stopped = False + self.__class__.instances.append(self) + + async def start(self): + return self + + async def upload(self, source, destination): + assert destination == f"{app.SANDBOX_ROOT}/runtime-builder-input.tar.gz" + assert Path(source).stat().st_mode & 0o777 == 0o644 + with tarfile.open(source, "r:gz") as archive: + self.upload_members = {member.name for member in archive.getmembers()} + + async def exec(self, command, **kwargs): + assert "user" not in kwargs + self.exec_calls.append((command, kwargs)) + return SimpleNamespace(return_code=0, error_type=None, stdout="", stderr="") + + async def download(self, source, destination): + assert source == f"{app.SANDBOX_ROOT}/runtime-builder-output.tar.gz" + payload = b"#!/bin/sh\n" + with tarfile.open(destination, "w:gz") as archive: + root = tarfile.TarInfo("agent_deps") + root.type = tarfile.DIRTYPE + archive.addfile(root) + executable = tarfile.TarInfo("agent_deps/bin/python") + executable.mode = 0o755 + executable.size = len(payload) + archive.addfile(executable, io.BytesIO(payload)) + + async def stop(self): + self.stopped = True def _config(**overrides) -> app.LegalAgentBenchAgentConfig: @@ -80,6 +117,27 @@ def _skills(tmp_path: Path) -> Path: return root +def _runtime_sources(monkeypatch, tmp_path: Path, key: str, script_text: str = "installer-v1"): + package_dir = tmp_path / "legal_agent_bench_agent" + package_dir.mkdir() + portable = package_dir / "_portable_python.sh" + portable.write_text("portable-v1") + script = tmp_path / f"{key}_deps.sh" + script.write_text(script_text) + agent_dir = tmp_path / "responses_api_agents" / key + agent_dir.mkdir(parents=True) + (agent_dir / "requirements.txt").write_text("nemo-gym\n") + (tmp_path / "pyproject.toml").write_text("[project]\nname = 'nemo-gym'\n") + (tmp_path / "README.md").write_text("# Gym\n") + (tmp_path / "nemo_gym").mkdir() + (tmp_path / "nemo_gym" / "runtime.py").write_text("VERSION = 1\n") + monkeypatch.setattr(app, "PACKAGE_DIR", package_dir) + monkeypatch.setattr(app, "PARENT_DIR", tmp_path) + monkeypatch.setattr(app, "PORTABLE_PYTHON_SH", portable) + monkeypatch.setattr(app, "resolve_agent_setup_script", lambda _module: script) + return package_dir, script + + def _successful_response(text: str = "Done") -> app.NeMoGymResponse: return app.NeMoGymResponse.model_validate( { @@ -123,6 +181,11 @@ def test_agent_key(module, expected) -> None: assert app.agent_key(module) == expected +def test_repository_root_is_derived_from_agent_source() -> None: + assert app.PARENT_DIR == Path(app.__file__).resolve().parents[2] + assert (app.PARENT_DIR / "resources_servers" / "legal_agent_bench" / "vendor" / "harvey_labs").is_dir() + + @pytest.mark.parametrize("module", ["hermes_agent", "responses_api_agents.bad-name.app", "x.y.z"]) def test_agent_key_rejects_invalid_modules(module) -> None: with pytest.raises(ValueError): @@ -139,88 +202,46 @@ def test_all_supported_agents_have_dependency_scripts() -> None: assert app.resolve_agent_setup_script(module).is_file() -def test_dependency_runtime_cache_is_harness_and_recipe_specific(monkeypatch, tmp_path) -> None: - package_dir = tmp_path / "legal_agent_bench_agent" - package_dir.mkdir() - portable = package_dir / "_portable_python.sh" - portable.write_text("portable-v1") - script = tmp_path / "hermes_agent_deps.sh" - script.write_text("hermes-v1") - agent_dir = tmp_path / "responses_api_agents" / "hermes_agent" - agent_dir.mkdir(parents=True) - (agent_dir / "requirements.txt").write_text("nemo-gym\n") - (tmp_path / "pyproject.toml").write_text("[project]\nname = 'nemo-gym'\n") - (tmp_path / "README.md").write_text("# Gym\n") - (tmp_path / "env.yaml").write_text("judge_key: must-not-be-mounted\n") - (tmp_path / "nemo_gym").mkdir() - (tmp_path / "nemo_gym" / "runtime.py").write_text("VERSION = 1\n") - calls = [] - - monkeypatch.setattr(app, "PACKAGE_DIR", package_dir) - monkeypatch.setattr(app, "PARENT_DIR", tmp_path) - monkeypatch.setattr(app, "PORTABLE_PYTHON_SH", portable) - monkeypatch.setattr(app, "resolve_agent_setup_script", lambda module: script) - - def fake_run(args, **kwargs): - if len(args) > 1 and args[1] == "run": - mounted_bundle = next(Path(arg.split(":", 1)[0]) for arg in args if ":/nemo_gym_mount:ro" in arg) - assert mounted_bundle != tmp_path - assert not (mounted_bundle / "env.yaml").exists() - calls.append((args, kwargs)) - return SimpleNamespace(stdout="sha256:image:linux/arm64\n") - - monkeypatch.setattr(app.shutil, "which", lambda name: "/usr/bin/docker") - monkeypatch.setattr(app.subprocess, "run", fake_run) +@pytest.mark.asyncio +async def test_dependency_runtime_cache_is_harness_and_recipe_specific(monkeypatch, tmp_path) -> None: + package_dir, script = _runtime_sources(monkeypatch, tmp_path, "hermes_agent", "hermes-v1") + (tmp_path / "env.yaml").write_text("judge_key: must-not-be-uploaded\n") # pragma: allowlist secret + _RuntimeBuilderSandbox.instances = [] + monkeypatch.setattr(app, "AsyncSandbox", _RuntimeBuilderSandbox) + monkeypatch.setattr(app.shutil, "which", lambda _name: pytest.fail("runtime provisioning used Docker")) + + provider = {"apptainer": {"exec": {"concurrency": 2}}} + runner = app.LegalAgentBenchAgent.model_construct(config=_config(sandbox_provider=provider)) + runner._runtime_lock = app.asyncio.Lock() + runner._runtime_archives = {} + monkeypatch.setattr(runner, "_provider_config", lambda: provider) + monkeypatch.setattr(runner, "_sandbox_metadata", lambda: {}) - first = app.ensure_agent_runtime( - "responses_api_agents.hermes_agent.app", - agent_kwargs={}, - image="lab:image", - docker_network="host", - timeout_seconds=60, - ) - second = app.ensure_agent_runtime( - "responses_api_agents.hermes_agent.app", - agent_kwargs={}, - image="lab:image", - docker_network="host", - timeout_seconds=60, - ) + first = await runner._ensure_runtime("lab:image") + second = await runner._ensure_runtime("lab:image") script.write_text("hermes-v2") - third = app.ensure_agent_runtime( - "responses_api_agents.hermes_agent.app", - agent_kwargs={}, - image="lab:image", - docker_network="host", - timeout_seconds=60, - ) + third = await runner._ensure_runtime("lab:image") (tmp_path / "pyproject.toml").write_text("[project]\nname = 'nemo-gym-updated'\n") - fourth = app.ensure_agent_runtime( - "responses_api_agents.hermes_agent.app", - agent_kwargs={}, - image="lab:image", - docker_network="host", - timeout_seconds=60, - ) + fourth = await runner._ensure_runtime("lab:image") (tmp_path / "nemo_gym" / "runtime.py").write_text("VERSION = 2\n") - fifth = app.ensure_agent_runtime( - "responses_api_agents.hermes_agent.app", - agent_kwargs={}, - image="lab:image", - docker_network="host", - timeout_seconds=60, - ) + fifth = await runner._ensure_runtime("lab:image") assert first == second assert len({first, third, fourth, fifth}) == 4 assert all(path.parent == package_dir / ".deps" / "hermes_agent" for path in (first, third, fourth, fifth)) - assert all(path.is_dir() for path in (first, third, fourth, fifth)) - docker_runs = [call for call in calls if call[0][1] == "run"] - assert len(docker_runs) == 4 - mounted_sources = [arg.split(":", 1)[0] for arg in docker_runs[0][0] if ":/nemo_gym_mount:ro" in arg] - assert mounted_sources and mounted_sources[0] != str(tmp_path) - assert "env.yaml" not in " ".join(docker_runs[0][0]) - assert "--network" in docker_runs[0][0] + assert all(path.is_file() for path in (first, third, fourth, fifth)) + assert len(_RuntimeBuilderSandbox.instances) == 4 + first_builder = _RuntimeBuilderSandbox.instances[0] + assert first_builder.provider == provider + assert first_builder.spec.workdir == "/tmp" + assert first_builder.spec.env == app.LAB_SANDBOX_ENV + assert first_builder.exec_calls[0][0] == f"mkdir -p {app.SANDBOX_ROOT}" + build_command, build_kwargs = first_builder.exec_calls[1] + assert f"export HOME={app.SANDBOX_ROOT}/runtime-builder/home" in build_command + assert "HOME" not in build_kwargs["env"] + assert build_kwargs["env"]["TMPDIR"] == f"{app.SANDBOX_ROOT}/runtime-builder/tmp" + assert not any("env.yaml" in name for name in first_builder.upload_members) + assert first_builder.stopped is True @pytest.mark.parametrize( @@ -264,65 +285,69 @@ def test_agent_runtime_env_rejects_non_exact_harness_pin(version) -> None: ) -def test_dependency_provisioning_uses_pin_and_invalidates_when_it_changes(monkeypatch, tmp_path) -> None: - package_dir = tmp_path / "legal_agent_bench_agent" - package_dir.mkdir() - portable = package_dir / "_portable_python.sh" - portable.write_text("portable") - script = tmp_path / "claude_code_agent_deps.sh" - script.write_text("claude") - agent_dir = tmp_path / "responses_api_agents" / "claude_code_agent" - agent_dir.mkdir(parents=True) - (agent_dir / "requirements.txt").write_text("nemo-gym\n") - (tmp_path / "pyproject.toml").write_text("[project]\nname = 'nemo-gym'\n") - (tmp_path / "README.md").write_text("# Gym\n") - (tmp_path / "nemo_gym").mkdir() - (tmp_path / "nemo_gym" / "runtime.py").write_text("VERSION = 1\n") - calls = [] - - monkeypatch.setattr(app, "PACKAGE_DIR", package_dir) - monkeypatch.setattr(app, "PARENT_DIR", tmp_path) - monkeypatch.setattr(app, "PORTABLE_PYTHON_SH", portable) - monkeypatch.setattr(app, "resolve_agent_setup_script", lambda module: script) - - def fake_run(args, **kwargs): - calls.append((args, kwargs)) - return SimpleNamespace(stdout="sha256:image:linux/arm64\n") - - monkeypatch.setattr(app.shutil, "which", lambda name: "/usr/bin/docker") - monkeypatch.setattr(app.subprocess, "run", fake_run) - - for version in ("2.1.211", "2.1.212"): - app.ensure_agent_runtime( - "responses_api_agents.claude_code_agent.app", - agent_kwargs={"claude_code_version": version}, - image="lab:image", - docker_network="host", - timeout_seconds=60, +@pytest.mark.asyncio +async def test_dependency_provisioning_uses_pin_and_invalidates_when_it_changes(monkeypatch, tmp_path) -> None: + _runtime_sources(monkeypatch, tmp_path, "claude_code_agent", "claude") + _RuntimeBuilderSandbox.instances = [] + monkeypatch.setattr(app, "AsyncSandbox", _RuntimeBuilderSandbox) + provider = {"opensandbox": {"connection": {"domain": "sandbox.example"}}} + runner = app.LegalAgentBenchAgent.model_construct( + config=_config( + agent_server_module="responses_api_agents.claude_code_agent.app", + agent_kwargs={"claude_code_version": "2.1.211"}, + sandbox_provider=provider, ) + ) + runner._runtime_lock = app.asyncio.Lock() + runner._runtime_archives = {} + monkeypatch.setattr(runner, "_provider_config", lambda: provider) + monkeypatch.setattr(runner, "_sandbox_metadata", lambda: {}) - docker_runs = [call[0] for call in calls if call[0][1] == "run"] - assert len(docker_runs) == 2 - assert "CLAUDE_SPEC=@anthropic-ai/claude-code@2.1.211" in docker_runs[0] - assert "CLAUDE_SPEC=@anthropic-ai/claude-code@2.1.212" in docker_runs[1] + first = await runner._ensure_runtime("lab:image") + first_env = _RuntimeBuilderSandbox.instances[-1].exec_calls[1][1]["env"] + runner.config.agent_kwargs["claude_code_version"] = "2.1.212" + second = await runner._ensure_runtime("lab:image") + second_env = _RuntimeBuilderSandbox.instances[-1].exec_calls[1][1]["env"] + assert first != second + assert first_env["CLAUDE_SPEC"] == "@anthropic-ai/claude-code@2.1.211" + assert second_env["CLAUDE_SPEC"] == "@anthropic-ai/claude-code@2.1.212" -def test_runtime_archive_is_published_once_across_processes(tmp_path) -> None: - deps = tmp_path / "hermes_agent" / "recipe" - deps.mkdir(parents=True) - (deps / ".installed").write_text("recipe") - (deps / "runtime.py").write_text("VALUE = 1\n") - context = multiprocessing.get_context("fork") - with context.Pool(2) as pool: - results = pool.map(_runtime_archive_worker, [str(deps), str(deps)]) +def test_runtime_archive_accepts_internal_links_and_rejects_escaping_links(tmp_path) -> None: + archive_path = tmp_path / "runtime.tar.gz" + with tarfile.open(archive_path, "w:gz") as archive: + root = tarfile.TarInfo("agent_deps") + root.type = tarfile.DIRTYPE + archive.addfile(root) + target = tarfile.TarInfo("agent_deps/bin/python3") + target.size = 0 + archive.addfile(target, io.BytesIO()) + link = tarfile.TarInfo("agent_deps/bin/python") + link.type = tarfile.SYMTYPE + link.linkname = "python3" + archive.addfile(link) + hardlink = tarfile.TarInfo("agent_deps/bin/python-copy") + hardlink.type = tarfile.LNKTYPE + hardlink.linkname = "agent_deps/bin/python3" + archive.addfile(hardlink) + app._validate_runtime_archive(archive_path) - assert results[0] == results[1] - archive = Path(results[0][0]) - assert archive.is_file() - with tarfile.open(archive, "r:gz") as contents: - assert "agent_deps_mount/runtime.py" in contents.getnames() - assert app.ensure_runtime_archive(deps) == archive + with tarfile.open(archive_path, "w:gz") as archive: + link = tarfile.TarInfo("agent_deps/bin/python") + link.type = tarfile.SYMTYPE + link.linkname = "../../../outside" + archive.addfile(link) + with pytest.raises(app.LegalAgentBenchArtifactError, match="escapes"): + app._validate_runtime_archive(archive_path) + + with tarfile.open(archive_path, "w:gz") as archive: + link = tarfile.TarInfo("agent_deps/bin/python") + link.type = tarfile.LNKTYPE + link.linkname = "outside" + archive.addfile(link) + with pytest.raises(app.LegalAgentBenchArtifactError, match="escapes"): + app._validate_runtime_archive(archive_path) def test_archive_helpers_reject_missing_unsafe_and_nonregular_inputs(tmp_path) -> None: @@ -350,6 +375,81 @@ def test_provider_and_installer_validation_rejects_ambiguous_or_missing_configur with pytest.raises(app.LegalAgentBenchConfigurationError, match="requires dependency setup script"): app.resolve_agent_setup_script("responses_api_agents.hermes_agent.app") + runner = app.LegalAgentBenchAgent.model_construct( + config=_config(sandbox_provider="sandbox"), + server_client=SimpleNamespace( + global_config_dict={"sandbox": {"docker": {}, "default_metadata": "not-a-mapping"}} + ), + ) + with pytest.raises(app.LegalAgentBenchConfigurationError, match="provider metadata"): + runner._sandbox_metadata() + + +@pytest.mark.parametrize( + "provider_name", + ["docker", "ecs_fargate", "enroot", "apptainer", "opensandbox", "daytona", "openshell"], +) +def test_every_builtin_provider_has_a_named_config(provider_name) -> None: + config_path = ( + app.PARENT_DIR / "nemo_gym" / "sandbox" / "providers" / provider_name / "configs" / f"{provider_name}.yaml" + ) + config = OmegaConf.load(config_path) + + assert provider_name in config.sandbox + assert "default_metadata" in config.sandbox + + +@pytest.mark.asyncio +async def test_provider_smoke_helper_uses_only_the_public_lifecycle_contract(monkeypatch) -> None: + instances = [] + + class LifecycleSandbox: + def __init__(self, provider, spec): + self.provider = provider + self.spec = spec + self.payload = None + self.stopped = False + instances.append(self) + + async def start(self): + return self + + async def upload(self, source, destination): + self.payload = Path(source).read_text() + assert destination.endswith("/input.txt") + + async def exec(self, command, **kwargs): + assert self.payload in command + assert "user" not in kwargs + return SimpleNamespace(return_code=0, error_type=None, stdout="", stderr="") + + async def download(self, source, destination): + assert source.endswith("/output.txt") + Path(destination).write_text(self.payload) + + async def stop(self): + self.stopped = True + + monkeypatch.setattr(smoke_provider, "AsyncSandbox", LifecycleSandbox) + result = await smoke_provider._smoke( + SimpleNamespace( + config=None, + provider="apptainer", + sandbox_name="sandbox", + image="provider-native-image", + workdir="/tmp", + timeout=30, + ready_timeout=60, + ttl=120, + ) + ) + + assert result["provider"] == "apptainer" + assert result["cleanup"] == "passed" + assert instances[0].provider == {"apptainer": {}} + assert instances[0].spec.image == "provider-native-image" + assert instances[0].stopped is True + def test_recipe_hash_ignores_transient_bytecode(tmp_path) -> None: package = tmp_path / "package" @@ -364,18 +464,55 @@ def test_recipe_hash_ignores_transient_bytecode(tmp_path) -> None: assert app._recipe_hash([package]) == first -def test_runtime_provisioning_requires_docker(monkeypatch) -> None: - monkeypatch.setattr(app, "resolve_agent_setup_script", lambda _module: Path("installer.sh")) - monkeypatch.setattr(app.shutil, "which", lambda _name: None) +@pytest.mark.asyncio +@pytest.mark.parametrize( + "provider_name", + ["docker", "ecs_fargate", "enroot", "apptainer", "opensandbox", "daytona", "openshell"], +) +async def test_runtime_provisioning_uses_selected_sandbox_provider(monkeypatch, tmp_path, provider_name) -> None: + _runtime_sources(monkeypatch, tmp_path, "hermes_agent") + _RuntimeBuilderSandbox.instances = [] + monkeypatch.setattr(app, "AsyncSandbox", _RuntimeBuilderSandbox) + monkeypatch.setattr(app.shutil, "which", lambda _name: pytest.fail("runtime provisioning used Docker CLI")) + provider = {provider_name: {}} + runner = app.LegalAgentBenchAgent.model_construct(config=_config(sandbox_provider=provider)) + runner._runtime_lock = app.asyncio.Lock() + runner._runtime_archives = {} + monkeypatch.setattr(runner, "_provider_config", lambda: provider) + monkeypatch.setattr(runner, "_sandbox_metadata", lambda: {}) - with pytest.raises(FileNotFoundError, match="Docker CLI"): - app.ensure_agent_runtime( - "responses_api_agents.hermes_agent.app", - agent_kwargs={}, - image="lab:image", - docker_network="host", - timeout_seconds=60, + archive = await runner._ensure_runtime("provider-native-image") + + assert archive.is_file() + assert _RuntimeBuilderSandbox.instances[0].provider == provider + expected_options = ( + {"resource_requests": {"cpu": 0.25, "memory_mib": 1024, "disk_gib": 10}} + if provider_name == "opensandbox" + else {} + ) + assert _RuntimeBuilderSandbox.instances[0].spec.provider_options == expected_options + + +@pytest.mark.asyncio +async def test_runtime_builder_merges_phase_specific_provider_options(monkeypatch, tmp_path) -> None: + _runtime_sources(monkeypatch, tmp_path, "hermes_agent") + _RuntimeBuilderSandbox.instances = [] + monkeypatch.setattr(app, "AsyncSandbox", _RuntimeBuilderSandbox) + provider = {"openshell": {}} + runner = app.LegalAgentBenchAgent.model_construct( + config=_config( + sandbox_provider=provider, + runtime_builder_provider_options={"policy": {"version": 1}}, ) + ) + runner._runtime_lock = app.asyncio.Lock() + runner._runtime_archives = {} + monkeypatch.setattr(runner, "_provider_config", lambda: provider) + monkeypatch.setattr(runner, "_sandbox_metadata", lambda: {}) + + await runner._ensure_runtime("provider-native-image") + + assert _RuntimeBuilderSandbox.instances[0].spec.provider_options == {"policy": {"version": 1}} def test_config_defaults_are_docker_and_single_concurrency() -> None: @@ -383,12 +520,14 @@ def test_config_defaults_are_docker_and_single_concurrency() -> None: assert config.concurrency == 1 assert config.sandbox_provider == {"docker": {}} assert config.sandbox_image is None + assert config.opensandbox_request_fraction == 0.25 assert config.docker_network == "host" assert config.results_dir == "results/legal_agent_bench" assert config.agent_timeout_seconds == 10800 assert config.model_connect_timeout_seconds == 10 assert config.verifier_timeout_seconds == 3600 assert config.runtime_build_timeout_seconds == 3600 + assert config.sandbox_staging_timeout_seconds == 900 @pytest.mark.parametrize( @@ -484,7 +623,7 @@ def test_compose_agent_input_uses_task_and_skills_without_rubric(monkeypatch, tm system = serialized["input"][0]["content"] assert "Write a memo." in system assert "Skill: docx" in system - assert "/workspace/vdr" in system + assert "$VDR_DIR" in system assert "criteria" not in system assert serialized["input"][1]["content"] == app.INITIAL_USER_PROMPT @@ -522,6 +661,20 @@ def test_verifier_credentials_are_read_separately_from_sandbox_resources(tmp_pat assert resources.cpu == 2 assert resources.memory_mib == 4096 assert resources.disk_gib == 10 + assert app._fractional_resource_requests(resources, 0.25) == { + "cpu": 0.5, + "memory_mib": 1024, + "disk_gib": 10, + } + + gpu_resources = app.SandboxResources(cpu=4, memory_mib=8192, disk_gib=30, gpu=2, gpu_type="H100") + assert app._fractional_resource_requests(gpu_resources, 0.25) == { + "cpu": 1.0, + "memory_mib": 2048, + "disk_gib": 30, + "gpu": 2, + "gpu_type": "H100", + } def test_environment_hash_is_deterministic_and_content_sensitive(tmp_path) -> None: @@ -553,18 +706,53 @@ def test_runner_config_preserves_dynamic_agent_configuration(tmp_path) -> None: assert payload["agent_server_class"] == "HermesAgent" assert payload["agent_config_class"] == "HermesAgentConfig" assert payload["agent_kwargs"]["max_turns"] == 17 + assert payload["model_name"] == "policy_model" assert payload["model_url"] == "http://model.internal:8000" assert payload["model_connect_timeout_seconds"] == 10 + assert payload["http_proxy_from_environment"] is False assert payload["disable_endpoint_metadata_probe"] is True + assert "LAB_POLICY_API_KEY" not in json.dumps(payload) assert "inspect.signature(agent.responses)" in (tmp_path / "agent_runner.py").read_text() assert "SimpleNamespace(path_params={})" in (tmp_path / "agent_runner.py").read_text() - assert "socket.create_connection" in (tmp_path / "agent_runner.py").read_text() + runner_source = (tmp_path / "agent_runner.py").read_text() + assert "urlopen(" in runner_source + assert "socket.create_connection" not in runner_source + assert "if exc.code in {401, 403}" in runner_source assert 'os.environ.get("LAB_POLICY_MODEL_URL", runner["model_url"])' in (tmp_path / "agent_runner.py").read_text() + assert 'model_url_root = model_url.removesuffix("/v1").rstrip("/")' in runner_source + assert "client._build_server_base_url = lambda _cfg: model_url_root" in runner_source + assert 'object.__setattr__(agent, "_resolve_base_url", lambda *args, **kwargs: model_url_root)' in runner_source assert "runner_status.json" in (tmp_path / "agent_runner.py").read_text() assert '"responses_api_models"' in (tmp_path / "agent_runner.py").read_text() + assert '"global_aiohttp_trust_env": runner.get("http_proxy_from_environment", False)' in runner_source + assert "if not is_global_aiohttp_client_setup():" in runner_source + assert "response = asyncio.run(invoke_agent())" in runner_source assert 'os.environ.setdefault("NEMO_GYM_CONFIG_DICT", "{}")' in (tmp_path / "agent_runner.py").read_text() assert "hermes_model_metadata.fetch_endpoint_model_metadata" in (tmp_path / "agent_runner.py").read_text() + assert "hermes_model_metadata._query_local_context_length" in (tmp_path / "agent_runner.py").read_text() assert "hermes_usage_pricing.fetch_endpoint_model_metadata" in (tmp_path / "agent_runner.py").read_text() + assert 'os.environ.get("LAB_POLICY_API_KEY")' in runner_source + assert 'headers.setdefault("Authorization", f"Bearer {model_api_key}")' in runner_source + assert 'object.__setattr__(config, "anthropic_api_key", model_api_key)' in runner_source + assert 'object.__setattr__(config, "openai_api_key", model_api_key)' in runner_source + assert 'kwargs["model"] = runner["model_name"]' in runner_source + assert 'body = body.model_copy(update={"model": runner["model_name"]})' in runner_source + assert "class AuthenticatedAIAgent(original_ai_agent):" in runner_source + assert "hermes_run_agent.AIAgent = AuthenticatedAIAgent" in runner_source + + +def test_runner_enables_environment_proxy_only_for_openshell(monkeypatch, tmp_path) -> None: + runner = app.LegalAgentBenchAgent.model_construct(config=_config()) + monkeypatch.setattr(runner, "_provider_config", lambda: {"openshell": {}}) + + runner._write_runner_config( + {"runtime": tmp_path}, + NeMoGymResponseCreateParamsNonStreaming(input="Do the work"), + "http://model.internal:8000", + ) + + payload = json.loads((tmp_path / "runner.json").read_text()) + assert payload["http_proxy_from_environment"] is True def test_runner_config_keeps_endpoint_metadata_for_non_hermes_harnesses(tmp_path) -> None: @@ -602,10 +790,33 @@ def test_runner_config_stages_default_codex_model_catalog(tmp_path) -> None: payload = json.loads((tmp_path / "runner.json").read_text()) assert payload["agent_kwargs"]["extra_config"]["model_catalog_json"] == app.CODEX_MODEL_CATALOG_PATH catalog = json.loads((tmp_path / "codex_model_catalog.json").read_text()) - assert catalog["models"][0]["slug"] == "gym-policy-model" + assert [model["slug"] for model in catalog["models"]] == ["gym-policy-model", "policy_model"] + assert catalog["models"][1]["display_name"] == "policy_model" assert catalog["models"][0]["base_instructions"] == "" +def test_runner_codex_model_catalog_uses_configured_direct_model_name(tmp_path) -> None: + runner = app.LegalAgentBenchAgent.model_construct( + config=_config( + agent_server_module="responses_api_agents.codex_agent.app", + agent_server_class="CodexAgent", + agent_config_class="CodexAgentConfig", + ), + server_client=SimpleNamespace(global_config_dict={"policy_model_name": "vendor/custom-model"}), + ) + + runner._write_runner_config( + {"runtime": tmp_path}, + NeMoGymResponseCreateParamsNonStreaming(input="Do the work"), + "http://model.internal:8000", + ) + + payload = json.loads((tmp_path / "runner.json").read_text()) + catalog = json.loads((tmp_path / "codex_model_catalog.json").read_text()) + assert payload["model_name"] == "vendor/custom-model" + assert [model["slug"] for model in catalog["models"]] == ["gym-policy-model", "vendor/custom-model"] + + def test_runner_config_preserves_explicit_codex_model_catalog(tmp_path) -> None: runner = app.LegalAgentBenchAgent.model_construct( config=_config( @@ -691,6 +902,49 @@ def test_ecs_model_url_uses_host_policy_proxy_for_reverse_tunnel(monkeypatch) -> assert runner._model_url(body) == "http://127.0.0.1:16300" +@pytest.mark.parametrize("provider_name", ["enroot", "apptainer"]) +def test_local_non_docker_providers_use_the_host_policy_proxy(monkeypatch, provider_name) -> None: + provider = {provider_name: {}} + runner = app.LegalAgentBenchAgent.model_construct( + config=_config(sandbox_provider=provider), + server_client=SimpleNamespace( + global_config_dict={}, + _build_server_base_url=lambda _config: "http://0.0.0.0:16300", + ), + ) + monkeypatch.setattr(app, "get_first_server_config_dict", lambda *_args: {}) + monkeypatch.setattr(app.LegalAgentBenchAgent, "rollout_id_from_run", lambda _self, _body: None) + body = app.LegalAgentBenchRunRequest( + instance_id="legal_agent_bench::area__task", + responses_create_params=NeMoGymResponseCreateParamsNonStreaming(input=[]), + ) + + assert runner._model_url(body) == "http://127.0.0.1:16300" + + +@pytest.mark.parametrize("provider_name", ["opensandbox", "daytona", "openshell"]) +def test_remote_providers_require_a_reachable_policy_proxy_for_host_local_models(monkeypatch, provider_name) -> None: + provider = {provider_name: {}} + runner = app.LegalAgentBenchAgent.model_construct( + config=_config(sandbox_provider=provider), + server_client=SimpleNamespace( + global_config_dict={}, + _build_server_base_url=lambda _config: "http://127.0.0.1:16300", + ), + ) + monkeypatch.setattr(app, "get_first_server_config_dict", lambda *_args: {}) + monkeypatch.setattr(app.LegalAgentBenchAgent, "rollout_id_from_run", lambda _self, _body: None) + body = app.LegalAgentBenchRunRequest( + instance_id="legal_agent_bench::area__task", + responses_create_params=NeMoGymResponseCreateParamsNonStreaming(input=[]), + ) + + with pytest.raises(app.LegalAgentBenchConfigurationError, match="sandbox_model_base_url"): + runner._model_url(body) + runner.config.sandbox_model_base_url = "https://policy-proxy.example/v1" + assert runner._model_url(body) == "https://policy-proxy.example/v1" + + @pytest.mark.parametrize( ("url", "network", "platform_name", "expected"), [ @@ -990,6 +1244,37 @@ async def reject_runtime(_image): assert "missing configured CLI pin" in response.failure_reason +@pytest.mark.asyncio +async def test_run_classifies_bad_sandbox_reference_as_terminal_configuration_failure(monkeypatch, tmp_path) -> None: + _root, task = _task_tree(tmp_path) + skills = _skills(tmp_path) + runner = app.LegalAgentBenchAgent.model_construct( + config=_config(sandbox_provider="missing_sandbox"), + server_client=SimpleNamespace(global_config_dict={}), + ) + runner._sem = app.asyncio.Semaphore(1) + runner._session_results_dir = tmp_path / "results" + + monkeypatch.setattr(app, "resolve_task_dir", lambda runtime, instance: task) + monkeypatch.setattr(app, "resolve_repo_path", lambda path: skills) + monkeypatch.setattr(app, "compose_agent_input", lambda task_dir, skills_dir, params: params) + monkeypatch.setattr(runner, "_model_name", lambda: "policy") + params = NeMoGymResponseCreateParamsNonStreaming(input=[]) + body = app.LegalAgentBenchRunRequest( + instance_id="legal_agent_bench::area__task", + responses_create_params=params, + ) + + response = await runner.run(None, body) + + assert response.configuration_failed is True + assert response.sandbox_failed is False + assert response.mask_sample is True + assert response.model_dump()["_ng_failure_class"] == "configuration_failed" + assert response.model_dump()["_ng_failure_terminal"] is True + assert "missing_sandbox" in response.failure_reason + + @pytest.mark.asyncio async def test_concurrent_image_requests_build_once(monkeypatch, tmp_path) -> None: _root, task = _task_tree(tmp_path) @@ -1057,12 +1342,6 @@ async def stop_after_overlap(_image): def test_agent_sandbox_has_no_host_mounts(monkeypatch, tmp_path) -> None: _root, task = _task_tree(tmp_path) - skills = _skills(tmp_path) - deps = tmp_path / "deps" - deps.mkdir() - paths = {name: tmp_path / name for name in ("runtime", "agent_source", "agent", "output", "workspace")} - for path in paths.values(): - path.mkdir() captured = {} def fake_sandbox(provider, spec): @@ -1078,24 +1357,68 @@ def fake_sandbox(provider, spec): runner._agent_sandbox( image="lab:image", task_dir=task, - skills_dir=skills, - deps_dir=deps, - paths=paths, model_url="http://model", ) assert captured["provider"] == {"docker": {}} + assert captured["spec"].workdir == "/tmp" + assert captured["spec"].env == app.LAB_SANDBOX_ENV assert captured["spec"].provider_options == {} +def test_direct_policy_key_is_injected_only_into_agent_sandbox(monkeypatch, tmp_path) -> None: + _root, task = _task_tree(tmp_path) + captured = [] + + def fake_sandbox(_provider, spec): + captured.append(spec) + return SimpleNamespace() + + monkeypatch.setenv("LAB_DIRECT_POLICY_KEY", "test-policy-key") # pragma: allowlist secret + runner = app.LegalAgentBenchAgent.model_construct( + config=_config( + sandbox_provider={"opensandbox": {}}, + sandbox_model_base_url="https://model.example/v1", + sandbox_model_api_key_env="LAB_DIRECT_POLICY_KEY", + ) + ) + monkeypatch.setattr(runner, "_sandbox_metadata", lambda: {}) + monkeypatch.setattr(app, "AsyncSandbox", fake_sandbox) + + runner._agent_sandbox( + image="registry.example/lab@sha256:" + "a" * 64, + task_dir=task, + model_url="https://model.example/v1", + ) + runner._verifier_sandbox( + image="registry.example/lab@sha256:" + "a" * 64, + task_dir=task, + ) + + assert captured[0].env == { + **app.LAB_SANDBOX_ENV, + "LAB_POLICY_API_KEY": "test-policy-key", # pragma: allowlist secret + } + assert captured[1].env == app.LAB_SANDBOX_ENV + assert "LAB_POLICY_API_KEY" not in captured[1].env + + +def test_direct_policy_key_must_be_present(monkeypatch, tmp_path) -> None: + _root, task = _task_tree(tmp_path) + monkeypatch.delenv("LAB_DIRECT_POLICY_KEY", raising=False) + runner = app.LegalAgentBenchAgent.model_construct( + config=_config(sandbox_model_api_key_env="LAB_DIRECT_POLICY_KEY") + ) + monkeypatch.setattr(runner, "_provider_config", lambda: {"opensandbox": {}}) + + with pytest.raises(app.LegalAgentBenchConfigurationError, match="unset or empty"): + runner._agent_sandbox( + image="registry.example/lab@sha256:" + "a" * 64, task_dir=task, model_url="https://model" + ) + + def test_ecs_agent_sandbox_tunnels_derived_policy_model_url(monkeypatch, tmp_path) -> None: _root, task = _task_tree(tmp_path) - skills = _skills(tmp_path) - deps = tmp_path / "deps" - deps.mkdir() - paths = {name: tmp_path / name for name in ("runtime", "agent_source", "agent", "output", "workspace")} - for path in paths.values(): - path.mkdir() captured = {} def fake_sandbox(provider, spec): @@ -1111,9 +1434,6 @@ def fake_sandbox(provider, spec): runner._agent_sandbox( image="registry.example/lab@sha256:" + "a" * 64, task_dir=task, - skills_dir=skills, - deps_dir=deps, - paths=paths, model_url="http://127.0.0.1:16300/rollout/run-1", ) @@ -1149,9 +1469,6 @@ def fake_sandbox(provider, spec): runner._agent_sandbox( image="registry.example/lab@sha256:" + "a" * 64, task_dir=task, - skills_dir=tmp_path / "skills", - deps_dir=tmp_path / "deps", - paths={}, model_url="https://model.example/v1", ) @@ -1160,8 +1477,6 @@ def fake_sandbox(provider, spec): def test_verifier_sandbox_has_no_host_mounts(monkeypatch, tmp_path) -> None: _root, task = _task_tree(tmp_path) - paths = {"lab_run": tmp_path / "lab-run"} - paths["lab_run"].mkdir() captured = {} def fake_sandbox(provider, spec): @@ -1174,12 +1489,107 @@ def fake_sandbox(provider, spec): monkeypatch.setattr(runner, "_sandbox_metadata", lambda: {}) monkeypatch.setattr(app, "AsyncSandbox", fake_sandbox) - runner._verifier_sandbox(image="lab:image", task_dir=task, paths=paths) + runner._verifier_sandbox(image="lab:image", task_dir=task) - assert captured["spec"].workdir == "/logs/agent/artifacts/lab-run/output" + assert captured["spec"].workdir == "/tmp" + assert captured["spec"].env == app.LAB_SANDBOX_ENV assert captured["spec"].provider_options == {} +def test_agent_and_verifier_merge_phase_specific_provider_options(monkeypatch, tmp_path) -> None: + _root, task = _task_tree(tmp_path) + captured = [] + + def fake_sandbox(_provider, spec): + captured.append(spec) + return SimpleNamespace() + + provider = {"openshell": {}} + runner = app.LegalAgentBenchAgent.model_construct( + config=_config( + sandbox_provider=provider, + agent_sandbox_provider_options={"policy": {"version": 1, "network_policies": {"agent": {}}}}, + verifier_sandbox_provider_options={"policy": {"version": 1, "network_policies": {"judge": {}}}}, + ) + ) + monkeypatch.setattr(runner, "_sandbox_metadata", lambda: {}) + monkeypatch.setattr(app, "AsyncSandbox", fake_sandbox) + + runner._agent_sandbox(image="lab:image", task_dir=task, model_url="https://model.example/v1") + runner._verifier_sandbox(image="lab:image", task_dir=task) + + assert captured[0].provider_options == {"policy": {"version": 1, "network_policies": {"agent": {}}}} + assert captured[1].provider_options == {"policy": {"version": 1, "network_policies": {"judge": {}}}} + + +@pytest.mark.parametrize( + "provider_name", + ["docker", "ecs_fargate", "enroot", "apptainer", "opensandbox", "daytona", "openshell"], +) +def test_agent_and_verifier_sandboxes_use_the_selected_provider(monkeypatch, tmp_path, provider_name) -> None: + _root, task = _task_tree(tmp_path) + provider = {provider_name: {}} + captured = [] + + def fake_sandbox(received_provider, spec): + captured.append((received_provider, spec)) + return SimpleNamespace() + + runner = app.LegalAgentBenchAgent.model_construct( + config=_config( + sandbox_provider=provider, + sandbox_model_base_url="https://policy-proxy.example/v1", + ) + ) + monkeypatch.setattr(runner, "_sandbox_metadata", lambda: {"benchmark": "legal-agent-bench"}) + monkeypatch.setattr(app, "AsyncSandbox", fake_sandbox) + + runner._agent_sandbox(image="provider-native-image", task_dir=task, model_url="https://policy-proxy.example/v1") + runner._verifier_sandbox( + image="provider-native-image", + task_dir=task, + ) + + assert [next(iter(received)) for received, _spec in captured] == [provider_name, provider_name] + if provider_name != "docker": + assert [received for received, _spec in captured] == [provider, provider] + assert all(spec.image == "provider-native-image" for _provider, spec in captured) + assert all(spec.workdir == "/tmp" for _provider, spec in captured) + assert all(spec.env == app.LAB_SANDBOX_ENV for _provider, spec in captured) + assert all(spec.metadata == {"benchmark": "legal-agent-bench"} for _provider, spec in captured) + expected_options = ( + {"resource_requests": {"cpu": 0.5, "memory_mib": 1024, "disk_gib": 10}} + if provider_name == "opensandbox" + else {} + ) + assert all(spec.provider_options == expected_options for _provider, spec in captured) + + +def test_opensandbox_request_fraction_can_be_disabled(monkeypatch, tmp_path) -> None: + _root, task = _task_tree(tmp_path) + captured = [] + + def fake_sandbox(_provider, spec): + captured.append(spec) + return SimpleNamespace() + + runner = app.LegalAgentBenchAgent.model_construct( + config=_config( + sandbox_provider={"opensandbox": {}}, + opensandbox_request_fraction=None, + sandbox_model_base_url="https://policy-proxy.example/v1", + ) + ) + monkeypatch.setattr(runner, "_sandbox_metadata", lambda: {}) + monkeypatch.setattr(app, "AsyncSandbox", fake_sandbox) + + runner._agent_sandbox(image="provider-native-image", task_dir=task, model_url="https://policy-proxy.example/v1") + runner._verifier_sandbox(image="provider-native-image", task_dir=task) + + assert len(captured) == 2 + assert all(spec.provider_options == {} for spec in captured) + + @pytest.mark.asyncio @pytest.mark.parametrize( ("reward_blob", "expected_reward", "expected_verifier_failed"), @@ -1220,6 +1630,7 @@ async def start(self): events.append(f"{self.phase}:start") async def exec(self, *args, **kwargs): + assert "user" not in kwargs events.append(f"{self.phase}:exec") return SimpleNamespace(return_code=0, error_type=None, stdout="", stderr="") @@ -1263,12 +1674,6 @@ async def run_verifier(sandbox, task_dir, paths): assert runner._session_results_dir.is_dir() return {"reward.json": reward_blob}, False, None - async def stage_agent(*args, **kwargs): - return None - - async def collect_agent(*args, **kwargs): - return {} - monkeypatch.setattr(app, "resolve_task_dir", lambda runtime, instance: task) monkeypatch.setattr(app, "resolve_repo_path", lambda path: skills) monkeypatch.setattr(app, "compose_agent_input", lambda task_dir, skills_dir, params, **kwargs: params) @@ -1488,29 +1893,42 @@ def write_runner(paths, params, model_url): class _FakeSandbox: - def __init__(self, reward: dict | None, *, timed_out: bool = False): + def __init__(self, reward: dict | None, *, timed_out: bool = False, fail_staging: bool = False): self.reward = reward self.timed_out = timed_out + self.fail_staging = fail_staging self.uploaded = [] + self.upload_modes = [] self.exec_calls = [] async def upload(self, source, destination): self.uploaded.append(destination) + self.upload_modes.append(Path(source).stat().st_mode & 0o777) async def exec(self, command, **kwargs): + assert "user" not in kwargs self.exec_calls.append((command, kwargs)) - if command.startswith("test -f"): - filename = command.rsplit("/", 1)[-1] - exists = self.reward is not None and filename in {"reward.json", "scores.json"} + if self.fail_staging and "verifier-input.tar.gz" in command: return type( - "Result", (), {"return_code": 0 if exists else 1, "error_type": None, "stdout": "", "stderr": ""} + "Result", + (), + { + "return_code": 1, + "error_type": None, + "stdout": "", + "stderr": "staging failed", + }, )() + if command.startswith("for filename in"): + stdout = "reward.json\nscores.json\n" if self.reward is not None else "" + return type("Result", (), {"return_code": 0, "error_type": None, "stdout": stdout, "stderr": ""})() + timed_out = self.timed_out and command.startswith("bash ") return type( "Result", (), { - "return_code": 124 if self.timed_out else 0, - "error_type": "timeout" if self.timed_out else None, + "return_code": 124 if timed_out else 0, + "error_type": "timeout" if timed_out else None, "stdout": "", "stderr": "", }, @@ -1535,14 +1953,45 @@ async def test_verifier_is_staged_after_agent_and_receives_secrets_only_on_exec( downloaded, timed_out, failure = await runner._stage_and_run_verifier(sandbox, task, paths) reward = runner._materialize_verifier_downloads(paths, downloaded) - assert sandbox.uploaded == ["/tmp/legal-agent-bench-verifier-input.tar.gz"] - verifier_call = next(call for call in sandbox.exec_calls if "bash /tests/test.sh" in call[0]) + assert sandbox.uploaded == [f"{app.SANDBOX_ROOT}/verifier-input.tar.gz"] + assert sandbox.upload_modes == [0o644] + assert len(sandbox.exec_calls) == 4 + assert sandbox.exec_calls[0][0] == f"mkdir -p {app.SANDBOX_ROOT}" + staging_call = sandbox.exec_calls[1] + assert f"tar -xzf {app.SANDBOX_ROOT}/verifier-input.tar.gz" in staging_call[0] + assert f"mkdir -p {app.SANDBOX_LOGS}/verifier" in staging_call[0] + assert staging_call[1]["cwd"] == "/tmp" + verifier_call = next(call for call in sandbox.exec_calls if f"bash {app.SANDBOX_TESTS}/test.sh" in call[0]) + assert "user" not in verifier_call[1] + assert verifier_call[1]["cwd"] == f"{app.SANDBOX_LOGS}/agent/artifacts/lab-run/output" assert verifier_call[1]["env"]["LAB_JUDGE_API_KEY"] == "secret" # pragma: allowlist secret + assert verifier_call[1]["env"]["LAB_TESTS_DIR"] == app.SANDBOX_TESTS + assert verifier_call[1]["env"]["LAB_LOGS_DIR"] == app.SANDBOX_LOGS + manifest_call = next(call for call in sandbox.exec_calls if call[0].startswith("for filename in")) + assert "error.json" in manifest_call[0] + assert manifest_call[0].endswith("done; true") assert reward["reward"] == 1.0 assert timed_out is False assert failure is None +@pytest.mark.asyncio +async def test_verifier_staging_failure_skips_execution_and_is_reported(tmp_path) -> None: + _root, task = _task_tree(tmp_path) + paths = {"verifier": tmp_path / "verifier", "lab_run": tmp_path / "lab-run"} + paths["verifier"].mkdir() + (paths["lab_run"] / "output").mkdir(parents=True) + sandbox = _FakeSandbox(None, fail_staging=True) + runner = app.LegalAgentBenchAgent.model_construct(config=_config()) + + reward, timed_out, failure = await runner._stage_and_run_verifier(sandbox, task, paths) + + assert reward == {} + assert timed_out is False + assert failure == "staging failed" + assert len(sandbox.exec_calls) == 2 + + @pytest.mark.asyncio async def test_missing_verifier_reward_returns_failure(tmp_path) -> None: _root, task = _task_tree(tmp_path) @@ -1590,7 +2039,9 @@ def test_materialized_output_is_owned_by_invoking_user(tmp_path) -> None: assert (output / "memo.txt").stat().st_gid == os.getgid() -def test_named_remote_provider_requires_immutable_image(tmp_path) -> None: +def test_named_remote_provider_accepts_provider_native_image_and_requires_reachable_model_url( + monkeypatch, tmp_path +) -> None: global_config = { "sandbox": { "default_metadata": {"sandbox-api": "remote"}, @@ -1599,8 +2050,13 @@ def test_named_remote_provider_requires_immutable_image(tmp_path) -> None: } runner = app.LegalAgentBenchAgent.model_construct( config=_config(sandbox_provider="sandbox", sandbox_image="registry.example/lab:latest"), - server_client=SimpleNamespace(global_config_dict=global_config), + server_client=SimpleNamespace( + global_config_dict=global_config, + _build_server_base_url=lambda _config: "http://127.0.0.1:8000", + ), ) + monkeypatch.setattr(app, "get_first_server_config_dict", lambda *_args: {}) + monkeypatch.setattr(app.LegalAgentBenchAgent, "rollout_id_from_run", lambda _self, _body: None) assert "remote" in runner._provider_config() assert runner._sandbox_metadata() == {"sandbox-api": "remote"} @@ -1610,41 +2066,33 @@ def test_named_remote_provider_requires_immutable_image(tmp_path) -> None: ) with pytest.raises(app.LegalAgentBenchConfigurationError, match="sandbox_model_base_url"): runner._model_url(body) - with pytest.raises(app.LegalAgentBenchConfigurationError, match="immutable"): - app.asyncio.run(runner._ensure_image(tmp_path)) - - runner.config.sandbox_image = "registry.example/lab@sha256:" + "a" * 64 assert app.asyncio.run(runner._ensure_image(tmp_path)) == runner.config.sandbox_image + runner.config.sandbox_model_base_url = "https://policy-proxy.example/v1" + assert runner._model_url(body) == "https://policy-proxy.example/v1" -@pytest.mark.parametrize( - "image", - [ - "registry.example/lab@sha256:", - "registry.example/lab@sha256:abc123", - "registry.example/lab@sha512:" + "a" * 128, - "registry.example/lab:latest@sha256:" + "g" * 64, - ], -) -def test_named_remote_provider_rejects_incomplete_or_invalid_digest(image, tmp_path) -> None: + +@pytest.mark.parametrize("image", ["docker://registry.example/lab:latest", "/images/lab.sif", "/images/lab.sqsh"]) +def test_non_docker_provider_preserves_native_image_reference(image, tmp_path) -> None: runner = app.LegalAgentBenchAgent.model_construct( config=_config( - sandbox_provider={"ecs_fargate": {"region": "us-east-1"}}, + sandbox_provider={"apptainer": {}}, sandbox_image=image, ) ) - with pytest.raises(app.LegalAgentBenchConfigurationError, match="complete immutable"): - app.asyncio.run(runner._ensure_image(tmp_path)) + assert app.asyncio.run(runner._ensure_image(tmp_path)) == image @pytest.mark.asyncio async def test_agent_staging_excludes_tests_and_judge_credentials(tmp_path) -> None: _root, task = _task_tree(tmp_path) skills = _skills(tmp_path) - deps = tmp_path / "cache" / "recipe" - deps.mkdir(parents=True) - (deps / ".installed").write_text("recipe") + runtime_archive = tmp_path / "runtime.tar.gz" + with tarfile.open(runtime_archive, "w:gz") as archive: + root = tarfile.TarInfo("agent_deps") + root.type = tarfile.DIRTYPE + archive.addfile(root) paths = {"agent_source": tmp_path / "agent-source", "runtime": tmp_path / "runtime"} for path in paths.values(): path.mkdir() @@ -1655,15 +2103,18 @@ async def test_agent_staging_excludes_tests_and_judge_credentials(tmp_path) -> N class CaptureSandbox: def __init__(self): self.members = {} + self.upload_modes = {} self.exec_commands = [] async def upload(self, source, destination): + self.upload_modes[destination] = Path(source).stat().st_mode & 0o777 with tarfile.open(source, "r:gz") as archive: self.members[destination] = {member.name for member in archive.getmembers()} async def exec(self, command, **kwargs): + assert "user" not in kwargs self.exec_commands.append(command) - return SimpleNamespace(return_code=0, stdout="", stderr="") + return SimpleNamespace(return_code=0, error_type=None, stdout="", stderr="") sandbox = CaptureSandbox() runner = app.LegalAgentBenchAgent.model_construct(config=_config()) @@ -1672,15 +2123,18 @@ async def exec(self, command, **kwargs): sandbox, task_dir=task, skills_dir=skills, - deps_dir=deps, + runtime_archive=runtime_archive, paths=paths, ) - staged = sandbox.members["/tmp/legal-agent-bench-agent-input.tar.gz"] + staged = sandbox.members[f"{app.SANDBOX_ROOT}/agent-input.tar.gz"] assert any(name.startswith("workspace/vdr") for name in staged) assert any(name.startswith("workspace/skills") for name in staged) assert not any("task.toml" in name or "/tests" in name or "LAB_JUDGE" in name for name in staged) - assert "chmod -R a+rX,a-w /agent_source_mount /agent_deps_mount" in sandbox.exec_commands[0] + assert set(sandbox.upload_modes.values()) == {0o644} + assert sandbox.exec_commands[0] == f"mkdir -p {app.SANDBOX_ROOT}" + assert f"chmod -R a+rX,a-w {app.SANDBOX_AGENT_SOURCE} {app.SANDBOX_AGENT_DEPS}" in sandbox.exec_commands[1] + assert "chown" not in sandbox.exec_commands[1] @pytest.mark.asyncio @@ -1702,21 +2156,20 @@ async def test_run_process_returns_output_and_terminates_on_timeout(tmp_path) -> @pytest.mark.asyncio async def test_runtime_is_provisioned_once_per_agent_instance(monkeypatch, tmp_path) -> None: + _runtime_sources(monkeypatch, tmp_path, "hermes_agent") + _RuntimeBuilderSandbox.instances = [] + monkeypatch.setattr(app, "AsyncSandbox", _RuntimeBuilderSandbox) + provider = {"enroot": {}} runner = app.LegalAgentBenchAgent.model_construct(config=_config()) runner._runtime_lock = app.asyncio.Lock() - runner._deps_dir = None - calls = [] - - def provision(*args, **kwargs): - calls.append((args, kwargs)) - return tmp_path / "deps" - - monkeypatch.setattr(app, "ensure_agent_runtime", provision) + runner._runtime_archives = {} + monkeypatch.setattr(runner, "_provider_config", lambda: provider) + monkeypatch.setattr(runner, "_sandbox_metadata", lambda: {}) first, second = await app.asyncio.gather(runner._ensure_runtime("lab:image"), runner._ensure_runtime("lab:image")) - assert first == second == tmp_path / "deps" - assert len(calls) == 1 + assert first == second + assert len(_RuntimeBuilderSandbox.instances) == 1 @pytest.mark.asyncio @@ -1773,7 +2226,7 @@ async def test_remote_provider_requires_an_image_and_missing_agent_source_is_con runner = app.LegalAgentBenchAgent.model_construct( config=_config(sandbox_provider={"ecs_fargate": {"region": "us-east-1"}}) ) - with pytest.raises(app.LegalAgentBenchConfigurationError, match="require sandbox_image"): + with pytest.raises(app.LegalAgentBenchConfigurationError, match="require.*sandbox_image"): await runner._ensure_image(tmp_path) paths = runner._paths_for_root(tmp_path / "staging", create=True) @@ -1819,9 +2272,11 @@ def test_task_prompt_and_skills_report_invalid_source_files(tmp_path) -> None: async def test_agent_staging_and_collection_surface_sandbox_failures(tmp_path) -> None: _root, task = _task_tree(tmp_path) skills = _skills(tmp_path) - deps = tmp_path / "deps" - deps.mkdir() - (deps / "runtime.py").write_text("pass\n") + runtime_archive = tmp_path / "runtime.tar.gz" + with tarfile.open(runtime_archive, "w:gz") as archive: + root = tarfile.TarInfo("agent_deps") + root.type = tarfile.DIRTYPE + archive.addfile(root) paths = app.LegalAgentBenchAgent._paths_for_root(tmp_path / "staged", create=True) (paths["agent_source"] / "app.py").write_text("pass\n") @@ -1834,7 +2289,9 @@ async def exec(self, command, **kwargs): runner = app.LegalAgentBenchAgent.model_construct(config=_config()) with pytest.raises(RuntimeError, match="sandbox command failed"): - await runner._stage_agent_sandbox(Sandbox(), task_dir=task, skills_dir=skills, deps_dir=deps, paths=paths) + await runner._stage_agent_sandbox( + Sandbox(), task_dir=task, skills_dir=skills, runtime_archive=runtime_archive, paths=paths + ) with pytest.raises(RuntimeError, match="sandbox command failed"): await runner._collect_agent_sandbox(Sandbox(), tmp_path / "downloads") diff --git a/tests/unit_tests/test_server_utils.py b/tests/unit_tests/test_server_utils.py index 35780418fa..dc6c2e4c79 100644 --- a/tests/unit_tests/test_server_utils.py +++ b/tests/unit_tests/test_server_utils.py @@ -287,10 +287,27 @@ def test_keepalive_socket_factory_skips_missing_platform_sockopts(self, monkeypa def test_GlobalAIOHTTPAsyncClientConfig_keepalive_defaults(self) -> None: cfg = GlobalAIOHTTPAsyncClientConfig() + assert cfg.global_aiohttp_trust_env is False assert cfg.global_aiohttp_tcp_keepalive_idle_seconds == 60 assert cfg.global_aiohttp_tcp_keepalive_interval_seconds == 10 assert cfg.global_aiohttp_tcp_keepalive_probes == 3 + def test_set_global_aiohttp_client_forwards_trust_env(self, monkeypatch: MonkeyPatch) -> None: + session = MagicMock() + session_ctor = MagicMock(return_value=session) + monkeypatch.setattr(nemo_gym.server_utils, "_GLOBAL_AIOHTTP_CLIENT", None) + monkeypatch.setattr(nemo_gym.server_utils, "ClientSession", session_ctor) + monkeypatch.setattr(nemo_gym.server_utils, "TCPConnector", MagicMock()) + monkeypatch.setattr(nemo_gym.server_utils, "DummyCookieJar", MagicMock()) + monkeypatch.setattr(nemo_gym.server_utils, "get_nemo_gym_fastapi_num_workers", lambda: 1) + + result = nemo_gym.server_utils.set_global_aiohttp_client( + GlobalAIOHTTPAsyncClientConfig(global_aiohttp_trust_env=True) + ) + + assert result is session + assert session_ctor.call_args.kwargs["trust_env"] is True + def test_keepalive_socket_factory_uses_configured_values(self, monkeypatch: MonkeyPatch) -> None: mock_sock = MagicMock() socket_ctor_mock = MagicMock(return_value=mock_sock) From a5c1e15ce10564738cf16f0e5b02d50342153261 Mon Sep 17 00:00:00 2001 From: Ryan Stewart Date: Thu, 13 Aug 2026 13:10:47 -0400 Subject: [PATCH 21/26] support LAB on Apptainer and Enroot Signed-off-by: Ryan Stewart --- .../sandbox/providers/apptainer/provider.py | 3 +- .../providers/enroot/configs/enroot.yaml | 5 +- nemo_gym/sandbox/providers/enroot/provider.py | 38 +++++++++++---- nemo_gym/sandbox/providers/utils.py | 20 ++++++++ tests/unit_tests/test_apptainer_provider.py | 46 +++++++++++++++---- tests/unit_tests/test_enroot_provider.py | 11 ++++- 6 files changed, 99 insertions(+), 24 deletions(-) diff --git a/nemo_gym/sandbox/providers/apptainer/provider.py b/nemo_gym/sandbox/providers/apptainer/provider.py index 33d5f5a7c7..29b84a0678 100644 --- a/nemo_gym/sandbox/providers/apptainer/provider.py +++ b/nemo_gym/sandbox/providers/apptainer/provider.py @@ -42,6 +42,7 @@ ) from nemo_gym.sandbox.providers.utils import coerce_config as _coerce_config from nemo_gym.sandbox.providers.utils import path_under_mount as _path_under_mount +from nemo_gym.sandbox.providers.utils import remove_writable_tree as _remove_writable_tree LOGGER = logging.getLogger(__name__) @@ -700,7 +701,7 @@ async def close(self, handle: SandboxHandle) -> None: # Always best-effort remove the host staging dir, even if stop failed. try: - shutil.rmtree(inst.staging_dir, ignore_errors=False) + _remove_writable_tree(inst.staging_dir) except OSError as e: LOGGER.warning("failed to remove staging dir %s: %s", inst.staging_dir, e) diff --git a/nemo_gym/sandbox/providers/enroot/configs/enroot.yaml b/nemo_gym/sandbox/providers/enroot/configs/enroot.yaml index 294eae7c03..966beca72a 100644 --- a/nemo_gym/sandbox/providers/enroot/configs/enroot.yaml +++ b/nemo_gym/sandbox/providers/enroot/configs/enroot.yaml @@ -32,9 +32,8 @@ sandbox: sqsh_cache_dir: ${oc.env:NEMO_GYM_ENROOT_SQSH_CACHE,null} # Clear the Docker ENTRYPOINT so images with a non-shell entrypoint # (e.g. ENTRYPOINT ["python"]) don't wrap the init and exit immediately. - # Set to false for SWE-bench images: enroot resolves /dev/null inside the - # container namespace before /dev is mounted, so passing --rc /dev/null - # (bypass_entrypoint: true) fails with "No such file or directory". + # The provider supplies a regular pass-through command script, as + # required by Enroot's --rc contract. bypass_entrypoint: true rw: true remap_root: false diff --git a/nemo_gym/sandbox/providers/enroot/provider.py b/nemo_gym/sandbox/providers/enroot/provider.py index 01e7f92a0a..32e3e81894 100644 --- a/nemo_gym/sandbox/providers/enroot/provider.py +++ b/nemo_gym/sandbox/providers/enroot/provider.py @@ -50,6 +50,7 @@ ) from nemo_gym.sandbox.providers.utils import coerce_config as _coerce_config from nemo_gym.sandbox.providers.utils import path_under_mount as _path_under_mount +from nemo_gym.sandbox.providers.utils import remove_writable_tree as _remove_writable_tree LOGGER = logging.getLogger(__name__) @@ -59,6 +60,11 @@ # Portable init: keep the container alive without relying on `sleep infinity`, # which busybox `sleep` rejects. A shell loop works on any image with `sh`. DEFAULT_INIT_COMMAND = "while true; do sleep 86400; done" +# Enroot's command script owns execution of the argv supplied after the +# container name. Replacing an image-generated /etc/rc with an empty file would +# therefore exit successfully without launching the requested init. This +# minimal script bypasses image ENTRYPOINT/CMD behavior while preserving argv. +ENTRYPOINT_BYPASS_RC = 'if [ "$#" -gt 0 ]; then\n exec "$@"\nfi\nexec /bin/sh\n' READY_PROBE_COMMAND = ( f"printf enroot-sandbox-ready > {DEFAULT_MOUNT_POINT}/.nemo-gym-ready && printf enroot-sandbox-ready" ) @@ -114,12 +120,17 @@ def _find_container_init_pid(base_init: str, marker: str) -> int | None: concurrency. The ``enroot start`` wrapper also carries the marker but its cmdline starts with the enroot binary, not ``sh -c ``, so it is excluded. """ - for entry in os.scandir("/proc"): - if not entry.name.isdigit(): - continue - cmd = _read_proc_cmdline(int(entry.name)) - if cmd.startswith("sh -c ") and marker in cmd and base_init in cmd: - return int(entry.name) + try: + entries = os.scandir("/proc") + except OSError: + return None + with entries: + for entry in entries: + if not entry.name.isdigit(): + continue + cmd = _read_proc_cmdline(int(entry.name)) + if cmd.startswith("sh -c ") and marker in cmd and base_init in cmd: + return int(entry.name) return None @@ -492,10 +503,17 @@ async def create(self, spec: SandboxSpec) -> SandboxHandle: # (e.g. ENTRYPOINT ["python"]) don't wrap the init and exit immediately. # Enroot bakes the ENTRYPOINT into /etc/rc inside the rootfs during # `enroot create`; the only way to bypass it at start time is --rc, - # which replaces /etc/rc entirely. We pass an empty script so the init - # command (argv after the container name) runs directly. + # which replaces /etc/rc entirely. Enroot requires --rc to name a + # regular file, so /dev/null is not a valid sentinel. The replacement + # must also exec the supplied argv; an empty script exits without + # starting the init. Keep the pass-through script in the private + # per-sandbox staging directory. Enroot copies it before the init starts + # and it is removed with staging. if self._create_config.bypass_entrypoint: - argv += ["--rc", "/dev/null"] + bypass_rc_path = staging_dir / ".nemo-gym-bypass-entrypoint-rc" + bypass_rc_path.write_text(ENTRYPOINT_BYPASS_RC) + bypass_rc_path.chmod(0o400) + argv += ["--rc", str(bypass_rc_path)] argv += list(self._create_config.extra_start_args) # Tag the init with the (unique) container name so the nested-in-pyxis PID # fallback can find THIS container's init process in /proc unambiguously. The @@ -864,7 +882,7 @@ async def close(self, handle: SandboxHandle) -> None: remove_error = e try: - shutil.rmtree(instance.staging_dir, ignore_errors=False) + _remove_writable_tree(instance.staging_dir) except OSError as e: LOGGER.warning("failed to remove staging dir %s: %s", instance.staging_dir, e) diff --git a/nemo_gym/sandbox/providers/utils.py b/nemo_gym/sandbox/providers/utils.py index bc68b6eabe..1787424a6b 100644 --- a/nemo_gym/sandbox/providers/utils.py +++ b/nemo_gym/sandbox/providers/utils.py @@ -18,8 +18,12 @@ opensandbox providers. They live here so there is a single implementation. """ +import os import posixpath +import shutil +import stat from collections.abc import Mapping +from pathlib import Path from typing import Any @@ -50,3 +54,19 @@ def path_under_mount(mount_point: str, path: str) -> str | None: if mp == "/": return normalized.lstrip("/") return normalized[len(mp) + 1 :] + + +def remove_writable_tree(path: Path) -> None: + """Remove a provider-owned staging tree even when sandbox files are read-only. + + Archive extraction can intentionally remove write permission from files and + directories exposed to an agent. Restore only the host owner's permissions + when removal encounters such an entry; never follow a symlink. + """ + + for root, directories, _files in os.walk(path, topdown=True, followlinks=False): + for candidate in (Path(root), *(Path(root) / name for name in directories)): + mode = candidate.lstat().st_mode + if not stat.S_ISLNK(mode): + candidate.chmod(stat.S_IMODE(mode) | stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) + shutil.rmtree(path) diff --git a/tests/unit_tests/test_apptainer_provider.py b/tests/unit_tests/test_apptainer_provider.py index b65e191e1b..04b4d1261a 100644 --- a/tests/unit_tests/test_apptainer_provider.py +++ b/tests/unit_tests/test_apptainer_provider.py @@ -184,7 +184,7 @@ def test_serialize_env_file_rejects_invalid_input(env: dict[Any, Any], error: ty def test_private_env_file_is_mode_0600_and_always_removed(tmp_path: Path) -> None: - content = b"TOKEN='not-a-real-secret'\n" + content = b"TOKEN='not-a-real-secret'\n" # pragma: allowlist secret with apptainer_provider._private_env_file(tmp_path, content) as path: assert path is not None @@ -387,13 +387,18 @@ def responder(argv: list[str]) -> tuple[int, str, str]: nonlocal env_file env_file = _env_file_path(argv) assert env_file.exists() - assert "not-a-real-secret" not in argv + assert "not-a-real-secret" not in argv # pragma: allowlist secret return (1, "", "boom") provider, _rec = _make_provider(monkeypatch, responder) with pytest.raises(apptainer_provider.ApptainerCreateError, match="failed"): - await provider.create(SandboxSpec(image="docker://img", env={"TOKEN": "not-a-real-secret"})) + await provider.create( + SandboxSpec( + image="docker://img", + env={"TOKEN": "not-a-real-secret"}, # pragma: allowlist secret + ) + ) assert env_file is not None and not env_file.exists() assert not staging.exists() @@ -414,8 +419,13 @@ def responder(argv: list[str]) -> tuple[int, str, str]: provider, _rec = _make_provider(monkeypatch, responder) with pytest.raises(apptainer_provider.ApptainerCreateError, match="timed out") as exc_info: - await provider.create(SandboxSpec(image="docker://img", env={"TOKEN": "not-a-real-secret"})) - assert "not-a-real-secret" not in str(exc_info.value) + await provider.create( + SandboxSpec( + image="docker://img", + env={"TOKEN": "not-a-real-secret"}, # pragma: allowlist secret + ) + ) + assert "not-a-real-secret" not in str(exc_info.value) # pragma: allowlist secret assert env_file is not None and not env_file.exists() assert not staging.exists() @@ -560,7 +570,10 @@ def responder(argv: list[str]) -> tuple[int, str, str]: provider, _rec = _make_provider(monkeypatch, responder) result = await provider.exec( - _make_handle(tmp_path, env={"TOKEN": "not-a-real-secret"}), + _make_handle( + tmp_path, + env={"TOKEN": "not-a-real-secret"}, # pragma: allowlist secret + ), "sleep 99", timeout_s=1, ) @@ -568,7 +581,7 @@ def responder(argv: list[str]) -> tuple[int, str, str]: assert result.return_code == apptainer_provider.SANDBOX_RUNTIME_RETURN_CODE assert result.error_type == "timeout" assert result.stdout is None - assert "not-a-real-secret" not in (result.stderr or "") + assert "not-a-real-secret" not in (result.stderr or "") # pragma: allowlist secret assert env_file is not None and not env_file.exists() @@ -752,6 +765,23 @@ async def test_close_success(fake_binary: str, monkeypatch: pytest.MonkeyPatch, assert _contains_seq(rec.calls[0]["argv"], [FAKE_BINARY, "instance", "stop", "nemo-gym-x"]) +async def test_close_removes_read_only_sandbox_tree( + fake_binary: str, monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + staging = tmp_path / "staging" + locked = staging / "runtime" + locked.mkdir(parents=True) + artifact = locked / "python" + artifact.write_text("runtime") + artifact.chmod(0o444) + locked.chmod(0o555) + provider, _rec = _make_provider(monkeypatch, lambda argv: (0, "", "")) + + await provider.close(_make_handle(staging)) + + assert not staging.exists() + + async def test_close_missing_instance_is_success( fake_binary: str, monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: @@ -807,7 +837,7 @@ async def test_close_staging_removal_failure_is_logged( def boom(path: Any, ignore_errors: bool = False) -> None: raise OSError("locked") - monkeypatch.setattr(apptainer_provider.shutil, "rmtree", boom) + monkeypatch.setattr(apptainer_provider, "_remove_writable_tree", boom) with caplog.at_level("WARNING"): await provider.close(_make_handle(staging)) # does not raise assert "failed to remove staging dir" in caplog.text diff --git a/tests/unit_tests/test_enroot_provider.py b/tests/unit_tests/test_enroot_provider.py index 8d4e93f674..ed96c73993 100644 --- a/tests/unit_tests/test_enroot_provider.py +++ b/tests/unit_tests/test_enroot_provider.py @@ -332,7 +332,14 @@ async def fake_ensure(image: str) -> Path: assert _contains_seq(start_argv, ["-e", "NVIDIA_VISIBLE_DEVICES=0"]) expected_init = f"{enroot_provider.DEFAULT_INIT_COMMAND} # {handle.sandbox_id}" assert start_argv[-4:] == [handle.sandbox_id, "sh", "-c", expected_init] - assert _contains_seq(start_argv, ["--rc", "/dev/null"]) + rc_index = start_argv.index("--rc") + bypass_rc_path = Path(start_argv[rc_index + 1]) + assert bypass_rc_path.parent == staging + assert bypass_rc_path.name == ".nemo-gym-bypass-entrypoint-rc" + assert bypass_rc_path.is_file() + assert bypass_rc_path.read_text() == enroot_provider.ENTRYPOINT_BYPASS_RC + assert 'exec "$@"' in bypass_rc_path.read_text() + assert bypass_rc_path.stat().st_mode & 0o777 == 0o400 # The generated container name is random; the create test above needs the `list` @@ -840,7 +847,7 @@ async def test_close_staging_removal_failure_is_logged( def boom(path: Any, ignore_errors: bool = False) -> None: raise OSError("locked") - monkeypatch.setattr(enroot_provider.shutil, "rmtree", boom) + monkeypatch.setattr(enroot_provider, "_remove_writable_tree", boom) with caplog.at_level("WARNING"): await provider.close(_make_handle(staging)) assert "failed to remove staging dir" in caplog.text From a877fbbb2bd26f80078b91a5f033b64f9c8cbea9 Mon Sep 17 00:00:00 2001 From: Ryan Stewart Date: Thu, 13 Aug 2026 13:11:10 -0400 Subject: [PATCH 22/26] support LAB on OpenSandbox Signed-off-by: Ryan Stewart --- .../opensandbox/configs/opensandbox.yaml | 2 + .../sandbox/providers/opensandbox/provider.py | 43 ++++++++++++++++--- tests/unit_tests/test_opensandbox_provider.py | 35 +++++++++++++++ 3 files changed, 73 insertions(+), 7 deletions(-) diff --git a/nemo_gym/sandbox/providers/opensandbox/configs/opensandbox.yaml b/nemo_gym/sandbox/providers/opensandbox/configs/opensandbox.yaml index 8f721e065f..f628627bf9 100644 --- a/nemo_gym/sandbox/providers/opensandbox/configs/opensandbox.yaml +++ b/nemo_gym/sandbox/providers/opensandbox/configs/opensandbox.yaml @@ -21,6 +21,8 @@ sandbox: sandbox-api: opensandbox-sdk opensandbox: connection: + # A bare host[:port] or full http(s) URL is accepted. When the URL + # includes a scheme, it must match `protocol` and must not include a path. domain: ${oc.env:OPENSANDBOX_DOMAIN,opensandbox-server.opensandbox-system.svc.cluster.local} api_key: ${oc.env:OPENSANDBOX_API_KEY} protocol: http diff --git a/nemo_gym/sandbox/providers/opensandbox/provider.py b/nemo_gym/sandbox/providers/opensandbox/provider.py index 683d2d1377..4c013da4d7 100644 --- a/nemo_gym/sandbox/providers/opensandbox/provider.py +++ b/nemo_gym/sandbox/providers/opensandbox/provider.py @@ -114,6 +114,36 @@ class SandboxBackendUnreachableError(RuntimeError): IMAGE_PULL_POLICY_ANNOTATION_EXTENSION_KEY = "opensandbox.extensions.image-pull-policy" VALID_IMAGE_PULL_POLICIES = {"Always", "IfNotPresent", "Never"} STATUS_CODE_RE = re.compile(r"(?:status code|http)\D+(\d{3})", re.IGNORECASE) +SERVER_PROXY_API_KEY_HEADER = "OPEN-SANDBOX-API-KEY" + + +def _normalize_domain_protocol(domain: str | None, protocol: str | None) -> tuple[str | None, str | None]: + """Validate bare domains and HTTP(S) service URLs without changing SDK semantics.""" + if domain is None: + return None, protocol + value = domain.strip().rstrip("/") + if not value: + raise ValueError("OpenSandbox connection domain must not be empty") + if "://" not in value: + if "/" in value: + raise ValueError("OpenSandbox connection domain must not contain a path") + return value, protocol + + parsed = urlsplit(value) + if parsed.scheme not in {"http", "https"} or not parsed.netloc: + raise ValueError("OpenSandbox connection domain URL must use http or https") + if parsed.username or parsed.password: + raise ValueError("OpenSandbox connection domain URL must not contain credentials") + if parsed.path not in {"", "/"} or parsed.query or parsed.fragment: + raise ValueError("OpenSandbox connection domain URL must not contain a path, query, or fragment") + if protocol is not None and protocol.lower() != parsed.scheme: + raise ValueError( + f"OpenSandbox connection protocol {protocol!r} conflicts with domain URL scheme {parsed.scheme!r}" + ) + # ConnectionConfig natively accepts a URL-form domain and lets its scheme + # override ``protocol``. Preserve that public SDK behavior for existing Gym + # consumers while still validating ambiguous or credential-bearing URLs. + return f"{parsed.scheme}://{parsed.netloc}", parsed.scheme def validate_image_pull_policy(image_pull_policy: str) -> str: @@ -636,14 +666,13 @@ def _connection_config( ) -> Any: _, ConnectionConfig, _, _, _ = _require_opensandbox_sdk() kwargs: dict[str, Any] = {} - if self._connection.domain is not None: - # OpenSandbox SDK 0.1.15 appends ``/v1`` directly. Normalizing here - # prevents a configured trailing slash from producing ``//v1``. - kwargs["domain"] = self._connection.domain.rstrip("/") + domain, protocol = _normalize_domain_protocol(self._connection.domain, self._connection.protocol) + if domain is not None: + kwargs["domain"] = domain if self._connection.api_key is not None: kwargs["api_key"] = self._connection.api_key - if self._connection.protocol is not None: - kwargs["protocol"] = self._connection.protocol + if protocol is not None: + kwargs["protocol"] = protocol if request_timeout_s is None: request_timeout_s = self._connection.request_timeout_s if request_timeout_s is not None: @@ -657,7 +686,7 @@ def _connection_config( # the key only in proxy mode: a direct sandbox endpoint runs # untrusted code and must never see it. if self._connection.api_key is not None: - kwargs["headers"] = {"OPEN-SANDBOX-API-KEY": self._connection.api_key} + kwargs["headers"] = {SERVER_PROXY_API_KEY_HEADER: self._connection.api_key} if self._connection.keepalive_expiry_s is not None or self._connection.disable_connection_pooling: kwargs["transport"] = self._get_transport() return ConnectionConfig(**kwargs) diff --git a/tests/unit_tests/test_opensandbox_provider.py b/tests/unit_tests/test_opensandbox_provider.py index ecb1b22484..5df1eec30e 100644 --- a/tests/unit_tests/test_opensandbox_provider.py +++ b/tests/unit_tests/test_opensandbox_provider.py @@ -518,6 +518,41 @@ def test_connection_config_and_image_policy(fake_opensandbox_sdk: None) -> None: assert "headers" not in direct._connection_config().kwargs +def test_connection_domain_url_normalization_and_proxy_secret_boundary(fake_opensandbox_sdk: None) -> None: + provider = opensandbox_provider.OpenSandboxProvider( + connection={ + "domain": "http://sandbox.example:8080/", + "api_key": "key", # pragma: allowlist secret + "protocol": "http", + "use_server_proxy": True, + } + ) + config = provider._connection_config() + assert config.kwargs["domain"] == "http://sandbox.example:8080" + assert config.kwargs["protocol"] == "http" + assert config.kwargs["headers"] == { + "OPEN-SANDBOX-API-KEY": "key", # pragma: allowlist secret + } + + direct = opensandbox_provider.OpenSandboxProvider( + connection={ + "domain": "sandbox.example", + "api_key": "key", # pragma: allowlist secret + "use_server_proxy": False, + } + )._connection_config() + assert "headers" not in direct.kwargs + + with pytest.raises(ValueError, match="conflicts"): + opensandbox_provider.OpenSandboxProvider( + connection={"domain": "https://sandbox.example", "protocol": "http"} + )._connection_config() + with pytest.raises(ValueError, match="must not contain a path"): + opensandbox_provider.OpenSandboxProvider( + connection={"domain": "http://sandbox.example/api", "protocol": "http"} + )._connection_config() + + def test_connection_transport_backends(fake_opensandbox_sdk: None, monkeypatch: pytest.MonkeyPatch) -> None: # Default backend is httpx, with the configured keepalive expiry on the pool. provider = opensandbox_provider.OpenSandboxProvider() From df0bb45d7bb2e4d69c7d97c07afa358968fb0642 Mon Sep 17 00:00:00 2001 From: Ryan Stewart Date: Thu, 13 Aug 2026 13:11:36 -0400 Subject: [PATCH 23/26] support LAB on OpenShell Signed-off-by: Ryan Stewart --- .../sandbox/providers/openshell/README.md | 8 ++ .../openshell/configs/openshell.yaml | 10 +- .../sandbox/providers/openshell/provider.py | 101 ++++++++++++--- .../legal_agent_bench/prepare.py | 26 ++-- .../legal_agent_bench/tests/test_prepare.py | 5 + .../legal_agent_bench_native_agent/app.py | 40 ++++-- .../tests/test_app.py | 18 +++ tests/unit_tests/test_openshell_provider.py | 116 ++++++++++++++++-- uv.lock | 8 +- 9 files changed, 278 insertions(+), 54 deletions(-) diff --git a/nemo_gym/sandbox/providers/openshell/README.md b/nemo_gym/sandbox/providers/openshell/README.md index 32cfd806b9..b5f43c45d4 100644 --- a/nemo_gym/sandbox/providers/openshell/README.md +++ b/nemo_gym/sandbox/providers/openshell/README.md @@ -12,6 +12,14 @@ from identical connection configs share one gRPC channel and one pool, so per-sa instances (the `AsyncSandbox` pattern) do not multiply threads or channels. Sandboxes live in the gateway workspace set by `connection.workspace` (`default` unless overridden). +OpenShell is an alpha API. Keep the CLI, Python SDK, and gateway protocol on the +same release. Gym supports both the legacy v0.0.36 lifecycle API and the +workspace-aware API introduced in v0.0.92, but an SDK cannot decode responses +from a gateway running an incompatible protocol version. Check the CLI with +`openshell --version`, the Python SDK with +`python -c 'import openshell; print(openshell.__version__)'`, and the gateway's +deployment metadata before running Gym. + ## Local quickstart (Docker compute driver) Run a local gateway with OpenShell's compose setup, which uses prebuilt GHCR images and a diff --git a/nemo_gym/sandbox/providers/openshell/configs/openshell.yaml b/nemo_gym/sandbox/providers/openshell/configs/openshell.yaml index 4a6e43ce29..00ae356566 100644 --- a/nemo_gym/sandbox/providers/openshell/configs/openshell.yaml +++ b/nemo_gym/sandbox/providers/openshell/configs/openshell.yaml @@ -28,10 +28,12 @@ sandbox: workspace: ${oc.env:OPENSHELL_WORKSPACE,default} # OIDC bearer token for authenticated gateways; null for the local compose gateway. bearer_token: ${oc.env:OPENSHELL_BEARER_TOKEN,null} - # mTLS material for TLS gateways (ca / cert+key paths); all null -> plaintext channel. - # tls_ca_path: /path/to/ca.pem - # tls_cert_path: /path/to/client.pem - # tls_key_path: /path/to/client-key.pem + # mTLS material for TLS gateways; all null selects a plaintext channel. + # `openshell gateway start` writes these files below its gateway metadata + # directory, while a compose-managed plaintext gateway leaves them unset. + tls_ca_path: ${oc.env:OPENSHELL_TLS_CA_PATH,null} + tls_cert_path: ${oc.env:OPENSHELL_TLS_CERT_PATH,null} + tls_key_path: ${oc.env:OPENSHELL_TLS_KEY_PATH,null} request_timeout_s: 30 create: # Provisioning can include an image pull on the gateway host. diff --git a/nemo_gym/sandbox/providers/openshell/provider.py b/nemo_gym/sandbox/providers/openshell/provider.py index cf64fe8e9a..910cb56690 100644 --- a/nemo_gym/sandbox/providers/openshell/provider.py +++ b/nemo_gym/sandbox/providers/openshell/provider.py @@ -27,6 +27,7 @@ import base64 import binascii import functools +import inspect import logging import math import posixpath @@ -51,7 +52,12 @@ LOGGER = logging.getLogger(__name__) -SANDBOX_NAME_PREFIX = "nemo-gym-" +# OpenShell 0.0.92 limits sandbox names to 19 characters. Preserve a readable +# Gym prefix and use 64 random bits; at one million generated names the birthday +# collision probability remains below 3e-8. +SANDBOX_NAME_PREFIX = "ng-" +SANDBOX_NAME_RANDOM_HEX = 16 +MAX_SANDBOX_NAME_LENGTH = 19 SANDBOX_LABEL = "nemo-gym.sandbox" READY_PROBE_COMMAND = "printf openshell-sandbox-ready" READY_PROBE_EXPECTED = "openshell-sandbox-ready" @@ -95,6 +101,12 @@ def _normalize_image(image: str) -> str: return image[len(prefix) :] if image.startswith(prefix) else image +def _new_sandbox_name() -> str: + name = SANDBOX_NAME_PREFIX + uuid.uuid4().hex[:SANDBOX_NAME_RANDOM_HEX] + assert len(name) <= MAX_SANDBOX_NAME_LENGTH + return name + + @functools.cache def _phase_to_status_map() -> dict[int, SandboxStatus]: """SandboxPhase -> SandboxStatus, built from the SDK's generated proto constants.""" @@ -357,12 +369,15 @@ def _build_client(connection: OpenShellConnectionConfig) -> Any: cert_path=Path(connection.tls_cert_path) if connection.tls_cert_path else None, key_path=Path(connection.tls_key_path) if connection.tls_key_path else None, ) - return SandboxClient( - connection.endpoint, - tls=tls, - bearer_token=connection.bearer_token, - timeout=connection.request_timeout_s, - ) + kwargs: dict[str, Any] = {"tls": tls, "timeout": connection.request_timeout_s} + parameters = inspect.signature(SandboxClient).parameters + if connection.bearer_token is not None: + if "bearer_token" not in parameters: + raise ValueError( + "connection.bearer_token requires an OpenShell SDK whose SandboxClient supports bearer tokens" + ) + kwargs["bearer_token"] = connection.bearer_token + return SandboxClient(connection.endpoint, **kwargs) def _acquire_shared_client(connection: OpenShellConnectionConfig, concurrency: int) -> _SharedClientState: @@ -414,6 +429,11 @@ def __init__( self._shared = _acquire_shared_client(self._connection, self._exec_config.concurrency) self._closed = False + @property + def _workspace_api(self) -> bool: + """Whether lifecycle calls use the workspace-aware OpenShell API.""" + return "workspace" in inspect.signature(self._client.create).parameters + @property def _client(self) -> Any: return self._shared.client @@ -445,11 +465,16 @@ def _build_sandbox_spec(self, spec: SandboxSpec, image: str | None, options: Ope kwargs: dict[str, Any] = {"environment": {str(k): str(v) for k, v in spec.env.items()}} if image or options.template_resources or options.driver_config: template = openshell_pb2.SandboxTemplate() + template_fields = template.DESCRIPTOR.fields_by_name if image: template.image = image if options.template_resources: template.resources.update(options.template_resources) if options.driver_config: + if "driver_config" not in template_fields: + raise OpenShellCreateError( + "provider_options['driver_config'] requires the workspace-aware OpenShell API" + ) template.driver_config.update(options.driver_config) kwargs["template"] = template if options.policy is not None: @@ -458,9 +483,18 @@ def _build_sandbox_spec(self, spec: SandboxSpec, image: str | None, options: Ope kwargs["providers"] = options.providers resources = spec.resources if resources.gpu: - kwargs["resource_requirements"] = openshell_pb2.ResourceRequirements( - gpu=openshell_pb2.GpuResourceRequirements(count=resources.gpu) - ) + spec_fields = openshell_pb2.SandboxSpec.DESCRIPTOR.fields_by_name + if "resource_requirements" in spec_fields: + kwargs["resource_requirements"] = openshell_pb2.ResourceRequirements( + gpu=openshell_pb2.GpuResourceRequirements(count=resources.gpu) + ) + elif resources.gpu == 1 and "gpu" in spec_fields: + kwargs["gpu"] = True + else: + raise OpenShellCreateError( + "This OpenShell API can request at most one unspecified GPU; upgrade the gateway and SDK " + "to request an exact GPU count" + ) ignored = [ key for key, value in ( @@ -479,6 +513,20 @@ def _build_sandbox_spec(self, spec: SandboxSpec, image: str | None, options: Ope ) return openshell_pb2.SandboxSpec(**kwargs) + async def _get_sandbox(self, name: str, workspace: str) -> Any: + if self._workspace_api: + return await self._call(self._client.get, name, workspace=workspace) + if workspace != "default": + raise OpenShellCreateError("This OpenShell API does not support non-default workspaces") + return await self._call(self._client.get, name) + + async def _delete_sandbox(self, name: str, workspace: str) -> Any: + if self._workspace_api: + return await self._call(self._client.delete, name, workspace=workspace) + if workspace != "default": + raise OpenShellCreateError("This OpenShell API does not support non-default workspaces") + return await self._call(self._client.delete, name) + async def _create_sandbox_with_retries(self, pb_spec: Any, name: str, labels: dict[str, str]) -> Any: """Issue CreateSandbox, retrying transient gRPC failures with the same name. @@ -491,12 +539,25 @@ async def _create_sandbox_with_retries(self, pb_spec: Any, name: str, labels: di attempt = 0 while True: try: - return await self._call( - self._client.create, workspace=workspace, spec=pb_spec, name=name, labels=labels - ) + if self._workspace_api: + return await self._call( + self._client.create, workspace=workspace, spec=pb_spec, name=name, labels=labels + ) + if workspace != "default": + raise OpenShellCreateError("This OpenShell API does not support non-default workspaces") + if labels: + LOGGER.warning( + "This OpenShell API does not support sandbox labels; LAB attribution metadata is omitted." + ) + return await self._call(self._client.create, spec=pb_spec) except Exception as e: if _is_already_exists(e): - return await self._call(self._client.get, name, workspace=workspace) + return await self._get_sandbox(name, workspace) + # The legacy API lets only the gateway choose the name. Retrying + # after an ambiguous transport failure could create a second + # sandbox that the client cannot identify or clean up. + if not self._workspace_api: + raise if attempt >= cfg.retries or not _is_retryable_create_error(e): raise attempt += 1 @@ -529,7 +590,7 @@ async def create(self, spec: SandboxSpec) -> SandboxHandle: image = _normalize_image(spec.image) if spec.image else None options = OpenShellProviderOptions.from_mapping(spec.provider_options) pb_spec = self._build_sandbox_spec(spec, image, options) - name = SANDBOX_NAME_PREFIX + uuid.uuid4().hex + name = _new_sandbox_name() # Marker label goes last so user metadata cannot clobber it. labels = {**{str(k): str(v) for k, v in spec.metadata.items()}, SANDBOX_LABEL: "1"} @@ -570,7 +631,7 @@ async def _wait_ready(self, handle: SandboxHandle, *, timeout_s: int | float) -> last_phase: int | None = None while True: try: - ref = await self._call(self._client.get, inst.name, workspace=inst.workspace) + ref = await self._get_sandbox(inst.name, inst.workspace) last_phase = ref.phase except Exception as e: if not _is_runtime_failure(e): @@ -625,7 +686,7 @@ async def _verify_created_handle(self, handle: SandboxHandle) -> None: async def _cleanup_failed_create_handle(self, handle: SandboxHandle) -> None: inst = handle.raw try: - await self._call(self._client.delete, inst.name, workspace=inst.workspace) + await self._delete_sandbox(inst.name, inst.workspace) except Exception as e: LOGGER.warning( f"Failed to delete half-created OpenShell sandbox {inst.name!r}; it may be leaked on the gateway: {e}" @@ -731,7 +792,7 @@ async def status(self, handle: SandboxHandle) -> SandboxStatus: """Sandbox phase via GetSandbox (missing -> STOPPED; RPC failure -> UNKNOWN).""" inst = handle.raw try: - ref = await self._call(self._client.get, inst.name, workspace=inst.workspace) + ref = await self._get_sandbox(inst.name, inst.workspace) except Exception as e: if _is_not_found(e): return SandboxStatus.STOPPED @@ -744,7 +805,7 @@ async def close(self, handle: SandboxHandle) -> None: """Delete the sandbox (already-gone counts as success), then wait until it is fully gone.""" inst = handle.raw try: - deleted = await self._call(self._client.delete, inst.name, workspace=inst.workspace) + deleted = await self._delete_sandbox(inst.name, inst.workspace) except Exception as e: if _is_not_found(e): return @@ -763,7 +824,7 @@ async def _wait_deleted(self, inst: _OpenShellSandbox) -> None: deadline = loop.time() + self._operations.close_timeout_s while True: try: - await self._call(self._client.get, inst.name, workspace=inst.workspace) + await self._get_sandbox(inst.name, inst.workspace) except Exception as e: if _is_not_found(e): return diff --git a/resources_servers/legal_agent_bench/prepare.py b/resources_servers/legal_agent_bench/prepare.py index cd0952a028..99bbeda90f 100644 --- a/resources_servers/legal_agent_bench/prepare.py +++ b/resources_servers/legal_agent_bench/prepare.py @@ -46,7 +46,7 @@ INDEX_FILENAME = "all.jsonl" DEFAULT_INDEX_FPATH = PACKAGE_DIR / "data" / "generated" / INDEX_FILENAME CACHE_MARKER = ".nemo_gym_asset.json" -CACHE_FORMAT_VERSION = 4 +CACHE_FORMAT_VERSION = 5 REWARD_MODES = ("full_task", "criteria_pass_rate") REWARD_MODE_ENV_KEY = "LEGAL_AGENT_BENCH_REWARD_MODE" LAB_HARBOR_SOURCE_DIR = PACKAGE_DIR / "vendor" / "harvey_labs" / "lab_harbor" @@ -80,9 +80,17 @@ && apt-get install -y --no-install-recommends \\ bash ca-certificates coreutils curl file findutils fonts-liberation g++ \\ gawk gcc git grep jq libreoffice nodejs npm pandoc poppler-utils procps \\ - qpdf ripgrep sed tesseract-ocr \\ + iproute2 qpdf ripgrep sed tesseract-ocr \\ && rm -rf /var/lib/apt/lists/* +# OpenShell executes commands as a restricted sandbox identity and requires +# iproute2 for its network namespace. The high UID/GID avoids colliding with +# ordinary host users when user-namespace remapping is unavailable. +RUN groupadd --gid 1000660000 sandbox \\ + && useradd -K UID_MAX=1000660000 --no-log-init --uid 1000660000 \\ + --gid sandbox --create-home --shell /bin/bash sandbox \\ + && install -d -o sandbox -g sandbox /workspace/output + RUN python -m pip install --upgrade pip \\ && python -m pip install \\ "defusedxml>=0.7.1" "diff-match-patch>=20230430" "docxtpl>=0.19.0" \\ @@ -102,12 +110,14 @@ _TEST_SCRIPT = """#!/usr/bin/env bash set -euo pipefail -mkdir -p /logs/verifier -python /tests/legal_agent_bench_verify.py \\ - --task-json /tests/task.json \\ - --run-dir /logs/agent/artifacts/lab-run \\ - --verifier-dir /logs/verifier \\ - --reward-json /logs/verifier/reward.json +tests_dir="${LAB_TESTS_DIR:-/tests}" +logs_dir="${LAB_LOGS_DIR:-/logs}" +mkdir -p "$logs_dir/verifier" +python "$tests_dir/legal_agent_bench_verify.py" \\ + --task-json "$tests_dir/task.json" \\ + --run-dir "$logs_dir/agent/artifacts/lab-run" \\ + --verifier-dir "$logs_dir/verifier" \\ + --reward-json "$logs_dir/verifier/reward.json" """ diff --git a/resources_servers/legal_agent_bench/tests/test_prepare.py b/resources_servers/legal_agent_bench/tests/test_prepare.py index a39c4d0ee6..d7763e0092 100644 --- a/resources_servers/legal_agent_bench/tests/test_prepare.py +++ b/resources_servers/legal_agent_bench/tests/test_prepare.py @@ -94,6 +94,11 @@ def test_generated_task_cache_is_deterministic_and_credential_free(monkeypatch, "legal_agent_bench::area__task-one", ] assert all("agent_ref" not in row for row in rows) + dockerfile = (first / "area__task-one" / "environment" / "Dockerfile").read_text(encoding="utf-8") + assert "iproute2" in dockerfile + assert "groupadd --gid 1000660000 sandbox" in dockerfile + assert "useradd -K UID_MAX=1000660000" in dockerfile + assert "install -d -o sandbox -g sandbox /workspace/output" in dockerfile for toml in first.glob("*/task.toml"): text = toml.read_text(encoding="utf-8") assert "[verifier.env]" not in text diff --git a/responses_api_agents/legal_agent_bench_native_agent/app.py b/responses_api_agents/legal_agent_bench_native_agent/app.py index b6e60a5b97..01f60ac260 100644 --- a/responses_api_agents/legal_agent_bench_native_agent/app.py +++ b/responses_api_agents/legal_agent_bench_native_agent/app.py @@ -62,6 +62,7 @@ class LegalAgentBenchNativeAgentConfig(BaseResponsesAPIAgentConfig): model_server: ModelServerRef max_turns: int = Field(default=60, ge=1) shell_timeout: int = Field(default=60, ge=1) + preflight_timeout_seconds: int = Field(default=120, ge=1) model_timeout_seconds: int = Field(default=1800, ge=1) max_output_chars: int = Field(default=16_384, ge=1) @@ -73,8 +74,15 @@ class LegalAgentBenchNativeRunRequest(BaseRunRequest): class LabToolExecutor: """Execute the upstream LAB tools in the current task sandbox.""" - def __init__(self, *, timeout_seconds: int, max_output_chars: int) -> None: + def __init__( + self, + *, + timeout_seconds: int, + max_output_chars: int, + preflight_timeout_seconds: int | None = None, + ) -> None: self.timeout_seconds = timeout_seconds + self.preflight_timeout_seconds = preflight_timeout_seconds or timeout_seconds self.max_output_chars = max_output_chars async def preflight(self) -> None: @@ -104,17 +112,30 @@ async def execute(self, name: str, arguments: str | dict[str, Any]) -> str: # but stream the command over stdin so argv remains fixed-size. result = await self._run(["/bin/bash", "-l", "-s"], stdin=command.encode()) else: - result = await self._run( - ["/usr/local/bin/python", str(CONTAINER_TOOL_RUNNER), name], - stdin=json.dumps(parsed).encode(), - ) + command = ["/usr/local/bin/python", str(CONTAINER_TOOL_RUNNER), name] + stdin = json.dumps(parsed).encode() + if name == "preflight": + result = await self._run( + command, + stdin=stdin, + timeout_seconds=self.preflight_timeout_seconds, + ) + else: + result = await self._run(command, stdin=stdin) full_read = name == "read" and "limit" in parsed and parsed.get("limit") in {0, None} return result if full_read else self._truncate(result) - async def _run(self, command: list[str], *, stdin: bytes | None = None) -> str: + async def _run( + self, + command: list[str], + *, + stdin: bytes | None = None, + timeout_seconds: int | None = None, + ) -> str: + effective_timeout = timeout_seconds or self.timeout_seconds process = await asyncio.create_subprocess_exec( *command, - cwd="/workspace/output", + cwd=os.environ.get("OUTPUT_DIR", "/workspace/output"), env=os.environ.copy(), stdin=asyncio.subprocess.PIPE if stdin is not None else None, stdout=asyncio.subprocess.PIPE, @@ -122,14 +143,14 @@ async def _run(self, command: list[str], *, stdin: bytes | None = None) -> str: start_new_session=True, ) try: - stdout, stderr = await asyncio.wait_for(process.communicate(input=stdin), timeout=self.timeout_seconds) + stdout, stderr = await asyncio.wait_for(process.communicate(input=stdin), timeout=effective_timeout) except asyncio.TimeoutError: try: os.killpg(process.pid, signal.SIGKILL) except ProcessLookupError: pass await process.communicate() - return f"Error: tool timed out after {self.timeout_seconds}s" + return f"Error: tool timed out after {effective_timeout}s" output = stdout.decode(errors="replace") error = stderr.decode(errors="replace") @@ -261,6 +282,7 @@ async def responses( executor = LabToolExecutor( timeout_seconds=self.config.shell_timeout, + preflight_timeout_seconds=self.config.preflight_timeout_seconds, max_output_chars=self.config.max_output_chars, ) try: diff --git a/responses_api_agents/legal_agent_bench_native_agent/tests/test_app.py b/responses_api_agents/legal_agent_bench_native_agent/tests/test_app.py index 69d8470d77..a6b8a9d09e 100644 --- a/responses_api_agents/legal_agent_bench_native_agent/tests/test_app.py +++ b/responses_api_agents/legal_agent_bench_native_agent/tests/test_app.py @@ -258,6 +258,24 @@ async def test_preflight_rejects_container_tool_error(monkeypatch) -> None: await _REAL_PREFLIGHT(executor) +async def test_preflight_uses_its_dedicated_timeout(monkeypatch) -> None: + executor = app.LabToolExecutor( + timeout_seconds=60, + preflight_timeout_seconds=120, + max_output_chars=100, + ) + run = AsyncMock(return_value="OK") + monkeypatch.setattr(executor, "_run", run) + + await _REAL_PREFLIGHT(executor) + + run.assert_awaited_once_with( + ["/usr/local/bin/python", str(app.CONTAINER_TOOL_RUNNER), "preflight"], + stdin=b"{}", + timeout_seconds=120, + ) + + async def test_explicit_full_read_is_not_truncated(monkeypatch) -> None: executor = app.LabToolExecutor(timeout_seconds=1, max_output_chars=5) monkeypatch.setattr(executor, "_run", AsyncMock(return_value="complete document")) diff --git a/tests/unit_tests/test_openshell_provider.py b/tests/unit_tests/test_openshell_provider.py index e5885230ba..f5af35e084 100644 --- a/tests/unit_tests/test_openshell_provider.py +++ b/tests/unit_tests/test_openshell_provider.py @@ -32,13 +32,26 @@ pytest.importorskip("openshell", reason="openshell optional dependency is not installed") import grpc # noqa: E402 (grpcio ships with the openshell SDK) -from openshell import SandboxError, SandboxRef, SandboxStatusRef # noqa: E402 +from openshell import SandboxClient, SandboxError, SandboxRef # noqa: E402 from openshell._proto import openshell_pb2, sandbox_pb2 # noqa: E402 + +try: # OpenShell 0.0.92+ nests lifecycle state in SandboxStatusRef. + from openshell import SandboxStatusRef # type: ignore[attr-defined] # noqa: E402 +except ImportError: # OpenShell 0.0.36 exposes phase directly on SandboxRef. + SandboxStatusRef = None # type: ignore[assignment,misc] + + +WORKSPACE_SDK = "workspace" in inspect.signature(SandboxClient.create).parameters +RESOURCE_REQUIREMENTS_SDK = "resource_requirements" in openshell_pb2.SandboxSpec.DESCRIPTOR.fields_by_name +DRIVER_CONFIG_SDK = "driver_config" in openshell_pb2.SandboxTemplate.DESCRIPTOR.fields_by_name + from nemo_gym.sandbox.providers.openshell import provider as openshell_provider # noqa: E402 from nemo_gym.sandbox.providers.openshell.provider import ( # noqa: E402 + MAX_SANDBOX_NAME_LENGTH, SANDBOX_LABEL, SANDBOX_NAME_PREFIX, + SANDBOX_NAME_RANDOM_HEX, SANDBOX_RUNTIME_RETURN_CODE, OpenShellConnectionConfig, OpenShellCreateConfig, @@ -49,6 +62,7 @@ OpenShellProbeConfig, OpenShellProvider, OpenShellProviderOptions, + _new_sandbox_name, _OpenShellSandbox, ) @@ -71,7 +85,9 @@ def make_ref( name: str = "nemo-gym-test", workspace: str = "default", ) -> SandboxRef: - """A real SDK SandboxRef, so tests exercise the SDK's actual shape (nested status/phase).""" + """A real SDK SandboxRef, exercising the installed SDK's lifecycle-result shape.""" + if SandboxStatusRef is None: + return SandboxRef(id=sandbox_id, name=name, namespace=workspace, phase=phase) return SandboxRef( id=sandbox_id, name=name, @@ -157,6 +173,22 @@ def close(self) -> None: self.close_calls += 1 +class LegacyFakeClient(FakeClient): + """OpenShell's pre-workspace lifecycle API used by local gateway 0.0.36.""" + + def create(self, *, spec: Any = None) -> Any: + self.create_calls.append({"spec": spec}) + return self._next(self.create_results) + + def get(self, sandbox_name: str) -> Any: + self.get_calls.append({"name": sandbox_name}) + return self._next(self.get_results) + + def delete(self, sandbox_name: str) -> Any: + self.delete_calls.append({"name": sandbox_name}) + return self._next(self.delete_results) + + @pytest.fixture(autouse=True) def clear_shared_clients() -> Any: openshell_provider._SHARED_CLIENTS.clear() @@ -230,23 +262,80 @@ def fake_import( openshell_provider._require_openshell() +def test_generated_name_respects_openshell_limit(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(openshell_provider.uuid, "uuid4", lambda: SimpleNamespace(hex="a" * 32)) + name = _new_sandbox_name() + assert name == SANDBOX_NAME_PREFIX + "a" * SANDBOX_NAME_RANDOM_HEX + assert len(name) == MAX_SANDBOX_NAME_LENGTH + + def test_provider_call_shapes_bind_against_installed_sdk() -> None: """Bind the provider's exact SDK call shapes against the installed openshell signatures. This is the drift guard: an SDK release that changes a lifecycle signature (like 0.0.92 adding required ``workspace``) fails here even though the unit tests run against fakes. """ - from openshell import SandboxClient - - inspect.signature(SandboxClient.create).bind(None, workspace="w", spec=None, name="n", labels={}) - inspect.signature(SandboxClient.get).bind(None, "n", workspace="w") - inspect.signature(SandboxClient.delete).bind(None, "n", workspace="w") + if WORKSPACE_SDK: + inspect.signature(SandboxClient.create).bind(None, workspace="w", spec=None, name="n", labels={}) + inspect.signature(SandboxClient.get).bind(None, "n", workspace="w") + inspect.signature(SandboxClient.delete).bind(None, "n", workspace="w") + else: + inspect.signature(SandboxClient.create).bind(None, spec=None) + inspect.signature(SandboxClient.get).bind(None, "n") + inspect.signature(SandboxClient.delete).bind(None, "n") inspect.signature(SandboxClient.exec).bind( None, "sid", ["/bin/sh", "-c", "true"], workdir=None, env=None, stdin=None, timeout_seconds=None ) inspect.signature(SandboxClient.close).bind(None) +@pytest.mark.asyncio +async def test_legacy_lifecycle_api_omits_unsupported_workspace_name_and_labels( + monkeypatch: pytest.MonkeyPatch, +) -> None: + client = LegacyFakeClient() + client.get_results = [ + make_ref(openshell_pb2.SANDBOX_PHASE_READY), + FakeRpcError(grpc.StatusCode.NOT_FOUND), + ] + monkeypatch.setattr(openshell_provider, "_build_client", lambda _connection: client) + provider = OpenShellProvider( + create={"poll_interval_s": 0.01}, + probe={"command": None}, + operations={"poll_interval_s": 0.01}, + ) + + handle = await provider.create(SandboxSpec(metadata={"purpose": "test"})) + await provider.close(handle) + + assert client.create_calls == [{"spec": client.create_calls[0]["spec"]}] + assert client.get_calls == [{"name": "nemo-gym-test"}, {"name": "nemo-gym-test"}] + assert client.delete_calls == [{"name": "nemo-gym-test"}] + + +@pytest.mark.asyncio +async def test_legacy_lifecycle_api_rejects_workspace_and_does_not_retry_ambiguous_create( + monkeypatch: pytest.MonkeyPatch, +) -> None: + client = LegacyFakeClient() + client.create_results = [FakeRpcError(grpc.StatusCode.UNAVAILABLE)] + monkeypatch.setattr(openshell_provider, "_build_client", lambda _connection: client) + provider = OpenShellProvider( + connection={"workspace": "team-a"}, + create={"retries": 3, "retry_delay_s": 0}, + probe={"command": None}, + ) + + with pytest.raises(OpenShellCreateError, match="non-default workspaces"): + await provider.create(SandboxSpec()) + assert client.create_calls == [] + + provider._connection = openshell_provider.OpenShellConnectionConfig() + with pytest.raises(OpenShellCreateError, match="CreateSandbox failed"): + await provider.create(SandboxSpec()) + assert len(client.create_calls) == 1 + + def test_constructor_builds_real_client_and_config_coercion() -> None: provider = OpenShellProvider(connection={"endpoint": "localhost:1", "request_timeout_s": 5}) try: @@ -319,7 +408,7 @@ async def test_create_success_maps_spec(make_provider, fake_client: FakeClient) env={"FOO": "bar"}, metadata={"task": "demo", SANDBOX_LABEL: "user-override"}, workdir="/workspace", - resources={"gpu": 2}, + resources={"gpu": 2 if RESOURCE_REQUIREMENTS_SDK else 1}, provider_options={"providers": ["nvidia"]}, ) handle = await provider.create(spec) @@ -327,13 +416,17 @@ async def test_create_success_maps_spec(make_provider, fake_client: FakeClient) call = fake_client.create_calls[0] assert call["workspace"] == "default" assert call["name"].startswith(SANDBOX_NAME_PREFIX) + assert len(call["name"]) <= MAX_SANDBOX_NAME_LENGTH # The marker label is applied last, so user metadata cannot clobber it. assert call["labels"] == {"task": "demo", SANDBOX_LABEL: "1"} pb_spec = call["spec"] assert pb_spec.template.image == "python:3.12-slim" assert dict(pb_spec.environment) == {"FOO": "bar"} assert list(pb_spec.providers) == ["nvidia"] - assert pb_spec.resource_requirements.gpu.count == 2 + if RESOURCE_REQUIREMENTS_SDK: + assert pb_spec.resource_requirements.gpu.count == 2 + else: + assert pb_spec.gpu is True assert handle.provider_name == "openshell" assert handle.sandbox_id == "sbx-1" @@ -360,6 +453,11 @@ async def test_create_without_image_uses_gateway_default(make_provider, fake_cli async def test_create_template_resources_and_driver_config(make_provider, fake_client: FakeClient) -> None: provider = make_provider() + if not DRIVER_CONFIG_SDK: + with pytest.raises(OpenShellCreateError, match="driver_config.*workspace-aware"): + await provider.create(SandboxSpec(provider_options={"driver_config": {"runtime": "kata"}})) + assert not fake_client.create_calls + return await provider.create( SandboxSpec( provider_options={ diff --git a/uv.lock b/uv.lock index c3f6d21401..d7a8f9495b 100644 --- a/uv.lock +++ b/uv.lock @@ -3340,7 +3340,7 @@ wheels = [ [[package]] name = "openshell" -version = "0.0.92" +version = "0.0.103" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cloudpickle" }, @@ -3349,9 +3349,9 @@ dependencies = [ { name = "protobuf" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/cc/e3/be7c389ae1b12312b302da847b82734475d2689188d2a19645ac8c1223c8/openshell-0.0.92-py3-none-macosx_13_0_arm64.whl", hash = "sha256:3d673a98e66520eabd4b06fbff126d55718fb92c868fbfeb994adb34970a4796", size = 8470512, upload-time = "2026-07-27T15:32:55.574Z" }, - { url = "https://files.pythonhosted.org/packages/5f/ca/604e9a4b9701f11c51ca4129699ca3a548aad734254431bdfcda48f17e15/openshell-0.0.92-py3-none-manylinux_2_39_aarch64.whl", hash = "sha256:036fa76cd89dd49375405edde55ee35f72c0a3bd22f990942a336f4ededf15ef", size = 8532736, upload-time = "2026-07-27T15:33:16.548Z" }, - { url = "https://files.pythonhosted.org/packages/4c/fc/24a15a862925d19b1d318be2796aa996233dc30b1b9e3c4862ec1039c6f4/openshell-0.0.92-py3-none-manylinux_2_39_x86_64.whl", hash = "sha256:8a20fea53ffff0c6127ca3f6841fb4b268ced1ccb03917cdbf278e419c1e9fdf", size = 9018946, upload-time = "2026-07-27T15:33:37.985Z" }, + { url = "https://files.pythonhosted.org/packages/6b/67/2ce32a11297ba68cdb2b26b276f5ccd1e97e869e90ce4e51925b18aa2b80/openshell-0.0.103-py3-none-macosx_13_0_arm64.whl", hash = "sha256:d1c467a0c6925385c415fb8856a3c9ec627b7e51b711c6b33cfea51d718d965d", size = 8493933, upload-time = "2026-08-11T15:07:59.193Z" }, + { url = "https://files.pythonhosted.org/packages/31/42/4d25a11e8ff20690b0f63c254d8cc870faf793722f89e9ca8617bda19577/openshell-0.0.103-py3-none-manylinux_2_39_aarch64.whl", hash = "sha256:b623addf09fea656f66306201877c5259fd7a20aaa9a15a704ee97a669535cd6", size = 8562102, upload-time = "2026-08-11T15:08:17.257Z" }, + { url = "https://files.pythonhosted.org/packages/b5/4b/33bce6de7993704f1acc00e5b6d476f292b3ae724fbf65121aadcf61724b/openshell-0.0.103-py3-none-manylinux_2_39_x86_64.whl", hash = "sha256:b0e961425bcdfed930967efd89f806e24142436c34a30234d4dc888f92fec8d0", size = 9055035, upload-time = "2026-08-11T15:08:41.267Z" }, ] [[package]] From 953bc8767ed898b308f328c391a7c6fcb6d65900 Mon Sep 17 00:00:00 2001 From: Ryan Stewart Date: Thu, 13 Aug 2026 13:11:53 -0400 Subject: [PATCH 24/26] normalize Codex response replay for LAB Signed-off-by: Ryan Stewart --- nemo_gym/responses_streaming.py | 31 +++++++++++++++++++ .../test_responses_api_model_streaming.py | 28 +++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/nemo_gym/responses_streaming.py b/nemo_gym/responses_streaming.py index 0f9c1d9e09..85c7a703c1 100644 --- a/nemo_gym/responses_streaming.py +++ b/nemo_gym/responses_streaming.py @@ -34,6 +34,7 @@ import json import logging from copy import deepcopy +from hashlib import sha256 from typing import Any, Iterator, Optional from uuid import uuid4 @@ -105,6 +106,35 @@ def _input_message_text(item: dict[str, Any]) -> str: return "".join(parts) +def _synthetic_replay_item_id(item: dict[str, Any], prefix: str) -> str: + """Return a stable ID for a replay item whose wire representation omitted one.""" + payload = json.dumps(item, sort_keys=True, separators=(",", ":"), default=str).encode() + return f"{prefix}_{sha256(payload).hexdigest()[:24]}" + + +def _normalize_replayed_output_item(item: Any) -> Any: + """Fill optional response metadata omitted when Codex replays prior output. + + The Responses wire API accepts output items without their server-generated + IDs and accepts ``output_text`` parts without annotations when those items + are supplied as subsequent input. Gym's shared response/input models require + both fields because they also model newly generated output. Add only those + transport defaults so the conversation content survives strict validation. + """ + if not isinstance(item, dict): + return item + item_type = item.get("type") + if item_type == "reasoning" and not item.get("id"): + item["id"] = _synthetic_replay_item_id(item, "rs") + elif item_type == "message" and item.get("role") == "assistant": + if not item.get("id"): + item["id"] = _synthetic_replay_item_id(item, "msg") + for part in item.get("content") or []: + if isinstance(part, dict) and part.get("type") == "output_text": + part.setdefault("annotations", []) + return item + + def sanitize_streaming_responses_body(body: dict[str, Any]) -> tuple[dict[str, Any], NamespaceMap]: """Map a streaming-dialect request body onto the strict non-streaming params shape. @@ -131,6 +161,7 @@ def sanitize_streaming_responses_body(body: dict[str, Any]) -> tuple[dict[str, A kept_items = [] carrier_tools: list[Any] = [] for item in input_items: + item = _normalize_replayed_output_item(item) if isinstance(item, dict) and item.get("type") == "function_call" and item.get("namespace"): item["name"] = f"{item.pop('namespace')}{NAMESPACE_TOOL_DELIMITER}{item.get('name')}" # Codex's code mode ships tools inside an `additional_tools` input item instead of the diff --git a/tests/unit_tests/test_responses_api_model_streaming.py b/tests/unit_tests/test_responses_api_model_streaming.py index 78d7703f11..9b3d44f580 100644 --- a/tests/unit_tests/test_responses_api_model_streaming.py +++ b/tests/unit_tests/test_responses_api_model_streaming.py @@ -317,6 +317,34 @@ def test_rewrites_namespaced_calls_in_input_history(self) -> None: assert "namespace" not in call NeMoGymResponseCreateParamsNonStreaming.model_validate(cleaned) + def test_normalizes_codex_replayed_reasoning_and_assistant_output(self) -> None: + body = { + "stream": True, + "input": [ + {"type": "reasoning", "summary": []}, + { + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "I inspected the files."}], + }, + {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Continue."}]}, + ], + } + + cleaned, _ = sanitize_streaming_responses_body(body) + cleaned_again, _ = sanitize_streaming_responses_body(body) + + reasoning, assistant, _user = cleaned["input"] + assert reasoning["id"].startswith("rs_") + assert assistant["id"].startswith("msg_") + assert assistant["content"][0]["annotations"] == [] + assert reasoning["id"] == cleaned_again["input"][0]["id"] + assert assistant["id"] == cleaned_again["input"][1]["id"] + assert "id" not in body["input"][0] + assert "annotations" not in body["input"][1]["content"][0] + NeMoGymResponseCreateParamsNonStreaming.model_validate(cleaned) + def test_drops_unsupported_input_items(self) -> None: # Codex's code_mode interleaves an `additional_tools` carrier item into the input history; # the Gym input union has no representation for it, so it is dropped item-by-item. From c7c19af793f568990b93986cdf222358f1fb86c7 Mon Sep 17 00:00:00 2001 From: Ryan Stewart Date: Thu, 13 Aug 2026 13:26:36 -0400 Subject: [PATCH 25/26] document how to publish LAB sandbox images Signed-off-by: Ryan Stewart --- benchmarks/legal_agent_bench/README.md | 41 ++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/benchmarks/legal_agent_bench/README.md b/benchmarks/legal_agent_bench/README.md index 01d3b0c95c..75d7e1499d 100644 --- a/benchmarks/legal_agent_bench/README.md +++ b/benchmarks/legal_agent_bench/README.md @@ -380,6 +380,47 @@ must permit writes under `/sandbox`. The first rollout starts a short-lived runt builder sandbox with outbound package-download access. The resulting portable, content-addressed runtime is reused by later rollouts. +### Build and publish the LAB image + +After preparation, every task contains the same generated LAB image context. +The following example selects one of those contexts and uses Docker Buildx to +build and publish an OCI image to a registry that the sandbox provider can +access: + +```bash +gym eval prepare --benchmark legal_agent_bench + +LAB_IMAGE_CONTEXT="$(find \ + resources_servers/legal_agent_bench/data/cache/harbor_tasks/legal_agent_bench \ + -mindepth 2 -maxdepth 2 -type d -name environment -print -quit)" +export LAB_IMAGE="docker.io//legal-agent-bench:" + +docker login docker.io +docker buildx build \ + --platform linux/amd64 \ + --tag "$LAB_IMAGE" \ + --push \ + "$LAB_IMAGE_CONTEXT" +docker buildx imagetools inspect "$LAB_IMAGE" +``` + +Replace `linux/amd64` when the target sandbox uses another architecture. For +reproducible evaluations, copy the published digest from the inspection output +and configure the immutable reference rather than its mutable tag: + +```bash +export NEMO_GYM_LAB_SANDBOX_IMAGE="docker.io//legal-agent-bench@sha256:" +``` + +Docker Buildx is only an example image-building workflow. The image may instead +be produced and published by any preferred OCI-compatible builder, including a +CI system; Docker does not need to be installed on the machine that launches a +non-Docker LAB sandbox. That machine needs only the selected provider and an +image reference it can use. Enroot and Apptainer may also use local `.sqsh` and +`.sif` images, respectively, without publishing them to a registry. For a +private registry, configure image-pull credentials through the selected +provider rather than placing credentials in the LAB image reference. + OpenShell's Docker compute driver additionally requires `iproute2`, a restricted `sandbox` user and group, and a work directory writable by that identity. Images generated by LAB preparation include these requirements. From 27f06f34bd150f6a679d0810029c48a5c6d5264a Mon Sep 17 00:00:00 2001 From: Ryan Stewart Date: Fri, 14 Aug 2026 23:22:35 -0400 Subject: [PATCH 26/26] fix LAB CI checks Signed-off-by: Ryan Stewart --- nemo_gym/sandbox/providers/opensandbox/provider.py | 2 +- .../legal_agent_bench_agent/tests/test_app.py | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/nemo_gym/sandbox/providers/opensandbox/provider.py b/nemo_gym/sandbox/providers/opensandbox/provider.py index 4c013da4d7..78ef5b6a89 100644 --- a/nemo_gym/sandbox/providers/opensandbox/provider.py +++ b/nemo_gym/sandbox/providers/opensandbox/provider.py @@ -114,7 +114,7 @@ class SandboxBackendUnreachableError(RuntimeError): IMAGE_PULL_POLICY_ANNOTATION_EXTENSION_KEY = "opensandbox.extensions.image-pull-policy" VALID_IMAGE_PULL_POLICIES = {"Always", "IfNotPresent", "Never"} STATUS_CODE_RE = re.compile(r"(?:status code|http)\D+(\d{3})", re.IGNORECASE) -SERVER_PROXY_API_KEY_HEADER = "OPEN-SANDBOX-API-KEY" +SERVER_PROXY_API_KEY_HEADER = "OPEN-SANDBOX-API-KEY" # pragma: allowlist secret def _normalize_domain_protocol(domain: str | None, protocol: str | None) -> tuple[str | None, str | None]: diff --git a/responses_api_agents/legal_agent_bench_agent/tests/test_app.py b/responses_api_agents/legal_agent_bench_agent/tests/test_app.py index 53462b1fd1..57f7b33f7c 100644 --- a/responses_api_agents/legal_agent_bench_agent/tests/test_app.py +++ b/responses_api_agents/legal_agent_bench_agent/tests/test_app.py @@ -1379,7 +1379,7 @@ def fake_sandbox(_provider, spec): config=_config( sandbox_provider={"opensandbox": {}}, sandbox_model_base_url="https://model.example/v1", - sandbox_model_api_key_env="LAB_DIRECT_POLICY_KEY", + sandbox_model_api_key_env="LAB_DIRECT_POLICY_KEY", # pragma: allowlist secret ) ) monkeypatch.setattr(runner, "_sandbox_metadata", lambda: {}) @@ -2206,7 +2206,9 @@ def test_model_url_reports_server_lookup_failures_and_routes_docker(monkeypatch) responses_create_params=NeMoGymResponseCreateParamsNonStreaming(input=[]), ) runner = app.LegalAgentBenchAgent.model_construct( - config=_config(), + # Bridge networking makes this assertion platform-independent: loopback + # must be rewritten for a Docker bridge on Linux and Docker Desktop. + config=_config(docker_network="bridge"), server_client=SimpleNamespace(global_config_dict={}), ) with pytest.raises(app.LegalAgentBenchConfigurationError, match="Unable to resolve policy model"): @@ -2218,7 +2220,8 @@ def test_model_url_reports_server_lookup_failures_and_routes_docker(monkeypatch) ) monkeypatch.setattr(app, "get_first_server_config_dict", lambda *_args: {}) monkeypatch.setattr(app.LegalAgentBenchAgent, "rollout_id_from_run", lambda _self, _body: None) - assert runner._model_url(body).startswith("http://host.docker.internal:8000") + monkeypatch.setattr(app.sys, "platform", "linux") + assert runner._model_url(body) == "http://host.docker.internal:8000" @pytest.mark.asyncio