diff --git a/benchmarks/legal_agent_bench/README.md b/benchmarks/legal_agent_bench/README.md index c25c1ef25e..75d7e1499d 100644 --- a/benchmarks/legal_agent_bench/README.md +++ b/benchmarks/legal_agent_bench/README.md @@ -1,29 +1,89 @@ # Legal Agent Bench Benchmark -This benchmark registers the existing +This benchmark registers the [Legal Agent Bench resource server](../../resources_servers/legal_agent_bench/README.md) -with Gym's benchmark catalog. It evaluates public Harvey LAB tasks through -the legal_agent_bench resource server's implementation. +with Gym's benchmark catalog. Every variant evaluates the same 1,749 public +Harvey LAB tasks with the same skills and verifier. -Benchmark preparation reuses -the resource server's pinned task and skill caches and writes only a small, -gitignored benchmark index. +The unqualified benchmark uses a LAB-owned, Gym-native implementation of the +upstream LAB model/tool loop. Four explicit variants run the same tasks through +Harbor or one of Gym's built-in agent harnesses. All non-Harbor choices use the +[configurable runner](../../responses_api_agents/legal_agent_bench_agent/README.md). + +| Benchmark | Agent | +| --- | --- | +| `legal_agent_bench` | LAB Gym-native loop (default) | +| `legal_agent_bench/config_harbor` | Harbor compatibility integration | +| `legal_agent_bench/config_hermes` | Hermes | +| `legal_agent_bench/config_claude_code` | Claude Code | +| `legal_agent_bench/config_codex` | Codex | + +Benchmark preparation reuses the resource server's pinned task and skill +caches and copies a small, gitignored, deterministic index. Source, example, +and benchmark JSONL rows are agent-neutral. Gym stamps the variant's configured +agent during dataset collation. ## Requirements -- Python 3.13.13 and the repository environment installed with `uv` -- Docker with a running daemon +- Python 3.13.14 and the repository environment installed with `uv` +- One supported Gym sandbox provider: Docker, ECS Fargate, Enroot, Apptainer, + OpenSandbox, Daytona, or OpenShell +- The provider's local CLI, service, credentials, and Gym dependency extra, as + documented under `nemo_gym/sandbox/providers//` - Authorized OpenAI-compatible policy and judge endpoints in the root `env.yaml` - At least 10 GB of free working space +Not required: + +- Separate Harbor, Hermes, Claude Code, or Codex installations +- Anthropic or OpenAI vendor subscriptions or CLI logins for the Claude Code + and Codex harnesses +- Docker when a non-Docker provider is selected and a compatible LAB image is + already available + +Gym provisions the pinned harness dependencies automatically. Every harness +uses the configured policy model endpoint. Access to your configured policy and +judge endpoints is still required and may itself be metered or paid. + See the [resource-server README](../../resources_servers/legal_agent_bench/README.md) for endpoint configuration, source and license details, cache locations, and troubleshooting. The initial source download is several hundred MiB, and the -first rollout builds a document-tooling Docker image that can take several -minutes. +first rollout provisions the selected harness inside a temporary sandbox. The +default Docker backend also builds the document-tooling image on first use. + +## Set up + +From the repository root: + +```bash +uv venv --python 3.13.14 +source .venv/bin/activate +uv sync --extra dev +``` + +For Docker, verify the default local backend: + +```bash +docker info >/dev/null +``` + +For SDK-backed providers, install the sandbox dependencies as well: + +```bash +uv sync --extra dev --extra sandbox +``` + +Configure the policy and judge endpoints in the gitignored root `env.yaml` as +shown in the +[resource-server README](../../resources_servers/legal_agent_bench/README.md#requirements). +Keep the virtual environment activated for every command below. -## Prepare +Use the `gym` executable from the activated environment directly. Do not prefix +these server-starting commands with `uv run`: Ray starts components from their +own working directories, which can conflict with uv's project discovery. + +## Prepare and validate From the repository root, run: @@ -31,43 +91,450 @@ From the repository root, run: gym eval prepare --benchmark legal_agent_bench ``` -This validates or prepares the shared task and skill caches, then writes the -deterministic benchmark index to +Preparation is shared by all five variants. It validates or prepares the task +and skill caches, then writes the deterministic benchmark index to `benchmarks/legal_agent_bench/data/legal_agent_bench_benchmark.jsonl`. Repeated preparation reuses valid caches and does not download a second copy of LAB. -## Run +Validate that all five configurations resolve before spending time on a +rollout: + +```bash +gym env validate --model-type vllm_model --benchmark legal_agent_bench +gym env validate --model-type vllm_model --benchmark legal_agent_bench/config_harbor +gym env validate --model-type vllm_model --benchmark legal_agent_bench/config_hermes +gym env validate --model-type vllm_model --benchmark legal_agent_bench/config_claude_code +gym env validate --model-type vllm_model --benchmark legal_agent_bench/config_codex +``` + +## Test the various harnesses -For the standard one-shot workflow: +Run these one at a time for each harness you want to test. Each command starts +the required Gym services, runs the first benchmark task, writes one JSONL +result row, and stops the services. +Alongside the path passed to `--output`, Gym writes +`_materialized_inputs.jsonl` and +`_aggregate_metrics.json`. + +Gym-native LAB loop: ```bash gym eval run \ --model-type vllm_model \ --benchmark legal_agent_bench \ --split benchmark \ - --output results/legal_agent_bench_benchmark.jsonl \ - --concurrency 1 + --output results/legal_agent_bench_native_smoke.jsonl \ + --concurrency 1 \ + --limit 1 ``` -For a one-task smoke test, add `--limit 1`. +Harbor compatibility integration: + +```bash +gym eval run \ + --model-type vllm_model \ + --benchmark legal_agent_bench/config_harbor \ + --split benchmark \ + --output results/legal_agent_bench_harbor_smoke.jsonl \ + --concurrency 1 \ + --limit 1 +``` -To manage the servers separately, start them first: +Hermes: + +```bash +gym eval run \ + --model-type vllm_model \ + --benchmark legal_agent_bench/config_hermes \ + --split benchmark \ + --output results/legal_agent_bench_hermes_smoke.jsonl \ + --concurrency 1 \ + --limit 1 +``` + +Claude Code: + +```bash +gym eval run \ + --model-type vllm_model \ + --benchmark legal_agent_bench/config_claude_code \ + --split benchmark \ + --output results/legal_agent_bench_claude_code_smoke.jsonl \ + --concurrency 1 \ + --limit 1 +``` + +Codex: + +```bash +gym eval run \ + --model-type vllm_model \ + --benchmark legal_agent_bench/config_codex \ + --split benchmark \ + --output results/legal_agent_bench_codex_smoke.jsonl \ + --concurrency 1 \ + --limit 1 +``` + +The default native loop sends LAB's canonical function tools through Gym's +Responses API and executes them directly inside the task sandbox. Hermes uses +chat completions. Claude Code and Codex use their respective CLI adapters +against the configured OpenAI-compatible policy endpoint. The first run of +each non-Harbor harness provisions its portable runtime, so it can be much +slower than a cached run. The LAB runner disables Hermes's optional pricing and +context-metadata lookups because Gym already supplies the model and its +internal policy proxy does not expose `/models`; real model-call access logging +is enabled. + +## Check a native or configurable result + +For the default native loop, inspect the result row and persisted artifact +bundle with: + +```bash +jq '{ + reward, + criteria_pass_rate, + mask_sample, + agent_failed, + model_connection_failed, + verifier_failed, + judge_error_count, + verifier_error, + artifact_dir +}' results/legal_agent_bench_native_smoke.jsonl + +ARTIFACT_DIR=$(jq -r '.artifact_dir' results/legal_agent_bench_native_smoke.jsonl) +jq . "$ARTIFACT_DIR/run_summary.json" +jq . "$ARTIFACT_DIR/agent/trajectory.json" +open "$ARTIFACT_DIR/verifier/report.html" # macOS; use xdg-open on Linux +``` + +Substitute the Hermes, Claude Code, or Codex output filename to inspect those +runs. +A reliable completed rollout has `mask_sample: false`, no failure flags, +`judge_error_count: 0`, and `verifier_error: 0`. Check `output_files` in +`run_summary.json` for the deliverables. A `reward` of `0.0` can still be a +valid run: the default `full_task` reward requires every rubric criterion to +pass, while `criteria_pass_rate` shows partial success. + +Harbor keeps its detailed trial files under +`results/legal_agent_bench/harbor_jobs`; the native and configurable variants +use the corresponding `native_jobs`, `hermes_jobs`, `claude_code_jobs`, or +`codex_jobs` directory. +Their model directory comes from `policy_model_name`, with path separators and +other unsafe characters normalized for the filesystem. A session directory is +created only when its first rollout starts. Each configurable rollout is stored +under +`_jobs///_/`, where +the task name is normalized and the run ID is an eight-character unique suffix. +Harbor retains its established response format and additionally reports +`mask_sample`, `agent_failed`, `model_connection_failed`, `agent_timed_out`, and +`failure_reason` for agent-phase failures. It preserves any partial trajectory, +but skips judging and forces reward to zero when one of those failures occurs. + +## Choose output, context, and timeout limits + +LAB does not prescribe one model-independent output-token limit. Its +[upstream adapters](https://github.com/harveyai/harvey-labs/tree/main/harness/adapters) +choose provider- and model-specific per-call limits, generally using the +model's large output capacity. For a locally hosted model, `64,000` is a +sensible starting point when the endpoint and hardware support it: + +```bash +++responses_create_params.max_output_tokens=64000 +``` + +Append that override to the native or configurable-harness command. Set the +limit as high as the model, server, total context window, and available KV +cache can sustain. A larger output reservation can reduce the input space +available within a fixed context window, so it is not always safe to use the +largest numerical value accepted by the API. If a higher limit exceeds local +memory capacity, reduce rollout and agent-server concurrency together before +lowering the limit. + +An unreasonably low output limit can stop a long tool call or reasoning turn +before the agent finishes its deliverables. A context window that is too small +can have the same effect later in a multi-turn task. LAB scores those incomplete +outcomes, so either constraint can skew model-quality results downward. Record +both limits with published results and compare models using settings that do +not prematurely truncate otherwise supported work. + +### Timeouts and turn limits + +LAB does not define one canonical wall-clock timeout for a task. Its current +upstream [runner CLI](https://github.com/harveyai/harvey-labs/blob/main/harness/run.py) +exposes a turn budget and a per-command shell timeout, but the +[agent loop](https://github.com/harveyai/harvey-labs/blob/main/harness/agent_loop.py) +has no overall deadline. Upstream +[model adapters](https://github.com/harveyai/harvey-labs/tree/main/harness/adapters) +also do not share a model-request timeout, and the +[evaluator CLI](https://github.com/harveyai/harvey-labs/blob/main/evaluation/run_eval.py) +does not expose a uniform judge timeout. Provider SDK defaults therefore +differ. + +Gym adds layered operational deadlines so a stalled endpoint or sandbox cannot +hold a rollout worker indefinitely. The checked-in defaults are the recommended +starting values for full evaluations: + +| Layer | Default | Guidance | +| --- | ---: | --- | +| Complete agent phase | 10,800 seconds (3 hours) | Use for all harnesses. Claude Code and Codex also receive this as their inner harness timeout. | +| One policy-model request | 1,800 seconds (30 minutes) | Used by the native and Harbor loops. This is intentionally generous for slow local reasoning models. | +| Native tool preflight | 120 seconds | Allows document tooling to initialize on a cold or CPU-throttled sandbox. This does not increase the timeout for normal tool calls. | +| Shell command | 60 seconds | Used by the native and Harbor loops. Hermes uses a 180-second terminal timeout. A shell timeout is returned to the agent as a tool error. | +| Sandbox staging/collection | 900 seconds (15 minutes) | Covers extraction and collection of portable runtimes and task artifacts. Remote providers should give file-transfer API requests at least the same budget. Reduce concurrency if large parallel transfers saturate the provider. | +| Complete verifier phase | 3,600 seconds (1 hour) | Covers output staging, all criterion calls, and artifact collection. | +| One judge request | 90 seconds, with one retry | Increase only when the judge endpoint is healthy but consistently needs longer than 90 seconds. | + +Start with these values. For a slow locally hosted policy model, reduce both +rollout and agent-server concurrency before changing timeouts. If successful +model generations genuinely take longer than 30 minutes, raise the policy-call +timeout and the enclosing agent-phase timeout together. Keep the outer agent +deadline comfortably above a single model call, and keep the verifier deadline +above the judge request timeout plus retries. Raising a timeout does not reserve +GPU memory; increasing output or context limits can. + +The default native and Harbor configurations use a 60-turn budget, while +Hermes uses 90 turns and the CLI harnesses use their own configured stopping +behavior. A turn limit is a model-behavior constraint, not a timeout. Current +upstream LAB `main` uses 200 turns, but this Gym integration is pinned to an +earlier LAB revision and does not silently adopt later harness changes. Record +the turn budget as well as output, context, and timeout settings with published +results. + +The common defaults are configured in +`responses_api_agents/legal_agent_bench_agent/configs/` and +`resources_servers/legal_agent_bench/configs/legal_agent_bench.yaml`. For the +default native benchmark, the relevant override paths are: + +```text ++legal_agent_bench_benchmark_native_agent.responses_api_agents.legal_agent_bench_agent.agent_timeout_seconds= ++legal_agent_bench_benchmark_native_agent.responses_api_agents.legal_agent_bench_agent.sandbox_staging_timeout_seconds= ++legal_agent_bench_benchmark_native_agent.responses_api_agents.legal_agent_bench_agent.verifier_timeout_seconds= ++legal_agent_bench_benchmark_native_agent.responses_api_agents.legal_agent_bench_agent.agent_kwargs.model_timeout_seconds= ++legal_agent_bench_benchmark_native_agent.responses_api_agents.legal_agent_bench_agent.agent_kwargs.preflight_timeout_seconds= ++legal_agent_bench_benchmark_native_agent.responses_api_agents.legal_agent_bench_agent.agent_kwargs.shell_timeout= ++legal_agent_bench_benchmark_resources_server.resources_servers.legal_agent_bench.judge_request_timeout_seconds= +``` + +Substitute the selected variant's agent and resource-server prefixes when +running Harbor, Hermes, Claude Code, or Codex. A whole-agent, policy-connection, +sandbox, or verifier timeout is an operational failure and is routed through +Gym's failure handling; it is not a completed zero-reward model outcome. + +## Run a larger evaluation + +Remove `--limit 1` from the desired smoke command. Choose a new output filename +and increase client and agent-server concurrency together only after confirming +that the sandbox backend, policy endpoint, and judge endpoint can sustain it. +For example, a two-way Hermes run uses both settings: + +```bash +gym eval run \ + --model-type vllm_model \ + --benchmark legal_agent_bench/config_hermes \ + --split benchmark \ + --output results/legal_agent_bench_hermes.jsonl \ + --concurrency 2 \ + +legal_agent_bench_benchmark_hermes_agent.responses_api_agents.legal_agent_bench_agent.concurrency=2 +``` + +Use `legal_agent_bench_benchmark_native_agent` for the default native loop, or +the corresponding `legal_agent_bench_benchmark_claude_code_agent` or +`legal_agent_bench_benchmark_codex_agent` prefix for those variants. The +agent-server default stays at `1`; changing only `--concurrency` leaves the +server-side semaphore serial. + +## Choose a sandbox provider + +The native, Hermes, Claude Code, and Codex variants use Gym's shared sandbox +API. Their runtime builder, agent phase, and isolated verifier phase all use the +same selected provider. The Harbor compatibility variant uses Harbor's own +container orchestration instead and is Docker-only. + +| Provider | LAB image | Policy-proxy routing | Provider setup | +| --- | --- | --- | --- | +| Docker (default) | Automatically built locally, or `sandbox_image` | Derived loopback URLs are translated for Docker Desktop or Linux bridge networking | Running Docker daemon | +| ECS Fargate | OCI registry image; prefer an immutable digest | Automatic Gym SSH reverse tunnel | AWS/SSM/ECR/S3 infrastructure and TCP access to the task SSH sidecar | +| Enroot | Registry/Docker URI or local `.sqsh` | Shares the orchestrator host network | `enroot` CLI | +| Apptainer | Registry/Docker URI or local `.sif` | Shares the orchestrator host network | `apptainer` CLI | +| OpenSandbox | Provider-accessible OCI image | Set a reachable proxy URL when Gym's proxy is host-local | OpenSandbox service and credentials | +| Daytona | Provider-accessible OCI image or provider-supported snapshot configuration | Set a reachable proxy URL when Gym's proxy is host-local | Daytona service and credentials | +| OpenShell | Provider-accessible OCI image | Set a reachable proxy URL when Gym's proxy is host-local | OpenShell gateway and credentials when required | + +For the checked-in ECS Fargate configuration, export `AWS_PROFILE`, +`AWS_REGION`, and `AWS_DEFAULT_REGION`. Gym discovers the reference +infrastructure from `//ecs-sandbox/config` in SSM; +`ssm_project` defaults to `harbor`. That configuration must identify the ECS +cluster, subnets, security groups, task roles, ECR repository, S3 staging +bucket, and SSH-sidecar key material. The orchestrator must be able to reach +the task SSH sidecar on TCP port `52222`. See each provider YAML and README for +its current environment variables and service-specific options. + +Non-Docker providers do not invoke the Docker CLI. Supply a provider-compatible +image through `NEMO_GYM_LAB_SANDBOX_IMAGE` or the `sandbox_image` override. The +image must contain LAB's document tooling plus `bash`, `curl`, and `tar`, and +must permit writes under `/sandbox`. The first rollout starts a short-lived runtime +builder sandbox with outbound package-download access. The resulting portable, +content-addressed runtime is reused by later rollouts. + +### Build and publish the LAB image + +After preparation, every task contains the same generated LAB image context. +The following example selects one of those contexts and uses Docker Buildx to +build and publish an OCI image to a registry that the sandbox provider can +access: + +```bash +gym eval prepare --benchmark legal_agent_bench + +LAB_IMAGE_CONTEXT="$(find \ + resources_servers/legal_agent_bench/data/cache/harbor_tasks/legal_agent_bench \ + -mindepth 2 -maxdepth 2 -type d -name environment -print -quit)" +export LAB_IMAGE="docker.io//legal-agent-bench:" + +docker login docker.io +docker buildx build \ + --platform linux/amd64 \ + --tag "$LAB_IMAGE" \ + --push \ + "$LAB_IMAGE_CONTEXT" +docker buildx imagetools inspect "$LAB_IMAGE" +``` + +Replace `linux/amd64` when the target sandbox uses another architecture. For +reproducible evaluations, copy the published digest from the inspection output +and configure the immutable reference rather than its mutable tag: + +```bash +export NEMO_GYM_LAB_SANDBOX_IMAGE="docker.io//legal-agent-bench@sha256:" +``` + +Docker Buildx is only an example image-building workflow. The image may instead +be produced and published by any preferred OCI-compatible builder, including a +CI system; Docker does not need to be installed on the machine that launches a +non-Docker LAB sandbox. That machine needs only the selected provider and an +image reference it can use. Enroot and Apptainer may also use local `.sqsh` and +`.sif` images, respectively, without publishing them to a registry. For a +private registry, configure image-pull credentials through the selected +provider rather than placing credentials in the LAB image reference. + +OpenShell's Docker compute driver additionally requires `iproute2`, a +restricted `sandbox` user and group, and a work directory writable by that +identity. Images generated by LAB preparation include these requirements. + +For OpenSandbox, LAB treats each task's declared CPU, memory, disk, and GPU as +its limits. The checked-in agent configs set `opensandbox_request_fraction: +0.25`, which requests 25% of the CPU and memory limits while retaining the full +disk and GPU requests. This request/limit split permits deliberate +oversubscription when many LAB sandboxes run concurrently. It matches the +[Mini SWE Agent 2 example](https://github.com/NVIDIA-NeMo/Gym/blob/main/responses_api_agents/mini_swe_agent_2/configs/mini_swe_agent_2.yaml), +which requests 0.5 of a 2 CPU limit and 2 GiB of an 8 GiB memory limit. It +applies to the runtime builder, agent, and verifier. Raise the fraction toward +`1.0` if your cluster needs firmer CPU or memory reservations, or set it to +`null` to disable the split. For example, add +`+legal_agent_bench_benchmark_native_agent.responses_api_agents.legal_agent_bench_agent.opensandbox_request_fraction=0.5` +to the native `gym eval run` command below. + +The standard provider YAMLs define a top-level sandbox named `sandbox`. Select +one by adding its config and pointing the LAB agent at that name. This native +example works for Docker, ECS Fargate, Enroot, Apptainer, OpenSandbox, Daytona, +or OpenShell by replacing `` and ``: + +```bash +gym eval run \ + --model-type vllm_model \ + --benchmark legal_agent_bench \ + --config nemo_gym/sandbox/providers//configs/.yaml \ + --split benchmark \ + --output results/legal_agent_bench_native__smoke.jsonl \ + --concurrency 1 \ + --limit 1 \ + +legal_agent_bench_benchmark_native_agent.responses_api_agents.legal_agent_bench_agent.sandbox_provider=sandbox \ + +legal_agent_bench_benchmark_native_agent.responses_api_agents.legal_agent_bench_agent.sandbox_image= +``` + +For Hermes, Claude Code, or Codex, replace the benchmark and the native agent +prefix with the corresponding variant prefix. `sandbox_image` is passed through +unchanged, so use the native reference expected by the selected provider rather +than converting it in LAB. + +ECS Fargate automatically tunnels the rollout-scoped Gym policy proxy. Enroot +and Apptainer can reach a host-local proxy through their shared host network. +OpenSandbox, Daytona, and OpenShell cannot reach the orchestrator's loopback +address. Prefer exposing the credential-free Gym policy proxy on a route +reachable from the sandbox, then set `NEMO_GYM_SANDBOX_MODEL_BASE_URL` or the +`sandbox_model_base_url` override. Do not put credentials in that URL. If only +a directly authenticated model endpoint is reachable, set +`NEMO_GYM_LAB_SANDBOX_MODEL_API_KEY_ENV` to the name of a launcher environment +variable containing a narrowly scoped, short-lived model key. LAB copies that +key only into the agent sandbox as `LAB_POLICY_API_KEY`; it is not serialized +into the runner configuration or supplied to the runtime builder or verifier. +The evaluated agent can read its own environment, so this fallback accepts key +exposure to untrusted agent code. Use a dedicated key with the minimum required +permissions and rotate it after the run. Every provider's verifier sandbox must +also be able to reach the configured judge endpoint. + +OpenShell should use three separate policies through +`runtime_builder_provider_options`, `agent_sandbox_provider_options`, and +`verifier_sandbox_provider_options`. Give the builder only dependency-registry +egress, the agent only policy-proxy egress, and the verifier only judge egress. +The filesystem policy must allow the image runtime and +`/opt/legal-agent-bench` read-only and `/sandbox` writable. OpenShell injects a +policy-enforcing HTTP(S) proxy; the LAB runner opts Gym's inner HTTP client into +that proxy only for this provider. In the builder policy, configure the +`registry.npmjs.org` endpoint with `protocol: rest`, `access: read-only`, +`enforcement: enforce`, and `allow_encoded_slash: true`; npm uses encoded +slashes when resolving scoped packages such as the Claude Code dependency. +See OpenShell's +[policy schema](https://docs.nvidia.com/openshell/latest/reference/policy-schema#endpoint-object). + +Benchmark variants inherit their agent configuration, so supply the complete +phase-option maps through the decoded environment settings rather than adding +nested map keys on the command line: + +```bash +export NEMO_GYM_LAB_RUNTIME_BUILDER_PROVIDER_OPTIONS='{policy: /path/to/builder-policy.yaml}' +export NEMO_GYM_LAB_AGENT_SANDBOX_PROVIDER_OPTIONS='{policy: /path/to/agent-policy.yaml}' +export NEMO_GYM_LAB_VERIFIER_SANDBOX_PROVIDER_OPTIONS='{policy: /path/to/verifier-policy.yaml}' +``` + +The values are OmegaConf mappings and default to `{}`, so they have no effect +on providers that do not need phase-specific options. + +Before spending model tokens, exercise create, upload, execute, download, and +cleanup through the same public API. For a checked-in provider YAML: + +```bash +python responses_api_agents/legal_agent_bench_agent/scripts/smoke_provider.py \ + --config nemo_gym/sandbox/providers//configs/.yaml \ + --image +``` + +You can also use `--provider ` instead of `--config ...` to smoke a +provider's constructor defaults. A passing lifecycle smoke does not test the +policy or judge endpoints; follow it with the one-task `gym eval run` above. + +## Manage servers separately + +To manage the servers separately, start the desired variant first: ```bash gym env start \ --model-type vllm_model \ - --benchmark legal_agent_bench + --benchmark legal_agent_bench/config_hermes ``` -Then run against them from a second activated terminal: +Then run against them with `--no-serve` from a second activated terminal: ```bash gym eval run --no-serve \ - --benchmark legal_agent_bench \ - --agent legal_agent_bench_benchmark_harbor_agent \ + --benchmark legal_agent_bench/config_hermes \ + --agent legal_agent_bench_benchmark_hermes_agent \ --input benchmarks/legal_agent_bench/data/legal_agent_bench_benchmark.jsonl \ - --output results/legal_agent_bench_benchmark.jsonl \ + --output results/legal_agent_bench_hermes_smoke.jsonl \ --concurrency 1 \ --limit 1 ``` @@ -84,7 +551,48 @@ pass rate instead, add this override to `gym env start` or the one-shot ``` This changes only the reported reward; it does not change the tasks, agent, or -judge criteria. +judge criteria. The command above uses the native default's resource-server +prefix. For an explicit variant, replace +`legal_agent_bench_benchmark_resources_server` with the corresponding +`legal_agent_bench_benchmark_harbor_resources_server`, +`legal_agent_bench_benchmark_hermes_resources_server`, +`legal_agent_bench_benchmark_claude_code_resources_server`, or +`legal_agent_bench_benchmark_codex_resources_server` prefix. + +For the native and configurable variants, the rollout also reports `agent_failed`, +`model_connection_failed`, `agent_timed_out`, `verifier_failed`, +`verifier_timed_out`, `sandbox_failed`, `task_failed`, +`configuration_failed`, `judge_error_count`, and `verifier_error`. +`task_failed` identifies an unsafe, unknown, incomplete, or malformed task; +`configuration_failed` identifies an invalid harness selection or missing +required pin. The runner checks policy-model connectivity from inside the +selected sandbox before starting the harness. A connectivity or harness failure +is masked and is not sent to the judge. A normal model/task result can still +receive zero reward without those flags. Infrastructure, configuration, +task-loading, or judge failures set `mask_sample`; do not treat those zeroes as +model-quality results. +The Harbor variant reports the agent-phase subset described above and applies +the same masking and no-judge behavior to those failures. + +Configurable-runner artifacts are grouped by harness, model, dated session, and +task. The default roots are `results/legal_agent_bench/native_jobs`, +`results/legal_agent_bench/hermes_jobs`, +`results/legal_agent_bench/claude_code_jobs`, and +`results/legal_agent_bench/codex_jobs`. Harbor uses +`results/legal_agent_bench/harbor_jobs`. +Set `NEMO_GYM_LAB_RESULTS_DIR` before `gym eval run` to redirect the native and +configurable artifact root, for example to VM-native storage when Gym runs in +a Linux VM over a macOS-shared checkout. The rollout JSONL paths remain the +ones supplied with `--output`. + +Each configurable output row includes `artifact_dir`, `run_summary_path`, +`agent_trace_path`, `agent_stdout_path`, `agent_stderr_path`, +`verifier_report_path`, and `output_dir`. The inspection commands above use +these paths. + +The agent-server log prints the artifact directory after the agent sandbox has +stopped and its downloaded files pass validation, then again when verification +completes or the rollout fails. ## Test @@ -93,7 +601,9 @@ Run the benchmark and resource-server tests with: ```bash uv run pytest -q \ benchmarks/legal_agent_bench/tests \ - resources_servers/legal_agent_bench/tests + resources_servers/legal_agent_bench/tests \ + responses_api_agents/legal_agent_bench_agent/tests \ + responses_api_agents/legal_agent_bench_native_agent/tests ``` Generated indexes, collation metrics, Harbor jobs, source documents, and skills diff --git a/benchmarks/legal_agent_bench/config.yaml b/benchmarks/legal_agent_bench/config.yaml index cadf64def6..ec8a4c8b32 100644 --- a/benchmarks/legal_agent_bench/config.yaml +++ b/benchmarks/legal_agent_bench/config.yaml @@ -2,20 +2,20 @@ # SPDX-License-Identifier: Apache-2.0 config_paths: - - resources_servers/legal_agent_bench/configs/legal_agent_bench.yaml + - responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_native.yaml legal_agent_bench_benchmark_resources_server: _inherit_from: legal_agent_bench -legal_agent_bench_benchmark_harbor_agent: - _inherit_from: legal_agent_bench_harbor_agent +legal_agent_bench_benchmark_native_agent: + _inherit_from: legal_agent_bench_native_agent responses_api_agents: - harbor_agent: - harbor_datasets: - legal_agent_bench: - local_dataset_path: ${legal_agent_bench_benchmark_resources_server.resources_servers.legal_agent_bench.harbor_tasks_dir} - harbor_agent_kwargs: - skills_dir: ${legal_agent_bench_benchmark_resources_server.resources_servers.legal_agent_bench.harness_skills_dir} + legal_agent_bench_agent: + resources_server: + type: resources_servers + name: legal_agent_bench_benchmark_resources_server + runtime_tasks_dir: ${legal_agent_bench_benchmark_resources_server.resources_servers.legal_agent_bench.harbor_tasks_dir} + skills_dir: ${legal_agent_bench_benchmark_resources_server.resources_servers.legal_agent_bench.harness_skills_dir} datasets: - name: legal_agent_bench type: benchmark diff --git a/benchmarks/legal_agent_bench/config_claude_code.yaml b/benchmarks/legal_agent_bench/config_claude_code.yaml new file mode 100644 index 0000000000..fe7dea3da6 --- /dev/null +++ b/benchmarks/legal_agent_bench/config_claude_code.yaml @@ -0,0 +1,25 @@ +config_paths: + - responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_claude_code.yaml + +legal_agent_bench_benchmark_claude_code_resources_server: + _inherit_from: legal_agent_bench + resources_servers: + legal_agent_bench: + description: Configurable-agent integration of Legal Agent Benchmark (LAB) using Claude Code + +legal_agent_bench_benchmark_claude_code_agent: + _inherit_from: legal_agent_bench_claude_code_agent + responses_api_agents: + legal_agent_bench_agent: + resources_server: + type: resources_servers + name: legal_agent_bench_benchmark_claude_code_resources_server + runtime_tasks_dir: ${legal_agent_bench_benchmark_claude_code_resources_server.resources_servers.legal_agent_bench.harbor_tasks_dir} + skills_dir: ${legal_agent_bench_benchmark_claude_code_resources_server.resources_servers.legal_agent_bench.harness_skills_dir} + datasets: + - name: legal_agent_bench + type: benchmark + jsonl_fpath: benchmarks/legal_agent_bench/data/legal_agent_bench_benchmark.jsonl + prompt_config: null + prepare_script: benchmarks/legal_agent_bench/prepare.py + num_repeats: 1 diff --git a/benchmarks/legal_agent_bench/config_codex.yaml b/benchmarks/legal_agent_bench/config_codex.yaml new file mode 100644 index 0000000000..5b827e2ea3 --- /dev/null +++ b/benchmarks/legal_agent_bench/config_codex.yaml @@ -0,0 +1,25 @@ +config_paths: + - responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_codex.yaml + +legal_agent_bench_benchmark_codex_resources_server: + _inherit_from: legal_agent_bench + resources_servers: + legal_agent_bench: + description: Configurable-agent integration of Legal Agent Benchmark (LAB) using Codex + +legal_agent_bench_benchmark_codex_agent: + _inherit_from: legal_agent_bench_codex_agent + responses_api_agents: + legal_agent_bench_agent: + resources_server: + type: resources_servers + name: legal_agent_bench_benchmark_codex_resources_server + runtime_tasks_dir: ${legal_agent_bench_benchmark_codex_resources_server.resources_servers.legal_agent_bench.harbor_tasks_dir} + skills_dir: ${legal_agent_bench_benchmark_codex_resources_server.resources_servers.legal_agent_bench.harness_skills_dir} + datasets: + - name: legal_agent_bench + type: benchmark + jsonl_fpath: benchmarks/legal_agent_bench/data/legal_agent_bench_benchmark.jsonl + prompt_config: null + prepare_script: benchmarks/legal_agent_bench/prepare.py + num_repeats: 1 diff --git a/benchmarks/legal_agent_bench/config_harbor.yaml b/benchmarks/legal_agent_bench/config_harbor.yaml new file mode 100644 index 0000000000..883659ebc8 --- /dev/null +++ b/benchmarks/legal_agent_bench/config_harbor.yaml @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +config_paths: + - resources_servers/legal_agent_bench/configs/legal_agent_bench.yaml + +legal_agent_bench_benchmark_harbor_resources_server: + _inherit_from: legal_agent_bench + +legal_agent_bench_benchmark_harbor_agent: + _inherit_from: legal_agent_bench_harbor_agent + responses_api_agents: + harbor_agent: + harbor_datasets: + legal_agent_bench: + local_dataset_path: ${legal_agent_bench_benchmark_harbor_resources_server.resources_servers.legal_agent_bench.harbor_tasks_dir} + harbor_agent_kwargs: + skills_dir: ${legal_agent_bench_benchmark_harbor_resources_server.resources_servers.legal_agent_bench.harness_skills_dir} + datasets: + - name: legal_agent_bench + type: benchmark + jsonl_fpath: benchmarks/legal_agent_bench/data/legal_agent_bench_benchmark.jsonl + prompt_config: null + prepare_script: benchmarks/legal_agent_bench/prepare.py + num_repeats: 1 diff --git a/benchmarks/legal_agent_bench/config_hermes.yaml b/benchmarks/legal_agent_bench/config_hermes.yaml new file mode 100644 index 0000000000..bd5d6e6ec2 --- /dev/null +++ b/benchmarks/legal_agent_bench/config_hermes.yaml @@ -0,0 +1,25 @@ +config_paths: + - responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_hermes.yaml + +legal_agent_bench_benchmark_hermes_resources_server: + _inherit_from: legal_agent_bench + resources_servers: + legal_agent_bench: + description: Configurable-agent integration of Legal Agent Benchmark (LAB) using Hermes + +legal_agent_bench_benchmark_hermes_agent: + _inherit_from: legal_agent_bench_hermes_agent + responses_api_agents: + legal_agent_bench_agent: + resources_server: + type: resources_servers + name: legal_agent_bench_benchmark_hermes_resources_server + runtime_tasks_dir: ${legal_agent_bench_benchmark_hermes_resources_server.resources_servers.legal_agent_bench.harbor_tasks_dir} + skills_dir: ${legal_agent_bench_benchmark_hermes_resources_server.resources_servers.legal_agent_bench.harness_skills_dir} + datasets: + - name: legal_agent_bench + type: benchmark + jsonl_fpath: benchmarks/legal_agent_bench/data/legal_agent_bench_benchmark.jsonl + prompt_config: null + prepare_script: benchmarks/legal_agent_bench/prepare.py + num_repeats: 1 diff --git a/benchmarks/legal_agent_bench/prepare.py b/benchmarks/legal_agent_bench/prepare.py index 6a2345a580..1a0c67cda4 100644 --- a/benchmarks/legal_agent_bench/prepare.py +++ b/benchmarks/legal_agent_bench/prepare.py @@ -15,7 +15,6 @@ BENCHMARK_DIR = Path(__file__).resolve().parent DATA_DIR = BENCHMARK_DIR / "data" OUTPUT_FPATH = DATA_DIR / "legal_agent_bench_benchmark.jsonl" -BENCHMARK_AGENT_NAME = "legal_agent_bench_benchmark_harbor_agent" def _render_benchmark_index(source_index: Path) -> str: @@ -31,10 +30,7 @@ def _render_benchmark_index(source_index: Path) -> str: if not isinstance(row, dict): raise ValueError(f"LAB task index line {line_number} must contain a JSON object") - row["agent_ref"] = { - "name": BENCHMARK_AGENT_NAME, - "type": "responses_api_agents", - } + row.pop("agent_ref", None) rendered_rows.append(json.dumps(row, ensure_ascii=False, sort_keys=True) + "\n") if len(rendered_rows) != EXPECTED_TASK_COUNT: diff --git a/benchmarks/legal_agent_bench/tests/test_prepare.py b/benchmarks/legal_agent_bench/tests/test_prepare.py index 7aecea87c2..ebb7ed4a64 100644 --- a/benchmarks/legal_agent_bench/tests/test_prepare.py +++ b/benchmarks/legal_agent_bench/tests/test_prepare.py @@ -8,11 +8,13 @@ from pathlib import Path import pytest -from omegaconf import OmegaConf +from omegaconf import DictConfig, OmegaConf from benchmarks.legal_agent_bench import prepare as benchmark_prepare from nemo_gym.benchmarks import BenchmarkConfig +from nemo_gym.config_types import ResponsesAPIAgentServerInstanceConfig from nemo_gym.global_config import GlobalConfigDictParser, GlobalConfigDictParserConfig +from nemo_gym.train_data_utils import TrainDataProcessor from resources_servers.legal_agent_bench.prepare import EXPECTED_TASK_COUNT, INDEX_FILENAME @@ -28,10 +30,6 @@ def _write_task_index(parent: Path, count: int) -> tuple[Path, list[str]]: for task_name in task_names: rows.append( { - "agent_ref": { - "name": "legal_agent_bench_harbor_agent", - "type": "responses_api_agents", - }, "instance_id": f"legal_agent_bench::{task_name}", "responses_create_params": { "input": [], @@ -73,14 +71,7 @@ def test_prepare_writes_deterministic_complete_benchmark_index(monkeypatch, tmp_ rows = [json.loads(line) for line in output_path.read_text(encoding="utf-8").splitlines()] assert len(rows) == EXPECTED_TASK_COUNT assert [row["instance_id"].split("::", 1)[1] for row in rows] == task_names - assert all( - row["agent_ref"] - == { - "name": benchmark_prepare.BENCHMARK_AGENT_NAME, - "type": "responses_api_agents", - } - for row in rows - ) + assert all("agent_ref" not in row for row in rows) def test_wrong_row_count_does_not_replace_existing_output(monkeypatch, tmp_path) -> None: @@ -129,7 +120,7 @@ def test_benchmark_config_is_isolated_and_resolves_shared_cache_paths() -> None: benchmark = BenchmarkConfig.from_config_path(CONFIG_FPATH, strict=False) assert benchmark is not None assert benchmark.name == "legal_agent_bench" - assert benchmark.agent_name == "legal_agent_bench_benchmark_harbor_agent" + assert benchmark.agent_name == "legal_agent_bench_benchmark_native_agent" assert benchmark.num_repeats == 1 assert benchmark.dataset.prompt_config is None assert benchmark.dataset.jsonl_fpath == Path("benchmarks/legal_agent_bench/data/legal_agent_bench_benchmark.jsonl") @@ -141,12 +132,170 @@ def test_benchmark_config_is_isolated_and_resolves_shared_cache_paths() -> None: ) resolved = GlobalConfigDictParser().parse_no_environment(initial_global_config_dict=initial_config) assert "legal_agent_bench" not in resolved - assert "legal_agent_bench_harbor_agent" not in resolved + assert "legal_agent_bench_native_agent" not in resolved resource = resolved.legal_agent_bench_benchmark_resources_server.resources_servers.legal_agent_bench + agent = resolved.legal_agent_bench_benchmark_native_agent.responses_api_agents.legal_agent_bench_agent + assert agent.agent_server_module == "responses_api_agents.legal_agent_bench_native_agent.app" + assert agent.runtime_tasks_dir == resource.harbor_tasks_dir + assert agent.skills_dir == resource.harness_skills_dir + assert agent.runtime_builder_provider_options == {} + assert agent.agent_sandbox_provider_options == {} + assert agent.verifier_sandbox_provider_options == {} + assert agent.agent_kwargs.max_turns == 60 + assert len(agent.datasets) == 1 + assert agent.datasets[0].type == "benchmark" + + +def test_harbor_compatibility_variant_resolves() -> None: + config_path = BENCHMARK_DIR / "config_harbor.yaml" + benchmark = BenchmarkConfig.from_config_path(config_path, strict=False) + assert benchmark is not None + assert benchmark.name == "legal_agent_bench" + assert benchmark.agent_name == "legal_agent_bench_benchmark_harbor_agent" + + initial_config = OmegaConf.merge( + OmegaConf.load(config_path), + GlobalConfigDictParserConfig.NO_MODEL_GLOBAL_CONFIG_DICT, + ) + resolved = GlobalConfigDictParser().parse_no_environment(initial_global_config_dict=initial_config) + resource = resolved.legal_agent_bench_benchmark_harbor_resources_server.resources_servers.legal_agent_bench agent = resolved.legal_agent_bench_benchmark_harbor_agent.responses_api_agents.harbor_agent assert agent.harbor_datasets.legal_agent_bench.local_dataset_path == resource.harbor_tasks_dir assert agent.harbor_agent_kwargs.skills_dir == resource.harness_skills_dir assert agent.harbor_agent_kwargs.max_turns == 60 - assert len(agent.datasets) == 1 assert agent.datasets[0].type == "benchmark" + + +@pytest.mark.parametrize( + ("filename", "expected_agent", "expected_module"), + [ + ("config_hermes.yaml", "legal_agent_bench_benchmark_hermes_agent", "responses_api_agents.hermes_agent.app"), + ( + "config_claude_code.yaml", + "legal_agent_bench_benchmark_claude_code_agent", + "responses_api_agents.claude_code_agent.app", + ), + ("config_codex.yaml", "legal_agent_bench_benchmark_codex_agent", "responses_api_agents.codex_agent.app"), + ], +) +def test_configurable_benchmark_variants_resolve(filename, expected_agent, expected_module) -> None: + config_path = BENCHMARK_DIR / filename + benchmark = BenchmarkConfig.from_config_path(config_path, strict=False) + assert benchmark is not None + assert benchmark.name == "legal_agent_bench" + assert benchmark.agent_name == expected_agent + assert benchmark.dataset.jsonl_fpath == Path("benchmarks/legal_agent_bench/data/legal_agent_bench_benchmark.jsonl") + + initial_config = OmegaConf.merge( + OmegaConf.load(config_path), + GlobalConfigDictParserConfig.NO_MODEL_GLOBAL_CONFIG_DICT, + ) + resolved = GlobalConfigDictParser().parse_no_environment(initial_global_config_dict=initial_config) + agent_names = [ + name for name, value in resolved.items() if isinstance(value, DictConfig) and "responses_api_agents" in value + ] + resource_names = [ + name for name, value in resolved.items() if isinstance(value, DictConfig) and "resources_servers" in value + ] + assert agent_names == [expected_agent] + assert resource_names == [f"{expected_agent.removesuffix('_agent')}_resources_server"] + agent = resolved[expected_agent].responses_api_agents.legal_agent_bench_agent + assert agent.agent_server_module == expected_module + if expected_module == "responses_api_agents.claude_code_agent.app": + assert agent.agent_kwargs.claude_code_version == "2.1.211" + elif expected_module == "responses_api_agents.codex_agent.app": + assert agent.agent_kwargs.codex_version == "0.144.4" + assert agent.agent_kwargs.cwd == "/sandbox/nemo-gym-legal-agent-bench/workspace/output" + assert agent.datasets[0].type == "benchmark" + assert agent.runtime_tasks_dir.endswith("data/runtime/harbor_tasks/legal_agent_bench") + assert agent.runtime_builder_provider_options == {} + assert agent.agent_sandbox_provider_options == {} + assert agent.verifier_sandbox_provider_options == {} + + +@pytest.mark.parametrize( + ("filename", "expected_agent"), + [ + ("config.yaml", "legal_agent_bench_benchmark_native_agent"), + ("config_hermes.yaml", "legal_agent_bench_benchmark_hermes_agent"), + ("config_claude_code.yaml", "legal_agent_bench_benchmark_claude_code_agent"), + ("config_codex.yaml", "legal_agent_bench_benchmark_codex_agent"), + ], +) +def test_configurable_variants_decode_phase_provider_options_from_environment( + monkeypatch, filename, expected_agent +) -> None: + monkeypatch.setenv( + "NEMO_GYM_LAB_RUNTIME_BUILDER_PROVIDER_OPTIONS", + "{policy: /tmp/lab-builder-policy.yaml}", + ) + monkeypatch.setenv( + "NEMO_GYM_LAB_AGENT_SANDBOX_PROVIDER_OPTIONS", + "{policy: /tmp/lab-agent-policy.yaml}", + ) + monkeypatch.setenv( + "NEMO_GYM_LAB_VERIFIER_SANDBOX_PROVIDER_OPTIONS", + "{policy: /tmp/lab-verifier-policy.yaml}", + ) + initial_config = OmegaConf.merge( + OmegaConf.load(BENCHMARK_DIR / filename), + GlobalConfigDictParserConfig.NO_MODEL_GLOBAL_CONFIG_DICT, + ) + + resolved = GlobalConfigDictParser().parse_no_environment(initial_global_config_dict=initial_config) + agent = resolved[expected_agent].responses_api_agents.legal_agent_bench_agent + + assert agent.runtime_builder_provider_options == {"policy": "/tmp/lab-builder-policy.yaml"} + assert agent.agent_sandbox_provider_options == {"policy": "/tmp/lab-agent-policy.yaml"} + assert agent.verifier_sandbox_provider_options == {"policy": "/tmp/lab-verifier-policy.yaml"} + + +@pytest.mark.parametrize( + "agent_name", + [ + "legal_agent_bench_benchmark_native_agent", + "legal_agent_bench_benchmark_harbor_agent", + "legal_agent_bench_benchmark_hermes_agent", + "legal_agent_bench_benchmark_claude_code_agent", + "legal_agent_bench_benchmark_codex_agent", + ], +) +def test_benchmark_collation_stamps_selected_agent_without_changing_source(tmp_path, agent_name) -> None: + source = tmp_path / "legal_agent_bench.jsonl" + neutral_row = { + "instance_id": "legal_agent_bench::corporate__task", + "responses_create_params": {"input": []}, + } + source.write_text(json.dumps(neutral_row) + "\n", encoding="utf-8") + agent_config = { + "responses_api_agents": { + "agent": { + "host": "127.0.0.1", + "port": 12345, + "entrypoint": "app.py", + "resources_server": {"type": "resources_servers", "name": "legal_agent_bench"}, + "model_server": {"type": "responses_api_models", "name": "policy_model"}, + "datasets": [ + { + "name": "legal_agent_bench", + "type": "benchmark", + "jsonl_fpath": str(source), + "prepare_script": "benchmarks/legal_agent_bench/prepare.py", + "num_repeats": 1, + } + ], + } + } + } + instance = ResponsesAPIAgentServerInstanceConfig( + name=agent_name, + server_type_config_dict=DictConfig(agent_config), + responses_api_agents=agent_config["responses_api_agents"], + ) + + prepared = TrainDataProcessor()._collate_samples_single_type("benchmark", [instance])[0] + collated_row = json.loads(prepared.read_text(encoding="utf-8")) + + assert json.loads(source.read_text(encoding="utf-8")) == neutral_row + assert collated_row["agent_ref"] == {"type": "responses_api_agents", "name": agent_name} diff --git a/nemo_gym/responses_streaming.py b/nemo_gym/responses_streaming.py index 0f9c1d9e09..85c7a703c1 100644 --- a/nemo_gym/responses_streaming.py +++ b/nemo_gym/responses_streaming.py @@ -34,6 +34,7 @@ import json import logging from copy import deepcopy +from hashlib import sha256 from typing import Any, Iterator, Optional from uuid import uuid4 @@ -105,6 +106,35 @@ def _input_message_text(item: dict[str, Any]) -> str: return "".join(parts) +def _synthetic_replay_item_id(item: dict[str, Any], prefix: str) -> str: + """Return a stable ID for a replay item whose wire representation omitted one.""" + payload = json.dumps(item, sort_keys=True, separators=(",", ":"), default=str).encode() + return f"{prefix}_{sha256(payload).hexdigest()[:24]}" + + +def _normalize_replayed_output_item(item: Any) -> Any: + """Fill optional response metadata omitted when Codex replays prior output. + + The Responses wire API accepts output items without their server-generated + IDs and accepts ``output_text`` parts without annotations when those items + are supplied as subsequent input. Gym's shared response/input models require + both fields because they also model newly generated output. Add only those + transport defaults so the conversation content survives strict validation. + """ + if not isinstance(item, dict): + return item + item_type = item.get("type") + if item_type == "reasoning" and not item.get("id"): + item["id"] = _synthetic_replay_item_id(item, "rs") + elif item_type == "message" and item.get("role") == "assistant": + if not item.get("id"): + item["id"] = _synthetic_replay_item_id(item, "msg") + for part in item.get("content") or []: + if isinstance(part, dict) and part.get("type") == "output_text": + part.setdefault("annotations", []) + return item + + def sanitize_streaming_responses_body(body: dict[str, Any]) -> tuple[dict[str, Any], NamespaceMap]: """Map a streaming-dialect request body onto the strict non-streaming params shape. @@ -131,6 +161,7 @@ def sanitize_streaming_responses_body(body: dict[str, Any]) -> tuple[dict[str, A kept_items = [] carrier_tools: list[Any] = [] for item in input_items: + item = _normalize_replayed_output_item(item) if isinstance(item, dict) and item.get("type") == "function_call" and item.get("namespace"): item["name"] = f"{item.pop('namespace')}{NAMESPACE_TOOL_DELIMITER}{item.get('name')}" # Codex's code mode ships tools inside an `additional_tools` input item instead of the diff --git a/nemo_gym/sandbox/providers/apptainer/configs/apptainer.yaml b/nemo_gym/sandbox/providers/apptainer/configs/apptainer.yaml new file mode 100644 index 0000000000..e23b5e1431 --- /dev/null +++ b/nemo_gym/sandbox/providers/apptainer/configs/apptainer.yaml @@ -0,0 +1,18 @@ +# Apptainer sandbox provider config. +# +# `sandbox` is the instance name an agent references via `sandbox_provider: +# sandbox`; the child key `apptainer` selects the provider class. Provider +# defaults use a private host staging directory mounted at `/sandbox` and run +# each sandbox as a persistent Apptainer instance. +# +# Requires the `apptainer` CLI on PATH. See ../README.md for image formats, +# fakeroot, cgroup, bind, and writable-tmpfs options. +sandbox: + default_metadata: + sandbox-api: apptainer + apptainer: + create: + # Apptainer otherwise bind-mounts the host's /tmp and home into every + # instance. Containment gives each sandbox private writable scratch + # directories and prevents state from leaking between concurrent tasks. + extra_start_args: ["--containall"] diff --git a/nemo_gym/sandbox/providers/apptainer/provider.py b/nemo_gym/sandbox/providers/apptainer/provider.py index 33d5f5a7c7..29b84a0678 100644 --- a/nemo_gym/sandbox/providers/apptainer/provider.py +++ b/nemo_gym/sandbox/providers/apptainer/provider.py @@ -42,6 +42,7 @@ ) from nemo_gym.sandbox.providers.utils import coerce_config as _coerce_config from nemo_gym.sandbox.providers.utils import path_under_mount as _path_under_mount +from nemo_gym.sandbox.providers.utils import remove_writable_tree as _remove_writable_tree LOGGER = logging.getLogger(__name__) @@ -700,7 +701,7 @@ async def close(self, handle: SandboxHandle) -> None: # Always best-effort remove the host staging dir, even if stop failed. try: - shutil.rmtree(inst.staging_dir, ignore_errors=False) + _remove_writable_tree(inst.staging_dir) except OSError as e: LOGGER.warning("failed to remove staging dir %s: %s", inst.staging_dir, e) diff --git a/nemo_gym/sandbox/providers/ecs_fargate/provider.py b/nemo_gym/sandbox/providers/ecs_fargate/provider.py index b586363940..10d8ab974e 100644 --- a/nemo_gym/sandbox/providers/ecs_fargate/provider.py +++ b/nemo_gym/sandbox/providers/ecs_fargate/provider.py @@ -94,7 +94,9 @@ def _apply_spec_overrides(cfg: engine.EcsFargateConfig, spec: SandboxSpec) -> en overrides["cpu"] = str(int(resources.cpu * 1024)) if resources.memory_mib is not None: overrides["memory"] = str(int(resources.memory_mib)) - if resources.disk_gib is not None: + # Fargate supplies 20 GiB when ephemeralStorage is omitted and accepts only + # 21-200 GiB explicitly. The implicit default satisfies smaller requests. + if resources.disk_gib is not None and resources.disk_gib > 20: overrides["ephemeral_storage_gib"] = int(resources.disk_gib) return replace(cfg, **overrides) if overrides else cfg diff --git a/nemo_gym/sandbox/providers/enroot/configs/enroot.yaml b/nemo_gym/sandbox/providers/enroot/configs/enroot.yaml index 294eae7c03..966beca72a 100644 --- a/nemo_gym/sandbox/providers/enroot/configs/enroot.yaml +++ b/nemo_gym/sandbox/providers/enroot/configs/enroot.yaml @@ -32,9 +32,8 @@ sandbox: sqsh_cache_dir: ${oc.env:NEMO_GYM_ENROOT_SQSH_CACHE,null} # Clear the Docker ENTRYPOINT so images with a non-shell entrypoint # (e.g. ENTRYPOINT ["python"]) don't wrap the init and exit immediately. - # Set to false for SWE-bench images: enroot resolves /dev/null inside the - # container namespace before /dev is mounted, so passing --rc /dev/null - # (bypass_entrypoint: true) fails with "No such file or directory". + # The provider supplies a regular pass-through command script, as + # required by Enroot's --rc contract. bypass_entrypoint: true rw: true remap_root: false diff --git a/nemo_gym/sandbox/providers/enroot/provider.py b/nemo_gym/sandbox/providers/enroot/provider.py index 01e7f92a0a..32e3e81894 100644 --- a/nemo_gym/sandbox/providers/enroot/provider.py +++ b/nemo_gym/sandbox/providers/enroot/provider.py @@ -50,6 +50,7 @@ ) from nemo_gym.sandbox.providers.utils import coerce_config as _coerce_config from nemo_gym.sandbox.providers.utils import path_under_mount as _path_under_mount +from nemo_gym.sandbox.providers.utils import remove_writable_tree as _remove_writable_tree LOGGER = logging.getLogger(__name__) @@ -59,6 +60,11 @@ # Portable init: keep the container alive without relying on `sleep infinity`, # which busybox `sleep` rejects. A shell loop works on any image with `sh`. DEFAULT_INIT_COMMAND = "while true; do sleep 86400; done" +# Enroot's command script owns execution of the argv supplied after the +# container name. Replacing an image-generated /etc/rc with an empty file would +# therefore exit successfully without launching the requested init. This +# minimal script bypasses image ENTRYPOINT/CMD behavior while preserving argv. +ENTRYPOINT_BYPASS_RC = 'if [ "$#" -gt 0 ]; then\n exec "$@"\nfi\nexec /bin/sh\n' READY_PROBE_COMMAND = ( f"printf enroot-sandbox-ready > {DEFAULT_MOUNT_POINT}/.nemo-gym-ready && printf enroot-sandbox-ready" ) @@ -114,12 +120,17 @@ def _find_container_init_pid(base_init: str, marker: str) -> int | None: concurrency. The ``enroot start`` wrapper also carries the marker but its cmdline starts with the enroot binary, not ``sh -c ``, so it is excluded. """ - for entry in os.scandir("/proc"): - if not entry.name.isdigit(): - continue - cmd = _read_proc_cmdline(int(entry.name)) - if cmd.startswith("sh -c ") and marker in cmd and base_init in cmd: - return int(entry.name) + try: + entries = os.scandir("/proc") + except OSError: + return None + with entries: + for entry in entries: + if not entry.name.isdigit(): + continue + cmd = _read_proc_cmdline(int(entry.name)) + if cmd.startswith("sh -c ") and marker in cmd and base_init in cmd: + return int(entry.name) return None @@ -492,10 +503,17 @@ async def create(self, spec: SandboxSpec) -> SandboxHandle: # (e.g. ENTRYPOINT ["python"]) don't wrap the init and exit immediately. # Enroot bakes the ENTRYPOINT into /etc/rc inside the rootfs during # `enroot create`; the only way to bypass it at start time is --rc, - # which replaces /etc/rc entirely. We pass an empty script so the init - # command (argv after the container name) runs directly. + # which replaces /etc/rc entirely. Enroot requires --rc to name a + # regular file, so /dev/null is not a valid sentinel. The replacement + # must also exec the supplied argv; an empty script exits without + # starting the init. Keep the pass-through script in the private + # per-sandbox staging directory. Enroot copies it before the init starts + # and it is removed with staging. if self._create_config.bypass_entrypoint: - argv += ["--rc", "/dev/null"] + bypass_rc_path = staging_dir / ".nemo-gym-bypass-entrypoint-rc" + bypass_rc_path.write_text(ENTRYPOINT_BYPASS_RC) + bypass_rc_path.chmod(0o400) + argv += ["--rc", str(bypass_rc_path)] argv += list(self._create_config.extra_start_args) # Tag the init with the (unique) container name so the nested-in-pyxis PID # fallback can find THIS container's init process in /proc unambiguously. The @@ -864,7 +882,7 @@ async def close(self, handle: SandboxHandle) -> None: remove_error = e try: - shutil.rmtree(instance.staging_dir, ignore_errors=False) + _remove_writable_tree(instance.staging_dir) except OSError as e: LOGGER.warning("failed to remove staging dir %s: %s", instance.staging_dir, e) diff --git a/nemo_gym/sandbox/providers/opensandbox/configs/opensandbox.yaml b/nemo_gym/sandbox/providers/opensandbox/configs/opensandbox.yaml index a389d1b00d..f9041b92e9 100644 --- a/nemo_gym/sandbox/providers/opensandbox/configs/opensandbox.yaml +++ b/nemo_gym/sandbox/providers/opensandbox/configs/opensandbox.yaml @@ -21,6 +21,8 @@ sandbox: sandbox-api: opensandbox-sdk opensandbox: connection: + # A bare host[:port] or full http(s) URL is accepted. When the URL + # includes a scheme, it must match `protocol` and must not include a path. domain: ${oc.env:OPENSANDBOX_DOMAIN,opensandbox-server.opensandbox-system.svc.cluster.local} api_key: ${oc.env:OPENSANDBOX_API_KEY} protocol: http diff --git a/nemo_gym/sandbox/providers/opensandbox/provider.py b/nemo_gym/sandbox/providers/opensandbox/provider.py index 64daa82882..821b387fae 100644 --- a/nemo_gym/sandbox/providers/opensandbox/provider.py +++ b/nemo_gym/sandbox/providers/opensandbox/provider.py @@ -115,6 +115,36 @@ class SandboxBackendUnreachableError(RuntimeError): IMAGE_PULL_POLICY_ANNOTATION_EXTENSION_KEY = "opensandbox.extensions.image-pull-policy" VALID_IMAGE_PULL_POLICIES = {"Always", "IfNotPresent", "Never"} STATUS_CODE_RE = re.compile(r"(?:status code|http)\D+(\d{3})", re.IGNORECASE) +SERVER_PROXY_API_KEY_HEADER = "OPEN-SANDBOX-API-KEY" # pragma: allowlist secret + + +def _normalize_domain_protocol(domain: str | None, protocol: str | None) -> tuple[str | None, str | None]: + """Validate bare domains and HTTP(S) service URLs without changing SDK semantics.""" + if domain is None: + return None, protocol + value = domain.strip().rstrip("/") + if not value: + raise ValueError("OpenSandbox connection domain must not be empty") + if "://" not in value: + if "/" in value: + raise ValueError("OpenSandbox connection domain must not contain a path") + return value, protocol + + parsed = urlsplit(value) + if parsed.scheme not in {"http", "https"} or not parsed.netloc: + raise ValueError("OpenSandbox connection domain URL must use http or https") + if parsed.username or parsed.password: + raise ValueError("OpenSandbox connection domain URL must not contain credentials") + if parsed.path not in {"", "/"} or parsed.query or parsed.fragment: + raise ValueError("OpenSandbox connection domain URL must not contain a path, query, or fragment") + if protocol is not None and protocol.lower() != parsed.scheme: + raise ValueError( + f"OpenSandbox connection protocol {protocol!r} conflicts with domain URL scheme {parsed.scheme!r}" + ) + # ConnectionConfig natively accepts a URL-form domain and lets its scheme + # override ``protocol``. Preserve that public SDK behavior for existing Gym + # consumers while still validating ambiguous or credential-bearing URLs. + return f"{parsed.scheme}://{parsed.netloc}", parsed.scheme def validate_image_pull_policy(image_pull_policy: str) -> str: @@ -637,14 +667,13 @@ def _connection_config( ) -> Any: _, ConnectionConfig, _, _, _ = _require_opensandbox_sdk() kwargs: dict[str, Any] = {} - if self._connection.domain is not None: - # OpenSandbox SDK 0.1.15 appends ``/v1`` directly. Normalizing here - # prevents a configured trailing slash from producing ``//v1``. - kwargs["domain"] = self._connection.domain.rstrip("/") + domain, protocol = _normalize_domain_protocol(self._connection.domain, self._connection.protocol) + if domain is not None: + kwargs["domain"] = domain if self._connection.api_key is not None: kwargs["api_key"] = self._connection.api_key - if self._connection.protocol is not None: - kwargs["protocol"] = self._connection.protocol + if protocol is not None: + kwargs["protocol"] = protocol if request_timeout_s is None: request_timeout_s = self._connection.request_timeout_s if request_timeout_s is not None: @@ -658,7 +687,7 @@ def _connection_config( # the key only in proxy mode: a direct sandbox endpoint runs # untrusted code and must never see it. if self._connection.api_key is not None: - kwargs["headers"] = {"OPEN-SANDBOX-API-KEY": self._connection.api_key} + kwargs["headers"] = {SERVER_PROXY_API_KEY_HEADER: self._connection.api_key} if self._connection.keepalive_expiry_s is not None or self._connection.disable_connection_pooling: kwargs["transport"] = self._get_transport() return ConnectionConfig(**kwargs) diff --git a/nemo_gym/sandbox/providers/openshell/README.md b/nemo_gym/sandbox/providers/openshell/README.md index 32cfd806b9..b5f43c45d4 100644 --- a/nemo_gym/sandbox/providers/openshell/README.md +++ b/nemo_gym/sandbox/providers/openshell/README.md @@ -12,6 +12,14 @@ from identical connection configs share one gRPC channel and one pool, so per-sa instances (the `AsyncSandbox` pattern) do not multiply threads or channels. Sandboxes live in the gateway workspace set by `connection.workspace` (`default` unless overridden). +OpenShell is an alpha API. Keep the CLI, Python SDK, and gateway protocol on the +same release. Gym supports both the legacy v0.0.36 lifecycle API and the +workspace-aware API introduced in v0.0.92, but an SDK cannot decode responses +from a gateway running an incompatible protocol version. Check the CLI with +`openshell --version`, the Python SDK with +`python -c 'import openshell; print(openshell.__version__)'`, and the gateway's +deployment metadata before running Gym. + ## Local quickstart (Docker compute driver) Run a local gateway with OpenShell's compose setup, which uses prebuilt GHCR images and a diff --git a/nemo_gym/sandbox/providers/openshell/configs/openshell.yaml b/nemo_gym/sandbox/providers/openshell/configs/openshell.yaml index 4a6e43ce29..00ae356566 100644 --- a/nemo_gym/sandbox/providers/openshell/configs/openshell.yaml +++ b/nemo_gym/sandbox/providers/openshell/configs/openshell.yaml @@ -28,10 +28,12 @@ sandbox: workspace: ${oc.env:OPENSHELL_WORKSPACE,default} # OIDC bearer token for authenticated gateways; null for the local compose gateway. bearer_token: ${oc.env:OPENSHELL_BEARER_TOKEN,null} - # mTLS material for TLS gateways (ca / cert+key paths); all null -> plaintext channel. - # tls_ca_path: /path/to/ca.pem - # tls_cert_path: /path/to/client.pem - # tls_key_path: /path/to/client-key.pem + # mTLS material for TLS gateways; all null selects a plaintext channel. + # `openshell gateway start` writes these files below its gateway metadata + # directory, while a compose-managed plaintext gateway leaves them unset. + tls_ca_path: ${oc.env:OPENSHELL_TLS_CA_PATH,null} + tls_cert_path: ${oc.env:OPENSHELL_TLS_CERT_PATH,null} + tls_key_path: ${oc.env:OPENSHELL_TLS_KEY_PATH,null} request_timeout_s: 30 create: # Provisioning can include an image pull on the gateway host. diff --git a/nemo_gym/sandbox/providers/openshell/provider.py b/nemo_gym/sandbox/providers/openshell/provider.py index cf64fe8e9a..910cb56690 100644 --- a/nemo_gym/sandbox/providers/openshell/provider.py +++ b/nemo_gym/sandbox/providers/openshell/provider.py @@ -27,6 +27,7 @@ import base64 import binascii import functools +import inspect import logging import math import posixpath @@ -51,7 +52,12 @@ LOGGER = logging.getLogger(__name__) -SANDBOX_NAME_PREFIX = "nemo-gym-" +# OpenShell 0.0.92 limits sandbox names to 19 characters. Preserve a readable +# Gym prefix and use 64 random bits; at one million generated names the birthday +# collision probability remains below 3e-8. +SANDBOX_NAME_PREFIX = "ng-" +SANDBOX_NAME_RANDOM_HEX = 16 +MAX_SANDBOX_NAME_LENGTH = 19 SANDBOX_LABEL = "nemo-gym.sandbox" READY_PROBE_COMMAND = "printf openshell-sandbox-ready" READY_PROBE_EXPECTED = "openshell-sandbox-ready" @@ -95,6 +101,12 @@ def _normalize_image(image: str) -> str: return image[len(prefix) :] if image.startswith(prefix) else image +def _new_sandbox_name() -> str: + name = SANDBOX_NAME_PREFIX + uuid.uuid4().hex[:SANDBOX_NAME_RANDOM_HEX] + assert len(name) <= MAX_SANDBOX_NAME_LENGTH + return name + + @functools.cache def _phase_to_status_map() -> dict[int, SandboxStatus]: """SandboxPhase -> SandboxStatus, built from the SDK's generated proto constants.""" @@ -357,12 +369,15 @@ def _build_client(connection: OpenShellConnectionConfig) -> Any: cert_path=Path(connection.tls_cert_path) if connection.tls_cert_path else None, key_path=Path(connection.tls_key_path) if connection.tls_key_path else None, ) - return SandboxClient( - connection.endpoint, - tls=tls, - bearer_token=connection.bearer_token, - timeout=connection.request_timeout_s, - ) + kwargs: dict[str, Any] = {"tls": tls, "timeout": connection.request_timeout_s} + parameters = inspect.signature(SandboxClient).parameters + if connection.bearer_token is not None: + if "bearer_token" not in parameters: + raise ValueError( + "connection.bearer_token requires an OpenShell SDK whose SandboxClient supports bearer tokens" + ) + kwargs["bearer_token"] = connection.bearer_token + return SandboxClient(connection.endpoint, **kwargs) def _acquire_shared_client(connection: OpenShellConnectionConfig, concurrency: int) -> _SharedClientState: @@ -414,6 +429,11 @@ def __init__( self._shared = _acquire_shared_client(self._connection, self._exec_config.concurrency) self._closed = False + @property + def _workspace_api(self) -> bool: + """Whether lifecycle calls use the workspace-aware OpenShell API.""" + return "workspace" in inspect.signature(self._client.create).parameters + @property def _client(self) -> Any: return self._shared.client @@ -445,11 +465,16 @@ def _build_sandbox_spec(self, spec: SandboxSpec, image: str | None, options: Ope kwargs: dict[str, Any] = {"environment": {str(k): str(v) for k, v in spec.env.items()}} if image or options.template_resources or options.driver_config: template = openshell_pb2.SandboxTemplate() + template_fields = template.DESCRIPTOR.fields_by_name if image: template.image = image if options.template_resources: template.resources.update(options.template_resources) if options.driver_config: + if "driver_config" not in template_fields: + raise OpenShellCreateError( + "provider_options['driver_config'] requires the workspace-aware OpenShell API" + ) template.driver_config.update(options.driver_config) kwargs["template"] = template if options.policy is not None: @@ -458,9 +483,18 @@ def _build_sandbox_spec(self, spec: SandboxSpec, image: str | None, options: Ope kwargs["providers"] = options.providers resources = spec.resources if resources.gpu: - kwargs["resource_requirements"] = openshell_pb2.ResourceRequirements( - gpu=openshell_pb2.GpuResourceRequirements(count=resources.gpu) - ) + spec_fields = openshell_pb2.SandboxSpec.DESCRIPTOR.fields_by_name + if "resource_requirements" in spec_fields: + kwargs["resource_requirements"] = openshell_pb2.ResourceRequirements( + gpu=openshell_pb2.GpuResourceRequirements(count=resources.gpu) + ) + elif resources.gpu == 1 and "gpu" in spec_fields: + kwargs["gpu"] = True + else: + raise OpenShellCreateError( + "This OpenShell API can request at most one unspecified GPU; upgrade the gateway and SDK " + "to request an exact GPU count" + ) ignored = [ key for key, value in ( @@ -479,6 +513,20 @@ def _build_sandbox_spec(self, spec: SandboxSpec, image: str | None, options: Ope ) return openshell_pb2.SandboxSpec(**kwargs) + async def _get_sandbox(self, name: str, workspace: str) -> Any: + if self._workspace_api: + return await self._call(self._client.get, name, workspace=workspace) + if workspace != "default": + raise OpenShellCreateError("This OpenShell API does not support non-default workspaces") + return await self._call(self._client.get, name) + + async def _delete_sandbox(self, name: str, workspace: str) -> Any: + if self._workspace_api: + return await self._call(self._client.delete, name, workspace=workspace) + if workspace != "default": + raise OpenShellCreateError("This OpenShell API does not support non-default workspaces") + return await self._call(self._client.delete, name) + async def _create_sandbox_with_retries(self, pb_spec: Any, name: str, labels: dict[str, str]) -> Any: """Issue CreateSandbox, retrying transient gRPC failures with the same name. @@ -491,12 +539,25 @@ async def _create_sandbox_with_retries(self, pb_spec: Any, name: str, labels: di attempt = 0 while True: try: - return await self._call( - self._client.create, workspace=workspace, spec=pb_spec, name=name, labels=labels - ) + if self._workspace_api: + return await self._call( + self._client.create, workspace=workspace, spec=pb_spec, name=name, labels=labels + ) + if workspace != "default": + raise OpenShellCreateError("This OpenShell API does not support non-default workspaces") + if labels: + LOGGER.warning( + "This OpenShell API does not support sandbox labels; LAB attribution metadata is omitted." + ) + return await self._call(self._client.create, spec=pb_spec) except Exception as e: if _is_already_exists(e): - return await self._call(self._client.get, name, workspace=workspace) + return await self._get_sandbox(name, workspace) + # The legacy API lets only the gateway choose the name. Retrying + # after an ambiguous transport failure could create a second + # sandbox that the client cannot identify or clean up. + if not self._workspace_api: + raise if attempt >= cfg.retries or not _is_retryable_create_error(e): raise attempt += 1 @@ -529,7 +590,7 @@ async def create(self, spec: SandboxSpec) -> SandboxHandle: image = _normalize_image(spec.image) if spec.image else None options = OpenShellProviderOptions.from_mapping(spec.provider_options) pb_spec = self._build_sandbox_spec(spec, image, options) - name = SANDBOX_NAME_PREFIX + uuid.uuid4().hex + name = _new_sandbox_name() # Marker label goes last so user metadata cannot clobber it. labels = {**{str(k): str(v) for k, v in spec.metadata.items()}, SANDBOX_LABEL: "1"} @@ -570,7 +631,7 @@ async def _wait_ready(self, handle: SandboxHandle, *, timeout_s: int | float) -> last_phase: int | None = None while True: try: - ref = await self._call(self._client.get, inst.name, workspace=inst.workspace) + ref = await self._get_sandbox(inst.name, inst.workspace) last_phase = ref.phase except Exception as e: if not _is_runtime_failure(e): @@ -625,7 +686,7 @@ async def _verify_created_handle(self, handle: SandboxHandle) -> None: async def _cleanup_failed_create_handle(self, handle: SandboxHandle) -> None: inst = handle.raw try: - await self._call(self._client.delete, inst.name, workspace=inst.workspace) + await self._delete_sandbox(inst.name, inst.workspace) except Exception as e: LOGGER.warning( f"Failed to delete half-created OpenShell sandbox {inst.name!r}; it may be leaked on the gateway: {e}" @@ -731,7 +792,7 @@ async def status(self, handle: SandboxHandle) -> SandboxStatus: """Sandbox phase via GetSandbox (missing -> STOPPED; RPC failure -> UNKNOWN).""" inst = handle.raw try: - ref = await self._call(self._client.get, inst.name, workspace=inst.workspace) + ref = await self._get_sandbox(inst.name, inst.workspace) except Exception as e: if _is_not_found(e): return SandboxStatus.STOPPED @@ -744,7 +805,7 @@ async def close(self, handle: SandboxHandle) -> None: """Delete the sandbox (already-gone counts as success), then wait until it is fully gone.""" inst = handle.raw try: - deleted = await self._call(self._client.delete, inst.name, workspace=inst.workspace) + deleted = await self._delete_sandbox(inst.name, inst.workspace) except Exception as e: if _is_not_found(e): return @@ -763,7 +824,7 @@ async def _wait_deleted(self, inst: _OpenShellSandbox) -> None: deadline = loop.time() + self._operations.close_timeout_s while True: try: - await self._call(self._client.get, inst.name, workspace=inst.workspace) + await self._get_sandbox(inst.name, inst.workspace) except Exception as e: if _is_not_found(e): return diff --git a/nemo_gym/sandbox/providers/utils.py b/nemo_gym/sandbox/providers/utils.py index bc68b6eabe..1787424a6b 100644 --- a/nemo_gym/sandbox/providers/utils.py +++ b/nemo_gym/sandbox/providers/utils.py @@ -18,8 +18,12 @@ opensandbox providers. They live here so there is a single implementation. """ +import os import posixpath +import shutil +import stat from collections.abc import Mapping +from pathlib import Path from typing import Any @@ -50,3 +54,19 @@ def path_under_mount(mount_point: str, path: str) -> str | None: if mp == "/": return normalized.lstrip("/") return normalized[len(mp) + 1 :] + + +def remove_writable_tree(path: Path) -> None: + """Remove a provider-owned staging tree even when sandbox files are read-only. + + Archive extraction can intentionally remove write permission from files and + directories exposed to an agent. Restore only the host owner's permissions + when removal encounters such an entry; never follow a symlink. + """ + + for root, directories, _files in os.walk(path, topdown=True, followlinks=False): + for candidate in (Path(root), *(Path(root) / name for name in directories)): + mode = candidate.lstat().st_mode + if not stat.S_ISLNK(mode): + candidate.chmod(stat.S_IMODE(mode) | stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) + shutil.rmtree(path) diff --git a/nemo_gym/server_utils.py b/nemo_gym/server_utils.py index c2ba7a674b..140259f9c0 100644 --- a/nemo_gym/server_utils.py +++ b/nemo_gym/server_utils.py @@ -83,6 +83,10 @@ class GlobalAIOHTTPAsyncClientConfig(BaseModel): global_aiohttp_connector_limit_per_host: int = 1024 global_aiohttp_client_request_debug: bool = False + # Disabled by default so ordinary Gym traffic cannot be silently redirected by + # ambient proxy variables. Sandboxed runtimes whose provider deliberately + # exposes policy-enforced egress through HTTP(S)_PROXY may opt in. + global_aiohttp_trust_env: bool = False global_aiohttp_tcp_keepalive_idle_seconds: int = Field( default=60, @@ -157,6 +161,7 @@ def set_global_aiohttp_client(cfg: GlobalAIOHTTPAsyncClientConfig) -> ClientSess ), timeout=ClientTimeout(), cookie_jar=DummyCookieJar(), + trust_env=cfg.global_aiohttp_trust_env, ) global _GLOBAL_AIOHTTP_CLIENT diff --git a/resources_servers/legal_agent_bench/README.md b/resources_servers/legal_agent_bench/README.md index 4ff4a739f2..2b5b14eb68 100644 --- a/resources_servers/legal_agent_bench/README.md +++ b/resources_servers/legal_agent_bench/README.md @@ -1,26 +1,44 @@ # Legal Agent Bench -This resource server runs the public +This resource server runs the public Harvey [Legal Agent Benchmark (LAB)](https://github.com/harveyai/harvey-labs/tree/f46ef86e4788545622db25dcffa3aebb7a139929) -through NeMo Gym and Harbor. The integration is pinned to upstream commit +through NeMo Gym. The benchmark default is a direct Gym-native implementation +of LAB's model/tool loop. Compatibility configurations run Harbor or Gym's +Hermes, Claude Code, and Codex harnesses through the same LAB-owned sandbox +runner. The integration is pinned to upstream commit `f46ef86e4788545622db25dcffa3aebb7a139929`: 1,749 tasks and the public `docx`, `pptx`, and `xlsx` skills. -NeMo Gym schedules rollouts, the custom Harbor agent works with each task's -documents inside Docker, and the task-local verifier scores every rubric -criterion with an OpenAI-compatible judge model. +NeMo Gym schedules rollouts, the selected agent works with each task's +documents inside a sandbox, and the task-local verifier scores every rubric +criterion with an OpenAI-compatible judge model. `--benchmark legal_agent_bench` +selects the native loop; Harbor is available explicitly as +`legal_agent_bench/config_harbor`. ## Requirements -- Python 3.13.13 and [uv](https://docs.astral.sh/uv/) -- Docker with a running daemon (the only supported container backend) +- Python 3.13.14 and [uv](https://docs.astral.sh/uv/) +- One Gym sandbox backend for the native/configurable runner: Docker, ECS + Fargate, Enroot, Apptainer, OpenSandbox, Daytona, or OpenShell. Docker is the + zero-configuration default; the provider matrix is in the benchmark README. +- Docker specifically when using the separate Harbor compatibility variant - An OpenAI-compatible policy endpoint and judge endpoint - At least 10 GB of free working space for preparation and the first Docker build +Not required: + +- Separate Harbor, Hermes, Claude Code, or Codex installations +- Anthropic or OpenAI vendor subscriptions or CLI logins for the Claude Code + and Codex harnesses + +Gym provisions the pinned harness dependencies automatically. Every harness +uses the configured policy model endpoint. Access to the configured policy and +judge endpoints is still required and may itself be metered or paid. + The pinned source download is about 579 MiB; allow a few GiB of free working -space during preparation. The first task also builds a -document-tooling Docker image and can take several minutes. Later tasks reuse -Docker layers. +space during preparation. With the default Docker provider, the first task also +builds a document-tooling image and can take several minutes. Other providers +reuse a supplied compatible image. From a fresh clone, create the repository environment: @@ -28,10 +46,18 @@ From a fresh clone, create the repository environment: uv venv --python 3.13.14 source .venv/bin/activate uv sync --extra dev -docker info >/dev/null ``` -Add endpoint settings to the gitignored root `env.yaml`: +Run `docker info` when using Docker or Harbor. Install the repository's +`sandbox` extra and complete the provider-specific setup when using an +SDK-backed provider. + +Keep this environment activated when running LAB commands. Invoke `gym` +directly rather than `uv run gym` for workflows that start servers: Ray starts +components from their own working directories, which can conflict with uv's +project discovery. + +Create a `env.yaml` file in the root `Gym/` directory and add your endpoint settings (note: `env.yaml` is .gitignored): ```yaml policy_base_url: https://your-policy-endpoint.example/v1 @@ -41,10 +67,13 @@ policy_model_name: your-policy-model judge_base_url: https://your-judge-endpoint.example/v1 judge_api_key: your-judge-key judge_model_name: your-judge-model +judge_reasoning_effort: medium # optional; only for judges that support it ``` The judge credentials are injected only into the regenerated, gitignored -runtime task tree. They are never written into the cache. +runtime task tree. They are never written into the cache. The configurable +runner does not mount rubric files or pass judge credentials during the agent +phase; both are staged only after the agent exits. ## Prepare explicitly (recommended) @@ -57,7 +86,7 @@ python resources_servers/legal_agent_bench/prepare.py The command downloads the pinned LAB source archive from GitHub with retries and visible progress, verifies SHA-256 `e45cbdf3236b22866e034bcc62fb23bf00ef2f2e49db7a0cd8a4b07dbae9212c`, -rejects unsafe archive entries, generates deterministic Harbor tasks, and +rejects unsafe archive entries, generates deterministic runtime tasks, and builds each cache in staging before replacing the previous valid cache. A handled preparation failure leaves the previous cache in place. @@ -84,7 +113,7 @@ gym dataset collate \ --mode example_validation ``` -Preparation generates the full 1,749-row task index inside the task cache. Prepare the assets before collating the full validation dataset: +Preparation using `prepare.py` generates the full 1,749-row task index inside the task cache. Prepare the assets before collating the full validation dataset: ```bash python resources_servers/legal_agent_bench/prepare.py @@ -108,7 +137,14 @@ gym env test --resources-server legal_agent_bench ## Run and smoke test -Start the servers: +For the shortest copy-paste path that prepares and tests the native loop, +Harbor, Hermes, Claude Code, and Codex as benchmark variants, use the +[benchmark README](../../benchmarks/legal_agent_bench/README.md#test-the-various-harnesses). +The commands below describe the lower-level resource-server and standalone +agent workflow. + +The resource-server discovery config predates the benchmark wrapper and starts +the Docker-only Harbor compatibility agent: ```bash gym env start \ @@ -131,6 +167,44 @@ gym eval run --no-serve \ --limit 1 ``` +The generated and example JSONL files do not contain `agent_ref`. Direct runs +must therefore specify the desired agent. The available standalone +configurations and agent names are: + +| Config path | Direct-run agent | +| --- | --- | +| `resources_servers/legal_agent_bench/configs/legal_agent_bench.yaml` | `legal_agent_bench_harbor_agent` | +| `responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_native.yaml` | `legal_agent_bench_native_agent` | +| `responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_hermes.yaml` | `legal_agent_bench_hermes_agent` | +| `responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_claude_code.yaml` | `legal_agent_bench_claude_code_agent` | +| `responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_codex.yaml` | `legal_agent_bench_codex_agent` | + +The configurable files include +`resources_servers/legal_agent_bench/configs/resources_only.yaml`, which starts +the LAB resource server without also launching the compatibility Harbor agent. + +For example, start and run the Gym-native loop with: + +```bash +gym env start \ + --config responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_native.yaml \ + --model-type vllm_model + +gym eval run --no-serve \ + --config responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_native.yaml \ + --agent legal_agent_bench_native_agent \ + --input resources_servers/legal_agent_bench/data/example.jsonl \ + --output results/legal_agent_bench_native_smoke.jsonl \ + --concurrency 1 \ + --limit 1 +``` + +Use the corresponding config and agent name for Hermes, Claude Code, or Codex. +All four non-Harbor choices use the configured Gym policy endpoint. That +endpoint must support the selected harness's protocol. Their runtime +dependencies are provisioned into a portable cache on the first rollout; CLI +dependencies are pinned where applicable. + The default `full_task` reward is LAB's official all-criteria score: a task earns `1.0` only when every criterion passes. For diagnostic partial credit, start with: @@ -142,8 +216,8 @@ gym env start \ +legal_agent_bench.resources_servers.legal_agent_bench.reward_mode=criteria_pass_rate ``` -The verifier evaluates up to six criteria concurrently, matching the upstream LAB -default. Adjust the resource setting when the judge endpoint has a lower +The verifier evaluates up to six criteria concurrently by default. +Adjust the resource setting when the judge endpoint has a lower concurrency limit: ```bash @@ -156,10 +230,32 @@ gym env start \ This setting is forwarded to each task verifier as `LAB_JUDGE_PARALLELISM`. -The task container requires network access because its verifier calls the -configured judge endpoint. The agent and verifier share that container, so the -agent also has network access during a rollout. This differs from the upstream LAB -closed-network reference sandbox and should be recorded when comparing runs. +The selected agent sandbox requires access to Gym's policy proxy, and the +separate verifier sandbox requires access to the configured judge endpoint. +See the benchmark README for provider-specific proxy routing. + +### Output, context, and timeout limits + +LAB does not define one model-independent output-token limit. Upstream harness +adapters use provider- and model-specific per-call caps, typically near each +model's supported output capacity. For locally hosted policy models, start with +`++responses_create_params.max_output_tokens=64000` when the endpoint, total +context window, and available KV cache support it. See +[Choose output, context, and timeout limits](../../benchmarks/legal_agent_bench/README.md#choose-output-context-and-timeout-limits) +for the full guidance and concurrency tradeoff. + +Avoid an unnecessarily low output cap or context window: either can truncate a +valid long-running agent trajectory, and LAB will score that incomplete result. +Record the limits used when reporting benchmark scores. + +LAB does not specify one whole-task wall-clock timeout. Gym's recommended +defaults are a 3-hour agent phase, a 30-minute policy request for the native and +Harbor loops, a 1-hour verifier phase, and a 90-second judge request with one +retry. Native and Harbor shell commands use 60 seconds; Hermes terminal calls +use 180 seconds. These are operational safety limits rather than LAB scoring +parameters. See +[Timeouts and turn limits](../../benchmarks/legal_agent_bench/README.md#timeouts-and-turn-limits) +for the complete table, override paths, and guidance for slow local models. ## Caches and outputs @@ -169,6 +265,11 @@ The default paths are: - Public skills: `data/cache/harness/skills` - Credential-bearing runtime tasks: `data/runtime/harbor_tasks/legal_agent_bench` - Harbor jobs: `results/legal_agent_bench/harbor_jobs` +- Configurable agent runtimes: `responses_api_agents/legal_agent_bench_agent/.deps` +- Gym-native rollout artifacts: `results/legal_agent_bench/native_jobs` +- Hermes rollout artifacts: `results/legal_agent_bench/hermes_jobs` +- Claude Code rollout artifacts: `results/legal_agent_bench/claude_code_jobs` +- Codex rollout artifacts: `results/legal_agent_bench/codex_jobs` - Rollout output: the path passed to `gym eval run` The runtime tree hardlinks immutable documents from the cache when the @@ -181,17 +282,83 @@ Each successful Harbor trial contains `result.json`, `verifier/reward.json`, and `agent/artifacts/lab-run/transcript.jsonl`. The agent config artifact should list exactly `docx`, `pptx`, and `xlsx`. +Each configurable trial contains the inner Gym trajectory, agent stdout and +stderr, LAB `config.json` and `metrics.json`, completed deliverables under +`agent/artifacts/lab-run/output`, downloaded verifier artifacts, and a compact +top-level `run_summary.json`. The rollout JSONL row exposes direct paths to the +summary, trajectory, stdout, stderr, output directory, and verifier report. +Trials are grouped as +`_jobs///_` so runs +are browsable and safe under concurrent execution. `` is the configured +`policy_model_name`, normalized into one safe path segment. The task name is +normalized, and the run ID is an eight-character unique suffix. Starting or +validating an agent server does not create an empty session directory; the +directory is created by its first rollout. +Docker can build a content-addressed image from the task environment +automatically. Other providers use the configured image reference unchanged. +The selected provider is also used to build the portable harness runtime and is +reused for the agent and verifier phases. Agent and verifier phases use +separate sandboxes: the agent +sees only its selected Gym package and public task inputs, while the verifier +starts after the agent sandbox is destroyed and receives the completed LAB run +read-only. + +To inspect a native or configurable smoke run: + +```bash +ARTIFACT_DIR=$(jq -r '.artifact_dir' results/legal_agent_bench_native_smoke.jsonl) +jq . "$ARTIFACT_DIR/run_summary.json" +jq . "$ARTIFACT_DIR/agent/trajectory.json" +open "$ARTIFACT_DIR/verifier/report.html" # macOS; use xdg-open on Linux +``` + +In `run_summary.json`, a reliable scored rollout has `mask_sample: false`, all +failure flags false, `judge_error_count: 0`, and `verifier_error: 0`. This +includes incomplete max-turn or context-limit outcomes: their partial output is +still judged and saved in the main rollout JSONL. +`output_files` lists the deliverables and `agent/trajectory.json` is the inner +harness trace. A zero `full_task` reward can still be a valid rollout when one +or more criteria fail; use `criteria_pass_rate` to see partial success. + ## Troubleshooting -- A checksum, corrupt archive, unsafe path, wrong task count, or missing skill - fails before replacing an existing valid cache. - A missing judge setting produces a verifier error in the trial artifacts. Confirm the endpoint permits the exact `judge_model_name`. - Treat a nonzero `judge_error_count` or `verifier_error` as a judge or infrastructure failure, not an ordinary model failure, even though Harbor receives a numeric zero reward so it can preserve a complete trial result. -- If Docker appears idle on the first rollout, inspect `docker ps` and the - `gym env start` terminal; Harbor is normally building the task image. +- Configurable rollouts additionally expose `agent_failed`, + `model_connection_failed`, `agent_timed_out`, `verifier_failed`, + `verifier_timed_out`, `sandbox_failed`, `task_failed`, + `configuration_failed`, and `mask_sample`. Before the harness starts, the + runner checks the policy endpoint from inside the selected sandbox. Docker + translates derived loopback URLs when needed, ECS Fargate creates a reverse + tunnel, Enroot and Apptainer share the host network, and remote providers need + an explicitly reachable `sandbox_model_base_url` when Gym's proxy is + host-local. Prefer a credential-free reachable proxy. If a direct endpoint is + the only option, the configurable runner supports an agent-only key through + `sandbox_model_api_key_env`; use a narrowly scoped, short-lived key because + the evaluated agent can read its own environment. + Connectivity, harness, sandbox, and verifier failures are masked, skip + judging when applicable, and carry `_ng_failure_class`, which routes them to + the failure sidecar for bounded retry. Task-loading and harness-configuration + failures additionally carry `_ng_failure_terminal: true`, so they are not + retried. `mask_sample` is a training hint, not the routing signal. A zero + reward without an infrastructure/judge flag is an ordinary model/task result; + a flagged result should be excluded from model-quality comparisons. +- Harbor agent, adapter, connection, and timeout failures preserve any partial + trajectory but skip judging. Their result rows report `mask_sample`, + `agent_failed`, `model_connection_failed`, `agent_timed_out`, and + `failure_reason`, carry `_ng_failure_class`, and have their reward forced to + zero. Harbor context-limit and max-turn stops are instead judged as valid + incomplete outcomes. +- Hermes normally probes `/v1/models` and `/models` for optional pricing and + context metadata. The LAB runner disables those lookups because Gym supplies + the model explicitly and its internal policy proxy does not implement model + discovery. This does not suppress access logs for actual chat-completion + requests. +- If Docker appears idle on the first Harbor rollout, inspect `docker ps` and + the `gym env start` terminal; Harbor is normally building the task image. - Do not copy or publish `data/runtime/`: it can contain local judge credentials. - Results are revision-specific and should not be compared directly with runs that use a different task snapshot or skill set. diff --git a/resources_servers/legal_agent_bench/app.py b/resources_servers/legal_agent_bench/app.py index 88ea1fbecd..53abef7b10 100644 --- a/resources_servers/legal_agent_bench/app.py +++ b/resources_servers/legal_agent_bench/app.py @@ -29,6 +29,7 @@ "judge_base_url": "LAB_JUDGE_BASE_URL", "judge_api_key": "LAB_JUDGE_API_KEY", # pragma: allowlist secret "judge_model_name": "LAB_JUDGE_MODEL", + "judge_reasoning_effort": "LAB_JUDGE_REASONING_EFFORT", "judge_temperature": "LAB_JUDGE_TEMPERATURE", "judge_request_timeout_seconds": "LAB_JUDGE_REQUEST_TIMEOUT_SECONDS", "judge_max_retries": "LAB_JUDGE_MAX_RETRIES", @@ -51,6 +52,10 @@ class LegalAgentBenchResourcesServerConfig(BaseResourcesServerConfig): judge_base_url: str | None = Field(default=None, description="OpenAI-compatible base URL for the LAB judge.") judge_api_key: str | None = Field(default=None, description="API key for the LAB judge endpoint.") judge_model_name: str | None = Field(default=None, description="Model identifier sent to the LAB judge endpoint.") + judge_reasoning_effort: Literal["minimal", "low", "medium", "high"] | None = Field( + default=None, + description="Optional reasoning effort sent to compatible LAB judge endpoints.", + ) judge_temperature: float | None = Field( default=None, ge=0, diff --git a/resources_servers/legal_agent_bench/configs/legal_agent_bench.yaml b/resources_servers/legal_agent_bench/configs/legal_agent_bench.yaml index 3344f78106..a484b5d7ce 100644 --- a/resources_servers/legal_agent_bench/configs/legal_agent_bench.yaml +++ b/resources_servers/legal_agent_bench/configs/legal_agent_bench.yaml @@ -14,6 +14,7 @@ legal_agent_bench: judge_base_url: ${oc.select:judge_base_url,null} judge_api_key: ${oc.select:judge_api_key,null} judge_model_name: ${oc.select:judge_model_name,null} + judge_reasoning_effort: ${oc.select:judge_reasoning_effort,null} judge_temperature: null judge_request_timeout_seconds: 90 judge_max_retries: 1 @@ -69,6 +70,7 @@ legal_agent_bench_harbor_agent: harbor_agent_override_timeout: 10800 harbor_agent_max_timeout: null + harbor_skip_verification_on_agent_failure: true harbor_verifier_override_timeout: 3600 harbor_verifier_max_timeout: null harbor_timeout_multiplier: null diff --git a/resources_servers/legal_agent_bench/configs/resources_only.yaml b/resources_servers/legal_agent_bench/configs/resources_only.yaml new file mode 100644 index 0000000000..c0640067ec --- /dev/null +++ b/resources_servers/legal_agent_bench/configs/resources_only.yaml @@ -0,0 +1,29 @@ +legal_agent_bench: + resources_servers: + legal_agent_bench: + entrypoint: app.py + domain: agent + verified: false + description: Resource-only Legal Agent Benchmark (LAB) configuration for configurable Gym agents + value: Improve legal-agent document review, drafting, and analysis capability + harbor_tasks_cache_dir: ${oc.env:LEGAL_AGENT_BENCH_TASK_CACHE_DIR,resources_servers/legal_agent_bench/data/cache/harbor_tasks/legal_agent_bench} + harbor_tasks_dir: ${oc.env:LEGAL_AGENT_BENCH_RUNTIME_TASKS_DIR,resources_servers/legal_agent_bench/data/runtime/harbor_tasks/legal_agent_bench} + harness_skills_dir: ${oc.env:LEGAL_AGENT_BENCH_SKILLS_DIR,resources_servers/legal_agent_bench/data/cache/harness/skills} + auto_prepare_assets: true + reward_mode: full_task + judge_base_url: ${oc.select:judge_base_url,null} + judge_api_key: ${oc.select:judge_api_key,null} + judge_model_name: ${oc.select:judge_model_name,null} + judge_reasoning_effort: ${oc.select:judge_reasoning_effort,null} + judge_temperature: null + judge_request_timeout_seconds: 90 + judge_max_retries: 1 + judge_structured_output: true + judge_parse_repair_attempts: 1 + judge_repair_max_tokens: 4096 + judge_max_tokens: 4096 + judge_parallelism: 6 + +responses_create_params: + temperature: 1.0 + top_p: 0.95 diff --git a/resources_servers/legal_agent_bench/data/example.jsonl b/resources_servers/legal_agent_bench/data/example.jsonl index 425bace0cc..0e90ca5120 100644 --- a/resources_servers/legal_agent_bench/data/example.jsonl +++ b/resources_servers/legal_agent_bench/data/example.jsonl @@ -1,5 +1,5 @@ -{"agent_ref": {"name": "legal_agent_bench_harbor_agent", "type": "responses_api_agents"}, "instance_id": "legal_agent_bench::trusts-estates-private-client__compare-trust-documents-against-client-instructions", "responses_create_params": {"input": [], "temperature": 1.0, "top_p": 0.95}} -{"agent_ref": {"name": "legal_agent_bench_harbor_agent", "type": "responses_api_agents"}, "instance_id": "legal_agent_bench::corporate-ma__analyze-transition-services-agreement-markup", "responses_create_params": {"input": [], "temperature": 1.0, "top_p": 0.95}} -{"agent_ref": {"name": "legal_agent_bench_harbor_agent", "type": "responses_api_agents"}, "instance_id": "legal_agent_bench::healthcare-life-sciences__analyze-compliance-program-gaps", "responses_create_params": {"input": [], "temperature": 1.0, "top_p": 0.95}} -{"agent_ref": {"name": "legal_agent_bench_harbor_agent", "type": "responses_api_agents"}, "instance_id": "legal_agent_bench::employment-labor__analyze-reasonable-accommodation-request-under-ada-requirements", "responses_create_params": {"input": [], "temperature": 1.0, "top_p": 0.95}} -{"agent_ref": {"name": "legal_agent_bench_harbor_agent", "type": "responses_api_agents"}, "instance_id": "legal_agent_bench::litigation-dispute-resolution__categorize-document-production-set-by-relevance-and-privilege", "responses_create_params": {"input": [], "temperature": 1.0, "top_p": 0.95}} +{"instance_id": "legal_agent_bench::trusts-estates-private-client__compare-trust-documents-against-client-instructions", "responses_create_params": {"input": [], "temperature": 1.0, "top_p": 0.95}} +{"instance_id": "legal_agent_bench::corporate-ma__analyze-transition-services-agreement-markup", "responses_create_params": {"input": [], "temperature": 1.0, "top_p": 0.95}} +{"instance_id": "legal_agent_bench::healthcare-life-sciences__analyze-compliance-program-gaps", "responses_create_params": {"input": [], "temperature": 1.0, "top_p": 0.95}} +{"instance_id": "legal_agent_bench::employment-labor__analyze-reasonable-accommodation-request-under-ada-requirements", "responses_create_params": {"input": [], "temperature": 1.0, "top_p": 0.95}} +{"instance_id": "legal_agent_bench::litigation-dispute-resolution__categorize-document-production-set-by-relevance-and-privilege", "responses_create_params": {"input": [], "temperature": 1.0, "top_p": 0.95}} diff --git a/resources_servers/legal_agent_bench/legal_harbor_agent.py b/resources_servers/legal_agent_bench/legal_harbor_agent.py index 7961937c92..f2b516f838 100644 --- a/resources_servers/legal_agent_bench/legal_harbor_agent.py +++ b/resources_servers/legal_agent_bench/legal_harbor_agent.py @@ -58,6 +58,10 @@ def __init__(self, logs_dir: Path, model_name: str | None = None, logger=None, * SYSTEM_PROMPT_PREAMBLE = SYSTEM_PROMPT_PATH.read_text(encoding="utf-8") +class LegalAgentBenchHarnessError(RuntimeError): + """The LAB Harbor harness failed after preserving its partial artifacts.""" + + class HarborToolExecutor: """Execute LAB tools inside a Harbor environment.""" @@ -310,8 +314,11 @@ async def run(self, instruction: str, environment: BaseEnvironment, context: Age model_name=self.model, agent_name=self.name(), ) + agent_failed = bool(result.get("model_error")) _write_agent_error_flags(Path(self.logs_dir), metrics) _populate_context(context, result, metrics, task_id, self.agent_id, artifact_dir) + if agent_failed: + raise LegalAgentBenchHarnessError(result["model_error"]) async def _hydrate_environment(self, environment: BaseEnvironment, docs_dir: Path) -> None: validate_harness_skills(self.skills_dir) @@ -391,6 +398,9 @@ async def _run_agent_async( finished_cleanly = False context_overflow = False model_error = None + model_error_type = None + model_connection_failed = False + agent_timed_out = False empty_response_count = 0 start_time = time.time() @@ -405,9 +415,14 @@ async def _run_agent_async( err_msg = str(exc) _log_model_error(transcript_file, turn_count, exc) if _is_context_overflow_error(err_msg): + # Context exhaustion is a valid incomplete model outcome. + # Preserve the partial artifacts and let Harbor verify them. context_overflow = True break model_error = err_msg + model_error_type = type(exc).__name__ + agent_timed_out = isinstance(exc, TimeoutError) + model_connection_failed = isinstance(exc, (ConnectionError, OSError)) and not agent_timed_out break messages.append(response.message) @@ -453,6 +468,9 @@ async def _run_agent_async( "finished_cleanly": (not context_overflow and finished_cleanly), "context_overflow": context_overflow, "model_error": model_error, + "model_error_type": model_error_type, + "model_connection_failed": model_connection_failed, + "agent_timed_out": agent_timed_out, "tool_metrics": tool_executor.get_metrics(), } @@ -717,6 +735,9 @@ def _populate_context( "lab_run_id": metrics["run_id"], "artifact_dir": str(artifact_dir), "finished_cleanly": metrics["finished_cleanly"], + "agent_failed": bool(result.get("model_error")), + "model_connection_failed": bool(result.get("model_connection_failed")), + "agent_timed_out": bool(result.get("agent_timed_out")), "model_error": metrics.get("model_error"), "turn_count": metrics["turn_count"], "tool_metrics": { diff --git a/resources_servers/legal_agent_bench/prepare.py b/resources_servers/legal_agent_bench/prepare.py index 20264afaa0..99bbeda90f 100644 --- a/resources_servers/legal_agent_bench/prepare.py +++ b/resources_servers/legal_agent_bench/prepare.py @@ -5,6 +5,7 @@ from __future__ import annotations import argparse +import fcntl import hashlib import json import os @@ -45,7 +46,7 @@ INDEX_FILENAME = "all.jsonl" DEFAULT_INDEX_FPATH = PACKAGE_DIR / "data" / "generated" / INDEX_FILENAME CACHE_MARKER = ".nemo_gym_asset.json" -CACHE_FORMAT_VERSION = 4 +CACHE_FORMAT_VERSION = 5 REWARD_MODES = ("full_task", "criteria_pass_rate") REWARD_MODE_ENV_KEY = "LEGAL_AGENT_BENCH_REWARD_MODE" LAB_HARBOR_SOURCE_DIR = PACKAGE_DIR / "vendor" / "harvey_labs" / "lab_harbor" @@ -79,9 +80,17 @@ && apt-get install -y --no-install-recommends \\ bash ca-certificates coreutils curl file findutils fonts-liberation g++ \\ gawk gcc git grep jq libreoffice nodejs npm pandoc poppler-utils procps \\ - qpdf ripgrep sed tesseract-ocr \\ + iproute2 qpdf ripgrep sed tesseract-ocr \\ && rm -rf /var/lib/apt/lists/* +# OpenShell executes commands as a restricted sandbox identity and requires +# iproute2 for its network namespace. The high UID/GID avoids colliding with +# ordinary host users when user-namespace remapping is unavailable. +RUN groupadd --gid 1000660000 sandbox \\ + && useradd -K UID_MAX=1000660000 --no-log-init --uid 1000660000 \\ + --gid sandbox --create-home --shell /bin/bash sandbox \\ + && install -d -o sandbox -g sandbox /workspace/output + RUN python -m pip install --upgrade pip \\ && python -m pip install \\ "defusedxml>=0.7.1" "diff-match-patch>=20230430" "docxtpl>=0.19.0" \\ @@ -101,12 +110,14 @@ _TEST_SCRIPT = """#!/usr/bin/env bash set -euo pipefail -mkdir -p /logs/verifier -python /tests/legal_agent_bench_verify.py \\ - --task-json /tests/task.json \\ - --run-dir /logs/agent/artifacts/lab-run \\ - --verifier-dir /logs/verifier \\ - --reward-json /logs/verifier/reward.json +tests_dir="${LAB_TESTS_DIR:-/tests}" +logs_dir="${LAB_LOGS_DIR:-/logs}" +mkdir -p "$logs_dir/verifier" +python "$tests_dir/legal_agent_bench_verify.py" \\ + --task-json "$tests_dir/task.json" \\ + --run-dir "$logs_dir/agent/artifacts/lab-run" \\ + --verifier-dir "$logs_dir/verifier" \\ + --reward-json "$logs_dir/verifier/reward.json" """ @@ -242,7 +253,10 @@ def prepare_assets( print(f"Using cached Legal Agent Bench {name} in {prepared[name]}", flush=True) continue except (FileNotFoundError, ValueError): - pass + if name == "tasks" and _migrate_legacy_agent_index(targets[name]): + prepared[name] = validators[name](targets[name]) + print(f"Migrated cached Legal Agent Bench {name} index in {prepared[name]}", flush=True) + continue missing.append(name) if not missing: @@ -479,10 +493,6 @@ def _render_task_index(source_ids: Iterable[str]) -> str: rows = [] for source_id in sorted(source_ids): row = { - "agent_ref": { - "name": "legal_agent_bench_harbor_agent", - "type": "responses_api_agents", - }, "instance_id": f"legal_agent_bench::{flatten_task_id(source_id)}", "responses_create_params": { "input": [], @@ -658,21 +668,72 @@ def _hardlink_or_copy(source: str, destination: str) -> str: def _replace_directory(source: Path, target: Path) -> None: backup = target.with_name(f".{target.name}.backup") - if backup.exists(): - shutil.rmtree(backup) - if target.exists(): - target.rename(backup) + lock_path = target.with_name(f".{target.name}.replace.lock") + lock_path.parent.mkdir(parents=True, exist_ok=True) + with lock_path.open("a+") as lock_file: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX) + try: + if backup.exists(): + if target.exists(): + shutil.rmtree(backup) + else: + backup.rename(target) + if target.exists(): + target.rename(backup) + try: + source.rename(target) + except Exception: + if target.exists(): + shutil.rmtree(target) + if backup.exists(): + backup.rename(target) + raise + else: + if backup.exists(): + shutil.rmtree(backup) + finally: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN) + + +def _migrate_legacy_agent_index(tasks_dir: Path) -> bool: + """Atomically remove the retired Harbor agent_ref from an otherwise exact cached index.""" + index_path = tasks_dir / INDEX_FILENAME + try: + rows = [json.loads(line) for line in index_path.read_text(encoding="utf-8").splitlines()] + except (OSError, json.JSONDecodeError): + return False + legacy_ref = { + "name": "legal_agent_bench_harbor_agent", + "type": "responses_api_agents", + } + if not rows or any(not isinstance(row, dict) or row.get("agent_ref") != legacy_ref for row in rows): + return False + candidate_rows = [] + for row in rows: + row = dict(row) + row.pop("agent_ref") + candidate_rows.append(json.dumps(row, ensure_ascii=False, sort_keys=True) + "\n") + candidate = "".join(candidate_rows) + + source_ids = [] + try: + for task_dir in sorted(child for child in tasks_dir.iterdir() if child.is_dir()): + task = json.loads((task_dir / "task.json").read_text(encoding="utf-8")) + source_ids.append(str(task["metadata"]["lab_task_id"])) + except (OSError, KeyError, TypeError, ValueError, json.JSONDecodeError): + return False + if candidate != _render_task_index(source_ids): + return False + + file_descriptor, temp_name = tempfile.mkstemp(dir=index_path.parent, prefix=f".{index_path.name}.") + os.close(file_descriptor) + temp_path = Path(temp_name) try: - source.rename(target) - except Exception: - if target.exists(): - shutil.rmtree(target) - if backup.exists(): - backup.rename(target) - raise - else: - if backup.exists(): - shutil.rmtree(backup) + temp_path.write_text(candidate, encoding="utf-8") + os.replace(temp_path, index_path) + finally: + temp_path.unlink(missing_ok=True) + return True def _publish_task_index(tasks_dir: Path) -> Path: diff --git a/resources_servers/legal_agent_bench/tests/test_app.py b/resources_servers/legal_agent_bench/tests/test_app.py index 6e42f46a3b..a2ba5f2896 100644 --- a/resources_servers/legal_agent_bench/tests/test_app.py +++ b/resources_servers/legal_agent_bench/tests/test_app.py @@ -31,6 +31,7 @@ def test_defaults_use_full_task_and_auto_prepare(tmp_path) -> None: assert server.config.reward_mode == "full_task" assert server.config.auto_prepare_assets is True assert server.config.judge_request_timeout_seconds == 90 + assert server.config.judge_reasoning_effort is None assert server.config.judge_max_retries == 1 assert server.config.judge_structured_output is True assert server.config.judge_parse_repair_attempts == 1 @@ -45,12 +46,14 @@ def test_startup_prepares_assets_then_rebuilds_runtime(tmp_path) -> None: judge_base_url="https://judge.example/v1", judge_api_key="test-key", # pragma: allowlist secret judge_model_name="provider/model-name", + judge_reasoning_effort="medium", judge_temperature=0.2, ) judge_env = { "LAB_JUDGE_BASE_URL": "https://judge.example/v1", "LAB_JUDGE_API_KEY": "test-key", # pragma: allowlist secret "LAB_JUDGE_MODEL": "openai-compatible/provider/model-name", + "LAB_JUDGE_REASONING_EFFORT": "medium", "LAB_JUDGE_TEMPERATURE": "0.2", "LAB_JUDGE_REQUEST_TIMEOUT_SECONDS": "90", "LAB_JUDGE_MAX_RETRIES": "1", diff --git a/resources_servers/legal_agent_bench/tests/test_harbor_integration.py b/resources_servers/legal_agent_bench/tests/test_harbor_integration.py index c969081115..49ea4f2a0b 100644 --- a/resources_servers/legal_agent_bench/tests/test_harbor_integration.py +++ b/resources_servers/legal_agent_bench/tests/test_harbor_integration.py @@ -4,12 +4,14 @@ from __future__ import annotations import asyncio +import io import json import threading import time from asyncio import Semaphore from datetime import datetime, timezone from pathlib import Path +from types import SimpleNamespace from unittest.mock import MagicMock import pytest @@ -18,8 +20,11 @@ from resources_servers.legal_agent_bench.harbor_bridge import REPO_ROOT, LegalAgentBenchHarborBridge from resources_servers.legal_agent_bench.legal_harbor_agent import ( LegalAgentBenchHarborAgent, + LegalAgentBenchHarnessError, + ModelResponse, OpenAICompatibleAdapter, _chat_with_timeout, + _run_agent_async, ) from resources_servers.legal_agent_bench.prepare import ( EXPECTED_TASK_COUNT, @@ -28,8 +33,11 @@ _marker, flatten_task_id, ) +from resources_servers.legal_agent_bench.vendor.harvey_labs.lab_harbor import container_tool_runner +from resources_servers.legal_agent_bench.vendor.harvey_labs.lab_harbor import judge as lab_judge from resources_servers.legal_agent_bench.vendor.harvey_labs.lab_harbor import scoring as lab_scoring from resources_servers.legal_agent_bench.vendor.harvey_labs.lab_harbor.judge import ( + OpenAICompatibleJudge, _extract_judge_message_text, ) from resources_servers.legal_agent_bench.verifier import ( @@ -44,6 +52,24 @@ BENCH_DIR = Path(__file__).resolve().parents[1] +@pytest.mark.parametrize("use_stdin", [False, True]) +def test_container_tool_runner_accepts_legacy_argv_and_stdin(monkeypatch, capsys, use_stdin: bool) -> None: + arguments = {"file_path": "memo.txt"} + raw_arguments = json.dumps(arguments) + argv = ["container_tool_runner.py", "preflight"] + if not use_stdin: + argv.append(raw_arguments) + monkeypatch.setattr(container_tool_runner.sys, "argv", argv) + monkeypatch.setattr(container_tool_runner.sys, "stdin", io.StringIO(raw_arguments if use_stdin else "")) + preflight = MagicMock(return_value="ready") + monkeypatch.setattr(container_tool_runner, "_preflight", preflight) + + container_tool_runner.main() + + assert json.loads(capsys.readouterr().out) == {"result": "ready", "metrics": {}} + preflight.assert_called_once_with() + + def _write_skills(skills_dir: Path) -> None: for name in REQUIRED_SKILLS: (skills_dir / name).mkdir(parents=True) @@ -80,6 +106,7 @@ def test_folder_config_is_public_docker_only() -> None: assert resource["auto_prepare_assets"] is True assert agent["harbor_environment_type"] == "docker" assert agent["harbor_environment_import_path"] is None + assert agent["harbor_skip_verification_on_agent_failure"] is True assert "docker_image" not in json.dumps(agent) @@ -272,6 +299,262 @@ async def slow_chat(_messages, _tools): await _chat_with_timeout(adapter, [], []) +@pytest.mark.asyncio +async def test_harbor_agent_preserves_partial_trajectory_then_propagates_failure(monkeypatch, tmp_path) -> None: + task_dir = tmp_path / "task" + (task_dir / "documents").mkdir(parents=True) + (task_dir / "environment").mkdir() + (task_dir / "task.toml").write_text('version = "1.0"\n', encoding="utf-8") + (task_dir / "task.json").write_text( + json.dumps( + { + "title": "Test task", + "instructions": "Write the deliverable.", + "criteria": [{"id": "C-1", "title": "Done", "match_criteria": "Pass."}], + } + ), + encoding="utf-8", + ) + logs_dir = tmp_path / "logs" / "agent" + agent = LegalAgentBenchHarborAgent( + logs_dir=logs_dir, + model_name="policy-model", + api_base="http://policy/v1", + skills_dir=tmp_path / "skills", + ) + monkeypatch.setattr(agent, "_skill_names", lambda: []) + + async def hydrate(_environment, _docs_dir): + return None + + monkeypatch.setattr(agent, "_hydrate_environment", hydrate) + monkeypatch.setattr(agent, "_create_adapter", MagicMock()) + monkeypatch.setattr( + "resources_servers.legal_agent_bench.legal_harbor_agent.get_all_tool_definitions", + lambda: [], + ) + + class ToolExecutor: + def __init__(self, *_args, **_kwargs): + pass + + async def preflight(self): + return None + + monkeypatch.setattr( + "resources_servers.legal_agent_bench.legal_harbor_agent.HarborToolExecutor", + ToolExecutor, + ) + + async def partial_failure(*, transcript_path, **_kwargs): + transcript_path.write_text( + json.dumps( + { + "turn": 1, + "role": "assistant", + "message": {"role": "assistant", "content": "Partial work"}, + "text": "Partial work", + "tool_calls": None, + "input_tokens": 11, + "output_tokens": 3, + } + ) + + "\n", + encoding="utf-8", + ) + return { + "messages": [], + "turn_count": 1, + "input_tokens": 11, + "output_tokens": 3, + "wall_clock_seconds": 0.1, + "finished_cleanly": False, + "context_overflow": False, + "model_error": "adapter failed after partial output", + "model_error_type": "ConnectionError", + "model_connection_failed": True, + "agent_timed_out": False, + "tool_metrics": {}, + } + + monkeypatch.setattr( + "resources_servers.legal_agent_bench.legal_harbor_agent._run_agent_async", + partial_failure, + ) + + class Environment: + environment_dir = task_dir / "environment" + + async def download_dir(self, _source, target): + Path(target).mkdir(parents=True, exist_ok=True) + + context = SimpleNamespace() + with pytest.raises(LegalAgentBenchHarnessError, match="adapter failed after partial output"): + await agent.run("", Environment(), context) + + trajectory = json.loads((logs_dir / "trajectory.json").read_text(encoding="utf-8")) + assert trajectory["steps"][-1]["message"] == "Partial work" + assert context.metadata["agent_failed"] is True + assert context.metadata["model_connection_failed"] is True + assert context.metadata["agent_timed_out"] is False + assert context.metadata["model_error"] == "adapter failed after partial output" + + +@pytest.mark.asyncio +async def test_harbor_loop_classifies_timeout_after_partial_output(tmp_path) -> None: + class Adapter: + timeout_seconds = None + + def __init__(self): + self.calls = 0 + + def make_system_message(self, content): + return {"role": "system", "content": content} + + def make_user_message(self, content): + return {"role": "user", "content": content} + + def make_tool_result_messages(self, results): + return [{"role": "tool", "content": result} for _call_id, result in results] + + async def chat(self, _messages, _tools): + self.calls += 1 + if self.calls == 1: + return ModelResponse( + message={"role": "assistant", "content": "Working"}, + tool_calls=[SimpleNamespace(id="call-1", name="read", arguments='{"path":"input.docx"}')], + text="Working", + input_tokens=10, + output_tokens=2, + ) + raise TimeoutError("model request timed out") + + class ToolExecutor: + async def execute(self, _name, _arguments): + return "document text" + + def get_metrics(self): + return {} + + transcript = tmp_path / "transcript.jsonl" + result = await _run_agent_async( + adapter=Adapter(), + system_prompt="system", + tool_executor=ToolExecutor(), + tools=[], + max_turns=3, + transcript_path=transcript, + ) + + entries = [json.loads(line) for line in transcript.read_text(encoding="utf-8").splitlines()] + assert any(entry.get("role") == "assistant" for entry in entries) + assert any(entry.get("role") == "model_error" for entry in entries) + assert result["model_error"] == "model request timed out" + assert result["agent_timed_out"] is True + assert result["model_connection_failed"] is False + + +@pytest.mark.asyncio +async def test_harbor_loop_preserves_context_limit_as_scoreable_incomplete_output(tmp_path) -> None: + class Adapter: + timeout_seconds = None + + def __init__(self): + self.calls = 0 + + def make_system_message(self, content): + return {"role": "system", "content": content} + + def make_user_message(self, content): + return {"role": "user", "content": content} + + def make_tool_result_messages(self, results): + return [{"role": "tool", "content": result} for _call_id, result in results] + + async def chat(self, _messages, _tools): + self.calls += 1 + if self.calls == 1: + return ModelResponse( + message={"role": "assistant", "content": "Partial work"}, + tool_calls=[SimpleNamespace(id="call-1", name="read", arguments='{"path":"input.docx"}')], + text="Partial work", + input_tokens=10, + output_tokens=2, + ) + raise RuntimeError("maximum context length exceeded") + + class ToolExecutor: + async def execute(self, _name, _arguments): + return "document text" + + def get_metrics(self): + return {} + + result = await _run_agent_async( + adapter=Adapter(), + system_prompt="system", + tool_executor=ToolExecutor(), + tools=[], + max_turns=3, + transcript_path=tmp_path / "transcript.jsonl", + ) + + assert result["context_overflow"] is True + assert result["finished_cleanly"] is False + assert result["model_error"] is None + assert result["model_connection_failed"] is False + assert result["agent_timed_out"] is False + assert result["output_tokens"] == 2 + + +@pytest.mark.asyncio +async def test_harbor_loop_preserves_max_turns_as_scoreable_incomplete_output(tmp_path) -> None: + class Adapter: + timeout_seconds = None + + def make_system_message(self, content): + return {"role": "system", "content": content} + + def make_user_message(self, content): + return {"role": "user", "content": content} + + def make_tool_result_messages(self, results): + return [{"role": "tool", "content": result} for _call_id, result in results] + + async def chat(self, _messages, _tools): + return ModelResponse( + message={"role": "assistant", "content": "Partial work"}, + tool_calls=[SimpleNamespace(id="call-1", name="read", arguments='{"path":"input.docx"}')], + text="Partial work", + input_tokens=10, + output_tokens=2, + ) + + class ToolExecutor: + async def execute(self, _name, _arguments): + return "document text" + + def get_metrics(self): + return {} + + result = await _run_agent_async( + adapter=Adapter(), + system_prompt="system", + tool_executor=ToolExecutor(), + tools=[], + max_turns=1, + transcript_path=tmp_path / "transcript.jsonl", + ) + + assert result["turn_count"] == 1 + assert result["finished_cleanly"] is False + assert result["context_overflow"] is False + assert result["model_error"] is None + assert result["model_connection_failed"] is False + assert result["agent_timed_out"] is False + assert result["output_tokens"] == 2 + + @pytest.mark.parametrize( "message, expected", [ @@ -289,6 +572,48 @@ def test_empty_judge_message_fails_clearly() -> None: _extract_judge_message_text({"content": "", "reasoning_content": None}) +def test_judge_sends_configured_reasoning_effort(monkeypatch) -> None: + observed = {} + + class Response: + def __enter__(self): + return self + + def __exit__(self, *_args): + return None + + def read(self): + return json.dumps({"choices": [{"message": {"content": '{"verdict":"pass","reasoning":"ok"}'}}]}).encode() + + def urlopen(request, *, timeout): + observed["payload"] = json.loads(request.data) + observed["timeout"] = timeout + return Response() + + monkeypatch.setattr(lab_judge.urllib.request, "urlopen", urlopen) + judge = OpenAICompatibleJudge( + model="openai-compatible/provider/judge", + base_url="https://judge.example/v1", + api_key="test-key", # pragma: allowlist secret + temperature=None, + timeout_seconds=30, + reasoning_effort="medium", + ) + + result = judge.evaluate( + { + "task_description": "task", + "agent_output": "output", + "criterion_title": "criterion", + "match_criteria": "match", + } + ) + + assert result == {"verdict": "pass", "reasoning": "ok"} + assert observed["payload"]["reasoning_effort"] == "medium" + assert observed["timeout"] == 30 + + def test_reward_mode_validation() -> None: assert _validate_reward_mode("full_task") == "full_task" assert _validate_reward_mode("criteria_pass_rate") == "criteria_pass_rate" @@ -389,6 +714,21 @@ def test_deliverable_matching_ignores_thread_export() -> None: ) == {"contract": "final-contract.docx"} +def test_full_output_ignores_raw_ooxml_working_files(monkeypatch, tmp_path) -> None: + output_dir = tmp_path / "output" + (output_dir / "workdir" / "word" / "_rels").mkdir(parents=True) + (output_dir / "response.docx").write_bytes(b"placeholder") + (output_dir / "workdir" / "word" / "document.xml").write_text("raw document xml") + (output_dir / "workdir" / "word" / "_rels" / "document.xml.rels").write_text("raw relationships") + monkeypatch.setattr(lab_scoring, "_read_file_as_text", lambda path: f"content:{path.name}") + + content = lab_scoring._load_all_output(output_dir) + + assert "content:response.docx" in content + assert "document.xml" not in content + assert "document.xml.rels" not in content + + def test_parallel_judging_uses_isolated_judges_and_preserves_order(tmp_path) -> None: output_dir = tmp_path / "run" / "output" output_dir.mkdir(parents=True) diff --git a/resources_servers/legal_agent_bench/tests/test_prepare.py b/resources_servers/legal_agent_bench/tests/test_prepare.py index 973f53cabb..d7763e0092 100644 --- a/resources_servers/legal_agent_bench/tests/test_prepare.py +++ b/resources_servers/legal_agent_bench/tests/test_prepare.py @@ -7,6 +7,9 @@ import io import json import tarfile +import threading +import time +from concurrent.futures import ThreadPoolExecutor from pathlib import Path import pytest @@ -90,7 +93,12 @@ def test_generated_task_cache_is_deterministic_and_credential_free(monkeypatch, "legal_agent_bench::area__task-group__scenario-01", "legal_agent_bench::area__task-one", ] - assert all(row["agent_ref"]["name"] == "legal_agent_bench_harbor_agent" for row in rows) + assert all("agent_ref" not in row for row in rows) + dockerfile = (first / "area__task-one" / "environment" / "Dockerfile").read_text(encoding="utf-8") + assert "iproute2" in dockerfile + assert "groupadd --gid 1000660000 sandbox" in dockerfile + assert "useradd -K UID_MAX=1000660000" in dockerfile + assert "install -d -o sandbox -g sandbox /workspace/output" in dockerfile for toml in first.glob("*/task.toml"): text = toml.read_text(encoding="utf-8") assert "[verifier.env]" not in text @@ -112,6 +120,31 @@ def test_existing_valid_caches_skip_network(monkeypatch, tmp_path) -> None: assert (tmp_path / "all.jsonl").read_bytes() == (tasks / prepare.INDEX_FILENAME).read_bytes() +def test_legacy_harbor_index_migrates_without_network(monkeypatch, tmp_path) -> None: + _source, tasks, _skills = _build_caches(monkeypatch, tmp_path) + index_path = tasks / prepare.INDEX_FILENAME + legacy_rows = [] + for line in index_path.read_text(encoding="utf-8").splitlines(): + row = json.loads(line) + row["agent_ref"] = { + "name": "legal_agent_bench_harbor_agent", + "type": "responses_api_agents", + } + legacy_rows.append(json.dumps(row, sort_keys=True) + "\n") + index_path.write_text("".join(legacy_rows), encoding="utf-8") + monkeypatch.setattr( + prepare, + "_download_source_archive", + lambda _path: pytest.fail("the exact legacy index must migrate without a source download"), + ) + + result = prepare.prepare_assets("tasks", tasks_dir=tasks) + + assert result == {"tasks": tasks} + assert all("agent_ref" not in json.loads(line) for line in index_path.read_text().splitlines()) + prepare.validate_harbor_tasks(tasks) + + def test_missing_assets_download_extract_and_install(monkeypatch, tmp_path) -> None: _configure_small_snapshot(monkeypatch, tmp_path) source = _write_source(tmp_path / "source") @@ -236,6 +269,66 @@ def test_runtime_hydration_can_reuse_a_validated_cache(monkeypatch, tmp_path) -> assert (runtime / "area__task-one" / "task.toml").is_file() +def test_directory_replacement_is_serialized_across_concurrent_publishers(monkeypatch, tmp_path) -> None: + target = tmp_path / "runtime" + target.mkdir() + (target / "value").write_text("old") + sources = [] + for value in ("first", "second"): + source = tmp_path / value + source.mkdir() + (source / "value").write_text(value) + sources.append(source) + + original_rename = Path.rename + active_renames = 0 + maximum_active_renames = 0 + counter_lock = threading.Lock() + + def slow_rename(path: Path, destination: Path) -> Path: + nonlocal active_renames, maximum_active_renames + with counter_lock: + active_renames += 1 + maximum_active_renames = max(maximum_active_renames, active_renames) + try: + time.sleep(0.02) + return original_rename(path, destination) + finally: + with counter_lock: + active_renames -= 1 + + monkeypatch.setattr(Path, "rename", slow_rename) + with ThreadPoolExecutor(max_workers=2) as executor: + list(executor.map(lambda source: prepare._replace_directory(source, target), sources)) + + assert maximum_active_renames == 1 + assert (target / "value").read_text() in {"first", "second"} + assert not target.with_name(".runtime.backup").exists() + + +def test_directory_replacement_restores_stale_backup_before_failed_publish(monkeypatch, tmp_path) -> None: + target = tmp_path / "runtime" + backup = tmp_path / ".runtime.backup" + source = tmp_path / "new" + backup.mkdir() + source.mkdir() + (backup / "value").write_text("preserved") + + original_rename = Path.rename + + def fail_new_publish(path: Path, destination: Path) -> Path: + if path == source: + raise OSError("publish failed") + return original_rename(path, destination) + + monkeypatch.setattr(Path, "rename", fail_new_publish) + with pytest.raises(OSError, match="publish failed"): + prepare._replace_directory(source, target) + + assert (target / "value").read_text() == "preserved" + assert not backup.exists() + + def test_missing_public_skill_fails_clearly(monkeypatch, tmp_path) -> None: _configure_small_snapshot(monkeypatch, tmp_path) source = _write_source(tmp_path / "source", missing_skill="pptx") diff --git a/resources_servers/legal_agent_bench/vendor/harvey_labs/lab_harbor/container_tool_runner.py b/resources_servers/legal_agent_bench/vendor/harvey_labs/lab_harbor/container_tool_runner.py index db3b7fc70e..619d1ba37c 100644 --- a/resources_servers/legal_agent_bench/vendor/harvey_labs/lab_harbor/container_tool_runner.py +++ b/resources_servers/legal_agent_bench/vendor/harvey_labs/lab_harbor/container_tool_runner.py @@ -4,8 +4,9 @@ """Container-side implementation for non-bash Harbor agent tools. LegalAgentBenchHarborAgent invokes this script inside the Harbor environment -with a tool name and a JSON argument payload. It returns one JSON object on -stdout: +with a tool name and a JSON argument payload. The native Gym agent passes the +payload over stdin so large document writes do not exceed the operating +system's argument-size limit. It returns one JSON object on stdout: {"result": "...", "metrics": {...}} @@ -36,13 +37,17 @@ def main() -> None: - if len(sys.argv) != 3: + if len(sys.argv) == 3: + raw_arguments = sys.argv[2] + elif len(sys.argv) == 2: + raw_arguments = sys.stdin.read() + else: _emit("Error: expected tool name and JSON arguments", {}) return tool_name = sys.argv[1] try: - arguments = json.loads(sys.argv[2]) + arguments = json.loads(raw_arguments) except json.JSONDecodeError as exc: _emit(f"Error: invalid JSON arguments: {exc}", {}) return diff --git a/resources_servers/legal_agent_bench/vendor/harvey_labs/lab_harbor/judge.py b/resources_servers/legal_agent_bench/vendor/harvey_labs/lab_harbor/judge.py index 69b6e9f2e5..54baeb651a 100644 --- a/resources_servers/legal_agent_bench/vendor/harvey_labs/lab_harbor/judge.py +++ b/resources_servers/legal_agent_bench/vendor/harvey_labs/lab_harbor/judge.py @@ -75,6 +75,7 @@ def __init__( parse_repair_attempts: int = 2, repair_max_tokens: int = 1024, transcript_path: Path | None = None, + reasoning_effort: str | None = None, ): self.model = model self.api_model = self._normalize_api_model(model) @@ -87,6 +88,7 @@ def __init__( self.parse_repair_attempts = max(0, parse_repair_attempts) self.repair_max_tokens = max(1, repair_max_tokens) self.transcript_path = transcript_path + self.reasoning_effort = reasoning_effort self.trace_context: dict[str, Any] = {} self.last_raw_response: str | None = None self.last_structured: bool | None = None @@ -320,6 +322,8 @@ def _chat_completion( } if self.temperature is not None: payload["temperature"] = self.temperature + if self.reasoning_effort is not None: + payload["reasoning_effort"] = self.reasoning_effort if structured: payload["response_format"] = { "type": "json_schema", @@ -350,6 +354,7 @@ def _chat_completion( "model": self.api_model, "structured": structured, "max_tokens": max_tokens, + "reasoning_effort": self.reasoning_effort, "prompt_chars": len(prompt), "payload_bytes": len(body), "request_timeout_seconds": self.timeout_seconds, diff --git a/resources_servers/legal_agent_bench/vendor/harvey_labs/lab_harbor/scoring.py b/resources_servers/legal_agent_bench/vendor/harvey_labs/lab_harbor/scoring.py index 1727d28ab8..acdcf727b2 100644 --- a/resources_servers/legal_agent_bench/vendor/harvey_labs/lab_harbor/scoring.py +++ b/resources_servers/legal_agent_bench/vendor/harvey_labs/lab_harbor/scoring.py @@ -10,7 +10,8 @@ - uses the configured OpenAI-compatible judge instead of provider SDKs; - creates isolated judges for parallel criteria; - records per-criterion transcripts and judge errors; -- fails missing or unreadable deliverables without calling the judge; and +- fails missing or unreadable deliverables without calling the judge; +- excludes raw OOXML working files from aggregate judge input; and - uses deterministic filename matching without Anthropic-specific LLM fallback. """ @@ -27,7 +28,7 @@ SKIP_DIRS = {"node_modules", ".npm", "__pycache__", ".git", "venv", ".venv"} -SKIP_EXTENSIONS = {".lock", ".map"} +SKIP_EXTENSIONS = {".lock", ".map", ".rels", ".xml"} SKIP_FILES = {"package-lock.json"} diff --git a/resources_servers/legal_agent_bench/verifier.py b/resources_servers/legal_agent_bench/verifier.py index 7cafbcfebb..fa86386fa9 100644 --- a/resources_servers/legal_agent_bench/verifier.py +++ b/resources_servers/legal_agent_bench/verifier.py @@ -92,6 +92,7 @@ def run_verifier(args: argparse.Namespace) -> dict[str, Any]: "base_url": os.environ.get("LAB_JUDGE_BASE_URL") or args.judge_base_url, "api_key": os.environ.get("LAB_JUDGE_API_KEY") or args.judge_api_key, "temperature": _optional_float(os.environ.get("LAB_JUDGE_TEMPERATURE")), + "reasoning_effort": os.environ.get("LAB_JUDGE_REASONING_EFFORT") or None, "timeout_seconds": float( os.environ.get("LAB_JUDGE_REQUEST_TIMEOUT_SECONDS") or os.environ.get("LAB_JUDGE_TIMEOUT_SECONDS", "90") ), @@ -128,6 +129,7 @@ def judge_factory() -> OpenAICompatibleJudge: "task": config.get("task") or metrics.get("task") or task_config.get("title", ""), "judge_model": judge_model, "judge_model_base_url": judge.base_url, + "judge_reasoning_effort": judge.reasoning_effort, "judge_parallelism": judge_parallelism, "scored_at": datetime.now(timezone.utc).isoformat(), } diff --git a/responses_api_agents/claude_code_agent/app.py b/responses_api_agents/claude_code_agent/app.py index dd85306962..26268c1590 100644 --- a/responses_api_agents/claude_code_agent/app.py +++ b/responses_api_agents/claude_code_agent/app.py @@ -489,13 +489,13 @@ async def _run_claude_code( proc.kill() stdout, _ = await communication LOG.warning("claude-code timed out after %ds", self.config.timeout) - _, run_metadata = parse_stream_json(stdout.decode(errors="replace")) + output_items, run_metadata = parse_stream_json(stdout.decode(errors="replace")) run_metadata.update( status="incomplete", error_type="timeout", duration_ms=(monotonic() - process_started_at) * 1000, ) - return [], model, run_metadata + return output_items, model, run_metadata except asyncio.CancelledError: if proc.returncode is None: with suppress(ProcessLookupError): @@ -627,10 +627,23 @@ async def _create_response( input_tokens = run_metadata.get("input_tokens", 0) output_tokens = run_metadata.get("output_tokens", 0) + run_status = str(run_metadata.get("status") or "incomplete") + limit_reached = run_status == "incomplete" and run_metadata.get("error_type") == "error_max_turns" + failure_message = None + if run_status != "completed" and not limit_reached: + failure_message = str(run_metadata.get("error_type") or run_status) return NeMoGymResponse( id=f"resp_{uuid4().hex}", created_at=int(time()), + status="incomplete" if limit_reached else ("failed" if failure_message else "completed"), + error=( + {"code": "server_error", "message": f"Claude Code failed: {failure_message}"} + if failure_message + else None + ), + incomplete_details=({"reason": "max_output_tokens"} if limit_reached else None), + metadata=({"nemo_gym_stop_reason": "max_turns"} if limit_reached else None), model=model_name, object="response", output=output_items, diff --git a/responses_api_agents/claude_code_agent/scripts/claude_code_agent_deps.sh b/responses_api_agents/claude_code_agent/scripts/claude_code_agent_deps.sh index 3a12fca8e4..99d739cbb3 100755 --- a/responses_api_agents/claude_code_agent/scripts/claude_code_agent_deps.sh +++ b/responses_api_agents/claude_code_agent/scripts/claude_code_agent_deps.sh @@ -9,16 +9,26 @@ source "${PORTABLE_PYTHON_SH:-$SCRIPT_DIR/_portable_python.sh}" : "${DEPS_DIR:?DEPS_DIR must be set}" : "${NEMO_GYM_ROOT:?NEMO_GYM_ROOT must be set}" -NODE_VERSION="${NODE_VERSION:-20.18.1}" -CLAUDE_SPEC="${CLAUDE_SPEC:-@anthropic-ai/claude-code}" +NODE_VERSION="${NODE_VERSION:-22.15.0}" +: "${CLAUDE_SPEC:?CLAUDE_SPEC must select a pinned @anthropic-ai/claude-code version}" +if [ -z "${NODE_ARCH:-}" ]; then + case "$(uname -m)" in + x86_64) NODE_ARCH="x64" ;; + aarch64|arm64) NODE_ARCH="arm64" ;; + *) + echo "Unsupported Node architecture: $(uname -m)" >&2 + exit 1 + ;; + esac +fi install_portable_python install_nemo_gym_deps if [ ! -x "$DEPS_DIR/bin/node" ]; then - node_url="https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-x64.tar.xz" + node_url="https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-${NODE_ARCH}.tar.gz" echo "Downloading portable node: $node_url" - curl -fsSL "$node_url" | tar xJ -C "$DEPS_DIR" --strip-components=1 + curl -fsSL "$node_url" | tar xz -C "$DEPS_DIR" --strip-components=1 fi export PATH="$DEPS_DIR/bin:$PATH" diff --git a/responses_api_agents/claude_code_agent/tests/test_app.py b/responses_api_agents/claude_code_agent/tests/test_app.py index 1625d753ee..435d279116 100644 --- a/responses_api_agents/claude_code_agent/tests/test_app.py +++ b/responses_api_agents/claude_code_agent/tests/test_app.py @@ -1032,3 +1032,52 @@ def test_config_yaml_parses(self) -> None: assert inner["entrypoint"] == "app.py" assert inner["concurrency"] == 32 assert inner["max_turns"] == 30 + + +def test_partial_claude_output_preserves_trajectory_and_marks_response_failed() -> None: + agent = _make_agent() + partial = NeMoGymResponseOutputMessage.model_validate( + { + "id": "msg-1", + "content": [{"type": "output_text", "text": "partial answer", "annotations": []}], + "role": "assistant", + "status": "completed", + "type": "message", + } + ) + + async def fake_run_claude_code(*args, **kwargs): + return [partial], "claude-test", {"status": "failed", "error_type": "process_exit_1"} + + object.__setattr__(agent, "_run_claude_code", fake_run_claude_code) + response = asyncio.run(agent._create_response(NeMoGymResponseCreateParamsNonStreaming(input="perform the task"))) + + assert response.output + assert response.status == "failed" + assert response.error is not None + assert "process_exit_1" in response.error.message + + +def test_claude_max_turn_output_is_a_scoreable_incomplete_outcome() -> None: + agent = _make_agent() + partial = NeMoGymResponseOutputMessage.model_validate( + { + "id": "msg-1", + "content": [{"type": "output_text", "text": "partial answer", "annotations": []}], + "role": "assistant", + "status": "completed", + "type": "message", + } + ) + + async def fake_run_claude_code(*args, **kwargs): + return [partial], "claude-test", {"status": "incomplete", "error_type": "error_max_turns"} + + object.__setattr__(agent, "_run_claude_code", fake_run_claude_code) + response = asyncio.run(agent._create_response(NeMoGymResponseCreateParamsNonStreaming(input="perform the task"))) + + assert response.output + assert response.status == "incomplete" + assert response.error is None + assert response.incomplete_details.reason == "max_output_tokens" + assert response.metadata == {"nemo_gym_stop_reason": "max_turns"} diff --git a/responses_api_agents/codex_agent/app.py b/responses_api_agents/codex_agent/app.py index e5eaec65bd..3065e4b6ed 100644 --- a/responses_api_agents/codex_agent/app.py +++ b/responses_api_agents/codex_agent/app.py @@ -240,6 +240,22 @@ def _add_tool_pair(item: dict[str, Any], name: str, arguments: dict[str, Any], o return output_items, metadata +def _is_context_limit_error(message: str) -> bool: + normalized = message.lower() + return any( + marker in normalized + for marker in ( + "context length", + "context window", + "context_length_exceeded", + "maximum context", + "max context", + "too many tokens", + "token limit", + ) + ) + + def _kill_process_group(proc: Any) -> None: """Kill the codex subprocess and every child in its process group. @@ -463,8 +479,8 @@ async def _run_codex( mcp_servers: Optional[dict[str, Any]] = None, skills_path: Optional[str] = None, rollout_id: Optional[str] = None, - ) -> tuple[str, str]: - """Run ``codex exec --json`` and return (stdout, model_name). + ) -> tuple[str, str, dict[str, Any]]: + """Run ``codex exec --json`` and return stdout, model name, and process outcome. When ``rollout_id`` is set and a model server is configured, the per-rollout capture prefix is applied to the provider base_url so the CLI's streaming /v1/responses calls correlate to @@ -509,15 +525,29 @@ async def _run_codex( stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=self.config.timeout) except asyncio.TimeoutError: _kill_process_group(proc) - await proc.communicate() + stdout, stderr = await proc.communicate() LOG.warning("codex timed out after %ds", self.config.timeout) - return "", model + return ( + stdout.decode(errors="replace"), + model, + { + "status": "failed", + "error_type": "timeout", + "stderr": stderr.decode(errors="replace"), + }, + ) + outcome = {"status": "completed", "return_code": proc.returncode} if proc.returncode not in (0, None): LOG.warning("codex exited %d: %s", proc.returncode, stderr.decode(errors="replace")[:500]) + outcome.update( + status="failed", + error_type=f"process_exit_{proc.returncode}", + stderr=stderr.decode(errors="replace"), + ) LOG.debug("codex stdout (%d chars): %s", len(stdout), stdout[:2000].decode(errors="replace")) - return stdout.decode(errors="replace"), model + return stdout.decode(errors="replace"), model, outcome finally: if codex_home is not None: shutil.rmtree(codex_home, ignore_errors=True) @@ -572,7 +602,7 @@ async def _create_response( system_parts = [p for p in [self.config.system_prompt, input_system] if p] system_prompt = "\n\n".join(system_parts) if system_parts else None - stdout, model_name = await self._run_codex( + stdout, model_name, run_metadata = await self._run_codex( user_message, system_prompt=system_prompt, mcp_servers=mcp_servers, @@ -583,6 +613,11 @@ async def _create_response( if usage.get("errors"): LOG.warning("codex reported errors: %s", usage["errors"]) + stream_error = "; ".join(str(error) for error in usage["errors"]) + if all(_is_context_limit_error(str(error)) for error in usage["errors"]): + run_metadata.update(status="incomplete", error_type="context_limit", stderr=stream_error) + else: + run_metadata.update(status="failed", error_type="stream_error", stderr=stream_error) if not any( getattr(item, "type", None) == "message" and getattr(item, "role", None) == "assistant" @@ -601,10 +636,27 @@ async def _create_response( input_tokens = usage.get("input_tokens", 0) output_tokens = usage.get("output_tokens", 0) + run_status = run_metadata.get("status") + run_error = " ".join( + str(value) for value in (run_metadata.get("error_type"), run_metadata.get("stderr")) if value + ) + limit_reached = run_status != "completed" and _is_context_limit_error(run_error) + failure_message = None + if run_status != "completed" and not limit_reached: + failure_message = str(run_metadata.get("error_type") or "codex_failed") + stderr = str(run_metadata.get("stderr") or "").strip() + if stderr: + failure_message = f"{failure_message}: {stderr[-1000:]}" return NeMoGymResponse( id=f"resp_{uuid4().hex}", created_at=int(time()), + status="incomplete" if limit_reached else ("failed" if failure_message else "completed"), + error=( + {"code": "server_error", "message": f"Codex failed: {failure_message}"} if failure_message else None + ), + incomplete_details=({"reason": "max_output_tokens"} if limit_reached else None), + metadata=({"nemo_gym_stop_reason": "context_limit"} if limit_reached else None), model=model_name, object="response", output=output_items, diff --git a/responses_api_agents/codex_agent/scripts/codex_agent_deps.sh b/responses_api_agents/codex_agent/scripts/codex_agent_deps.sh new file mode 100755 index 0000000000..c487ab5c1c --- /dev/null +++ b/responses_api_agents/codex_agent/scripts/codex_agent_deps.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# Install Codex agent dependencies into a portable prefix mounted in a task sandbox. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${PORTABLE_PYTHON_SH:-$SCRIPT_DIR/_portable_python.sh}" + +: "${DEPS_DIR:?DEPS_DIR must be set}" +: "${NEMO_GYM_ROOT:?NEMO_GYM_ROOT must be set}" +NODE_VERSION="${NODE_VERSION:-22.15.0}" +: "${CODEX_SPEC:?CODEX_SPEC must select a pinned @openai/codex version}" +if [ -z "${NODE_ARCH:-}" ]; then + case "$(uname -m)" in + x86_64) NODE_ARCH="x64" ;; + aarch64|arm64) NODE_ARCH="arm64" ;; + *) + echo "Unsupported Node architecture: $(uname -m)" >&2 + exit 1 + ;; + esac +fi + +install_portable_python +install_nemo_gym_deps + +if [ ! -x "$DEPS_DIR/bin/node" ]; then + node_url="https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-${NODE_ARCH}.tar.gz" + curl -fsSL "$node_url" | tar xz -C "$DEPS_DIR" --strip-components=1 +fi + +export PATH="$DEPS_DIR/bin:$PATH" +npm install -g --prefix "$DEPS_DIR" "$CODEX_SPEC" +"$DEPS_DIR/bin/codex" --version diff --git a/responses_api_agents/codex_agent/tests/test_app.py b/responses_api_agents/codex_agent/tests/test_app.py index 78bdf18d99..264dceb789 100644 --- a/responses_api_agents/codex_agent/tests/test_app.py +++ b/responses_api_agents/codex_agent/tests/test_app.py @@ -311,7 +311,7 @@ def _run(self, agent: CodexAgent, body: CodexAgentRunRequest, run_codex: AsyncMo def test_skills_ref_path_forwarded(self) -> None: agent = _make_agent() - run_codex = AsyncMock(return_value=("", "codex-default")) + run_codex = AsyncMock(return_value=("", "codex-default", {"status": "completed"})) body = CodexAgentRunRequest.model_validate( { "responses_create_params": {"input": []}, @@ -325,7 +325,7 @@ def test_skills_ref_path_forwarded(self) -> None: def test_no_skills_ref_forwards_none(self) -> None: agent = _make_agent() - run_codex = AsyncMock(return_value=("", "codex-default")) + run_codex = AsyncMock(return_value=("", "codex-default", {"status": "completed"})) body = CodexAgentRunRequest.model_validate({"responses_create_params": {"input": []}}) self._run(agent, body, run_codex) @@ -365,7 +365,7 @@ async def fake_exec(*cmd, **kwargs): patch("responses_api_agents.codex_agent.app.Path.home", return_value=tmp_path), patch("responses_api_agents.codex_agent.app.asyncio.create_subprocess_exec", fake_exec), ): - stdout, model = asyncio.run(agent._run_codex("hello", system_prompt="be terse")) + stdout, model, outcome = asyncio.run(agent._run_codex("hello", system_prompt="be terse")) assert captured["cmd"][0] == "codex" assert captured["cmd"][-1] == "hello" @@ -375,6 +375,7 @@ async def fake_exec(*cmd, **kwargs): assert captured["start_new_session"] is True assert captured["config_during_run"]["developer_instructions"] == "be terse" assert captured["cwd_exists_during_run"] is True + assert outcome["status"] == "completed" # per-run home and scratch cwd are removed after the run (no leakage between rollouts) assert not Path(captured["codex_home"]).exists() assert not Path(captured["scratch_cwd"]).exists() @@ -446,11 +447,12 @@ async def fake_wait_for(coro, timeout): patch("responses_api_agents.codex_agent.app.asyncio.create_subprocess_exec", fake_exec), patch("responses_api_agents.codex_agent.app.asyncio.wait_for", fake_wait_for), ): - stdout, model = asyncio.run(agent._run_codex("hello")) + stdout, model, outcome = asyncio.run(agent._run_codex("hello")) assert stdout == "" assert killed["called"] is True assert model == "codex-default" + assert outcome["error_type"] == "timeout" class TestRolloutMCPServers: @@ -518,9 +520,13 @@ async def fake_run_codex(instruction, system_prompt=None, mcp_servers=None, **kw captured["instruction"] = instruction captured["mcp_servers"] = mcp_servers captured["config"] = agent._build_config("http://x/v1", mcp_servers=mcp_servers) - return _item_completed( - {"id": "item_1", "type": "agent_message", "text": "The weather in Paris is sunny and 72 F."} - ), "codex-default" + return ( + _item_completed( + {"id": "item_1", "type": "agent_message", "text": "The weather in Paris is sunny and 72 F."} + ), + "codex-default", + {"status": "completed"}, + ) agent.server_client.post.side_effect = fake_post object.__setattr__(agent, "_run_codex", fake_run_codex) @@ -563,7 +569,11 @@ async def fake_post(server_name, url_path, json=None, cookies=None): async def fake_run_codex(instruction, system_prompt=None, mcp_servers=None, **kwargs): captured["mcp_servers"] = mcp_servers - return _item_completed({"id": "item_1", "type": "agent_message", "text": "ok"}), "codex-default" + return ( + _item_completed({"id": "item_1", "type": "agent_message", "text": "ok"}), + "codex-default", + {"status": "completed"}, + ) agent.server_client.post.side_effect = fake_post object.__setattr__(agent, "_run_codex", fake_run_codex) @@ -855,3 +865,49 @@ def test_config_yaml_parses(self) -> None: assert inner["entrypoint"] == "app.py" assert inner["concurrency"] == 32 assert inner["sandbox_mode"] == "danger-full-access" + + +def test_partial_codex_output_preserves_trajectory_and_marks_response_failed() -> None: + agent = _make_agent() + stdout = "\n".join( + [ + _item_completed({"id": "msg-1", "type": "agent_message", "text": "partial answer"}), + json.dumps({"type": "turn.failed", "error": {"message": "stream disconnected"}}), + ] + ) + + async def fake_run_codex(*args, **kwargs): + return stdout, "codex-default", {"status": "completed"} + + object.__setattr__(agent, "_run_codex", fake_run_codex) + response = asyncio.run(agent._create_response(NeMoGymResponseCreateParamsNonStreaming(input="perform the task"))) + + assert response.output + assert response.status == "failed" + assert response.error is not None + assert "stream_error" in response.error.message + + +@pytest.mark.parametrize("reported_in", ["stdout", "stderr"]) +def test_codex_context_limit_output_is_a_scoreable_incomplete_outcome(reported_in: str) -> None: + agent = _make_agent() + partial = _item_completed({"id": "msg-1", "type": "agent_message", "text": "partial answer"}) + context_error = "maximum context length exceeded" + stdout = partial + metadata = {"status": "failed", "error_type": "process_exit_1", "stderr": ""} + if reported_in == "stdout": + stdout += "\n" + json.dumps({"type": "turn.failed", "error": {"message": context_error}}) + else: + metadata["stderr"] = context_error + + async def fake_run_codex(*args, **kwargs): + return stdout, "codex-default", metadata + + object.__setattr__(agent, "_run_codex", fake_run_codex) + response = asyncio.run(agent._create_response(NeMoGymResponseCreateParamsNonStreaming(input="perform the task"))) + + assert response.output + assert response.status == "incomplete" + assert response.error is None + assert response.incomplete_details.reason == "max_output_tokens" + assert response.metadata == {"nemo_gym_stop_reason": "context_limit"} diff --git a/responses_api_agents/harbor_agent/app.py b/responses_api_agents/harbor_agent/app.py index 43159df562..9ef748e9be 100644 --- a/responses_api_agents/harbor_agent/app.py +++ b/responses_api_agents/harbor_agent/app.py @@ -43,6 +43,7 @@ NeMoGymResponse, NeMoGymResponseCreateParamsNonStreaming, ) +from nemo_gym.rollout_collection import NG_FAILURE_CLASS_KEY, NG_TERMINAL_KEY from responses_api_agents.harbor_agent.utils import HarborAgentUtils @@ -111,6 +112,10 @@ class HarborAgentConfig(BaseResponsesAPIAgentConfig): # Keep Docker runtime images between trials (maps to EnvironmentConfig.delete=False). harbor_no_delete: bool = True + # Harbor normally proceeds to verification after an agent timeout. Benchmarks + # that cannot score partial agent output can opt out through a trial hook. + harbor_skip_verification_on_agent_failure: bool = False + # --- Model routing --- # NeMo Gym model server reference used to resolve Harbor model base URL. model_server: ModelServerRef @@ -134,7 +139,13 @@ def _find_trial_dir_with_result(job_dir: Path) -> Optional[Path]: return None -async def run_harbor_job(job_config_dict: dict) -> str: +async def _stop_verification_after_agent_failure(event: Any) -> None: + result = event.result + if result is not None and result.exception_info is not None: + raise RuntimeError(f"Skipping verification after agent failure: {result.exception_info.exception_type}") + + +async def run_harbor_job(job_config_dict: dict, skip_verification_on_agent_failure: bool = False) -> str: """Runs a single Harbor Job and returns the *absolute* trial directory path. The trial directory contains: @@ -164,6 +175,8 @@ async def run_harbor_job(job_config_dict: dict) -> str: config = JobConfig(**job_config_dict) job = Job(config) + if skip_verification_on_agent_failure: + job.on_verification_started(_stop_verification_after_agent_failure) job_error = None try: @@ -189,7 +202,7 @@ async def run_harbor_job(job_config_dict: dict) -> str: _RAY_WORKER_EVENT_LOOP: Optional[asyncio.AbstractEventLoop] = None -def _run_harbor_job_sync(job_config_dict: dict) -> str: +def _run_harbor_job_sync(job_config_dict: dict, skip_verification_on_agent_failure: bool = False) -> str: """Synchronous wrapper for run_harbor_job for use in Ray remote. Ray workers are long-lived processes. Reusing a single event loop per worker @@ -200,7 +213,9 @@ def _run_harbor_job_sync(job_config_dict: dict) -> str: if _RAY_WORKER_EVENT_LOOP is None or _RAY_WORKER_EVENT_LOOP.is_closed(): _RAY_WORKER_EVENT_LOOP = asyncio.new_event_loop() asyncio.set_event_loop(_RAY_WORKER_EVENT_LOOP) - return _RAY_WORKER_EVENT_LOOP.run_until_complete(run_harbor_job(job_config_dict)) + return _RAY_WORKER_EVENT_LOOP.run_until_complete( + run_harbor_job(job_config_dict, skip_verification_on_agent_failure) + ) @ray.remote( @@ -236,17 +251,66 @@ def setup_webserver(self) -> FastAPI: async def responses(self, body: NeMoGymResponseCreateParamsNonStreaming = Body()) -> NeMoGymResponse: raise NotImplementedError + def _terminal_failure_response( + self, + body: HarborRunRequest, + *, + failure_class: str, + reason: str, + model_name: str, + ) -> HarborVerifyResponse: + response = HarborAgentUtils.get_default_response_object() + response["model"] = model_name + response["output"] = [] + return HarborVerifyResponse.model_validate( + { + "responses_create_params": body.responses_create_params.model_dump(mode="json"), + "reward": 0.0, + "response": response, + "instance_id": body.instance_id, + "mask_sample": True, + "task_failed": failure_class == "task_failed", + "configuration_failed": failure_class == "configuration_failed", + "failure_reason": reason, + NG_FAILURE_CLASS_KEY: failure_class, + NG_TERMINAL_KEY: True, + } + ) + async def run(self, body: HarborRunRequest) -> HarborVerifyResponse: async with self.sem: - global_config_dict = get_global_config_dict() - - policy_model_name = global_config_dict["policy_model_name"] - base_url = self._resolve_model_base_url(global_config_dict) + try: + global_config_dict = get_global_config_dict() + policy_model_name = global_config_dict["policy_model_name"] + base_url = self._resolve_model_base_url(global_config_dict) + except (KeyError, TypeError, ValueError) as exc: + return self._terminal_failure_response( + body, + failure_class="configuration_failed", + reason=f"{type(exc).__name__}: {exc}", + model_name=self.config.model_server.name, + ) run_timestamp = datetime.now(timezone.utc) run_id = self._build_run_id(run_timestamp) instance_id = body.instance_id - dataset_alias, task_name = self._parse_instance_id(instance_id) + try: + dataset_alias, task_name = self._parse_instance_id(instance_id) + except ValueError as exc: + return self._terminal_failure_response( + body, + failure_class="task_failed", + reason=f"{type(exc).__name__}: {exc}", + model_name=policy_model_name, + ) + if dataset_alias not in self.config.harbor_datasets: + available = ", ".join(sorted(self.config.harbor_datasets)) + return self._terminal_failure_response( + body, + failure_class="task_failed", + reason=f"Unknown dataset alias in instance_id: {dataset_alias!r}. Available aliases: [{available}]", + model_name=policy_model_name, + ) output_file_dir = self._get_results_output_dir(policy_model_name, dataset_alias, run_timestamp) jobs_dir = self._get_jobs_output_dir(policy_model_name, dataset_alias, run_timestamp) @@ -257,19 +321,28 @@ async def run(self, body: HarborRunRequest) -> HarborVerifyResponse: exclude_none=True, ) - job_config_dict = self._build_job_config( - dataset_alias, - task_name, - policy_model_name, - base_url, - job_name=job_name, - jobs_dir=jobs_dir, - responses_create_params=responses_create_params, - ) + try: + job_config_dict = self._build_job_config( + dataset_alias, + task_name, + policy_model_name, + base_url, + job_name=job_name, + jobs_dir=jobs_dir, + responses_create_params=responses_create_params, + ) + except (KeyError, TypeError, ValueError) as exc: + return self._terminal_failure_response( + body, + failure_class="configuration_failed", + reason=f"{type(exc).__name__}: {exc}", + model_name=policy_model_name, + ) try: params = dict( job_config_dict=job_config_dict, + skip_verification_on_agent_failure=self.config.harbor_skip_verification_on_agent_failure, ) runner = runner_ray_remote if self.config.harbor_ray_task_num_cpus is not None: @@ -297,6 +370,73 @@ def _read_trial_files(): verifier_result = trial_result.get("verifier_result") reward = HarborAgentUtils.extract_reward(verifier_result) + failure_fields = {} + exception_info = trial_result.get("exception_info") or {} + agent_metadata = (trial_result.get("agent_result") or {}).get("metadata") or {} + context_limit_reached = bool(agent_error_flags.get("context_length_exceeded", False)) + memory_limit_reached = bool(agent_error_flags.get("memory_limit_exceeded", False)) + agent_started = trial_result.get("agent_execution") is not None + verifier_started = trial_result.get("verifier") is not None + failed_during_agent_phase = bool(exception_info and agent_started and not verifier_started) + agent_timed_out = bool( + exception_info.get("exception_type") == "AgentTimeoutError" + or agent_metadata.get("agent_timed_out", False) + ) + agent_failed = bool( + agent_metadata.get("agent_failed", False) or failed_during_agent_phase or agent_timed_out + ) + model_connection_failed = bool(agent_metadata.get("model_connection_failed", False)) + verifier_failed = bool( + (exception_info and verifier_started and not agent_failed) + or (not agent_failed and not context_limit_reached and verifier_result is None) + ) + sandbox_failed = bool( + memory_limit_reached or (exception_info and not agent_started and not verifier_started) + ) + + # Harbor exposes context exhaustion separately in + # context_length_exceeded_error. It is a scoreable incomplete + # outcome, not an operational failure. + if context_limit_reached: + agent_failed = False + model_connection_failed = False + agent_timed_out = False + + failure_reason = agent_metadata.get("model_error") + if agent_timed_out or verifier_failed or sandbox_failed: + failure_reason = exception_info.get("exception_message") or failure_reason + if memory_limit_reached and not failure_reason: + failure_reason = "Harbor sandbox exceeded its memory limit" + elif agent_failed and not failure_reason: + failure_reason = exception_info.get("exception_message") + + failure_class = None + if model_connection_failed: + failure_class = "model_connection_failed" + elif sandbox_failed: + failure_class = "sandbox_failed" + elif verifier_failed: + failure_class = "verifier_failed" + elif agent_timed_out: + failure_class = "agent_timed_out" + elif agent_failed: + failure_class = "agent_failed" + + mask_sample = failure_class is not None + if self.config.harbor_skip_verification_on_agent_failure or mask_sample: + failure_fields = { + "mask_sample": mask_sample, + "agent_failed": agent_failed, + "model_connection_failed": model_connection_failed, + "agent_timed_out": agent_timed_out, + "verifier_failed": verifier_failed, + "sandbox_failed": sandbox_failed, + "failure_reason": failure_reason, + } + if failure_class is not None: + failure_fields[NG_FAILURE_CLASS_KEY] = failure_class + reward = 0.0 + # Convert Harbor outputs to NeMo Gym response items: # keep rich trajectory details, then overlay rollout token details when present. output_items = HarborAgentUtils.trial_result_to_responses(trial_result, trajectory) @@ -316,6 +456,17 @@ def _read_trial_files(): input_messages = [] usage = None reward = 0.0 + failure_fields = {} + failure_fields = { + "mask_sample": True, + "agent_failed": False, + "model_connection_failed": False, + "agent_timed_out": False, + "verifier_failed": False, + "sandbox_failed": True, + "failure_reason": str(e), + NG_FAILURE_CLASS_KEY: "sandbox_failed", + } response = HarborAgentUtils.get_default_response_object() response["model"] = policy_model_name @@ -339,8 +490,10 @@ def _read_trial_files(): context_length_exceeded_error=int(agent_error_flags.get("context_length_exceeded", False)), memory_limit_exceeded_error=int(agent_error_flags.get("memory_limit_exceeded", False)), agent_timeout_error=int( - ((trial_result or {}).get("exception_info") or {}).get("exception_type") == "AgentTimeoutError" + not agent_error_flags.get("context_length_exceeded", False) + and ((trial_result or {}).get("exception_info") or {}).get("exception_type") == "AgentTimeoutError" ), + **failure_fields, ) # Save result to disk (folder = run_id, file = task name) diff --git a/responses_api_agents/harbor_agent/tests/test_app.py b/responses_api_agents/harbor_agent/tests/test_app.py index f7404bb57d..79c4a057f5 100644 --- a/responses_api_agents/harbor_agent/tests/test_app.py +++ b/responses_api_agents/harbor_agent/tests/test_app.py @@ -18,6 +18,7 @@ from contextlib import contextmanager from datetime import datetime, timezone from pathlib import Path +from types import SimpleNamespace from typing import Any, Dict, List, Optional from unittest.mock import MagicMock, patch @@ -29,6 +30,7 @@ HarborAgent, HarborAgentConfig, HarborRunRequest, + _stop_verification_after_agent_failure, ) from responses_api_agents.harbor_agent.utils import HarborAgentUtils @@ -273,6 +275,7 @@ def _make_run_request(instance_id="scientific::test_task_123", **kwargs) -> Harb def _harbor_run_mocks( trial_result: Optional[Dict[str, Any]] = None, trajectory: Optional[Dict[str, Any]] = None, + agent_error_flags: Optional[Dict[str, Any]] = None, side_effect: Optional[Exception] = None, ): """Patch external deps and wire up mocks for HarborAgent.run().""" @@ -296,10 +299,13 @@ async def _resolve_future(): else: trial_dir = tempfile.mkdtemp(prefix="harbor_trial_") (Path(trial_dir) / "result.json").write_text(json.dumps(trial_result or DEFAULT_TRIAL_RESULT)) - if trajectory is not None: + if trajectory is not None or agent_error_flags is not None: agent_dir = Path(trial_dir) / "agent" agent_dir.mkdir(parents=True, exist_ok=True) + if trajectory is not None: (agent_dir / "trajectory.json").write_text(json.dumps(trajectory)) + if agent_error_flags is not None: + (agent_dir / "agent_error_flags.json").write_text(json.dumps(agent_error_flags)) mock_ray.remote.side_effect = lambda *a, **k: _resolve_future() mock_ray.options.return_value.remote.side_effect = lambda *a, **k: _resolve_future() @@ -309,7 +315,7 @@ async def _run_inline(fn, *args, **kwargs): mock_to_thread.side_effect = _run_inline - yield + yield mock_ray # =========================================================================== @@ -443,6 +449,198 @@ async def test_run_failed_execution(self): assert len(response.response.output) == 0 assert response.responses_create_params.temperature == 0.3 assert response.responses_create_params.input == [] + assert response.mask_sample is True + assert response.sandbox_failed is True + assert response.model_dump()["_ng_failure_class"] == "sandbox_failed" + assert "_ng_failure_terminal" not in response.model_dump() + + async def test_run_masks_partial_agent_failure_and_ignores_verifier_reward(self): + server = _make_server(harbor_skip_verification_on_agent_failure=True) + trial_result = { + **DEFAULT_TRIAL_RESULT, + "agent_result": { + **DEFAULT_TRIAL_RESULT["agent_result"], + "metadata": { + "agent_failed": True, + "model_connection_failed": True, + "agent_timed_out": False, + "model_error": "adapter failed after partial output", + }, + }, + "exception_info": { + "exception_type": "LegalAgentBenchHarnessError", + "exception_message": "adapter failed after partial output", + }, + "agent_execution": {"started_at": "2026-01-01T00:00:00Z"}, + "verifier": None, + } + with _harbor_run_mocks(trial_result=trial_result, trajectory=DEFAULT_TRAJECTORY) as mock_ray: + response = await server.run(_make_run_request()) + + assert response.response.output + assert response.reward == 0.0 + assert response.mask_sample is True + assert response.agent_failed is True + assert response.model_connection_failed is True + assert response.agent_timed_out is False + assert response.failure_reason == "adapter failed after partial output" + assert response.model_dump()["_ng_failure_class"] == "model_connection_failed" + assert "_ng_failure_terminal" not in response.model_dump() + params = mock_ray.remote.call_args.args[1] + assert params["skip_verification_on_agent_failure"] is True + + async def test_verification_hook_rejects_agent_failure(self): + event = SimpleNamespace( + result=SimpleNamespace(exception_info=SimpleNamespace(exception_type="AgentTimeoutError")) + ) + + with pytest.raises(RuntimeError, match="Skipping verification after agent failure: AgentTimeoutError"): + await _stop_verification_after_agent_failure(event) + + await _stop_verification_after_agent_failure(SimpleNamespace(result=SimpleNamespace(exception_info=None))) + + async def test_run_masks_timeout_with_partial_output(self): + server = _make_server(harbor_skip_verification_on_agent_failure=True) + trial_result = { + **DEFAULT_TRIAL_RESULT, + "exception_info": { + "exception_type": "AgentTimeoutError", + "exception_message": "Agent execution timed out", + }, + "agent_execution": {"started_at": "2026-01-01T00:00:00Z"}, + "verifier": None, + } + with _harbor_run_mocks(trial_result=trial_result, trajectory=DEFAULT_TRAJECTORY): + response = await server.run(_make_run_request()) + + assert response.response.output + assert response.reward == 0.0 + assert response.mask_sample is True + assert response.agent_failed is True + assert response.agent_timed_out is True + assert response.failure_reason == "Agent execution timed out" + assert response.model_dump()["_ng_failure_class"] == "agent_timed_out" + assert "_ng_failure_terminal" not in response.model_dump() + + async def test_harbor_timeout_routes_to_retryable_failure_sidecar(self): + server = _make_server() + trial_result = { + **DEFAULT_TRIAL_RESULT, + "exception_info": { + "exception_type": "AgentTimeoutError", + "exception_message": "Agent execution timed out", + }, + "agent_execution": {"started_at": "2026-01-01T00:00:00Z"}, + "verifier": {"started_at": "2026-01-01T00:01:00Z"}, + } + with _harbor_run_mocks(trial_result=trial_result, trajectory=DEFAULT_TRAJECTORY): + response = await server.run(_make_run_request()) + + assert response.reward == 0.0 + assert response.agent_timeout_error == 1 + assert response.mask_sample is True + assert response.agent_failed is True + assert response.agent_timed_out is True + assert response.verifier_failed is False + assert response.model_dump()["_ng_failure_class"] == "agent_timed_out" + assert "_ng_failure_terminal" not in response.model_dump() + + async def test_harbor_verifier_failure_routes_to_retryable_failure_sidecar(self): + server = _make_server() + trial_result = { + **DEFAULT_TRIAL_RESULT, + "verifier_result": None, + "exception_info": { + "exception_type": "VerifierTimeoutError", + "exception_message": "Verifier execution timed out", + }, + "agent_execution": {"started_at": "2026-01-01T00:00:00Z"}, + "verifier": {"started_at": "2026-01-01T00:01:00Z"}, + } + with _harbor_run_mocks(trial_result=trial_result, trajectory=DEFAULT_TRAJECTORY): + response = await server.run(_make_run_request()) + + assert response.reward == 0.0 + assert response.mask_sample is True + assert response.agent_failed is False + assert response.verifier_failed is True + assert response.failure_reason == "Verifier execution timed out" + assert response.model_dump()["_ng_failure_class"] == "verifier_failed" + assert "_ng_failure_terminal" not in response.model_dump() + + async def test_harbor_context_limit_is_verified_and_scored(self): + server = _make_server() + trial_result = { + **DEFAULT_TRIAL_RESULT, + "exception_info": { + "exception_type": "AgentTimeoutError", + "exception_message": "maximum context length exceeded", + }, + "agent_execution": {"started_at": "2026-01-01T00:00:00Z"}, + "verifier": {"started_at": "2026-01-01T00:01:00Z"}, + } + with _harbor_run_mocks( + trial_result=trial_result, + trajectory=DEFAULT_TRAJECTORY, + agent_error_flags={"context_length_exceeded": True, "memory_limit_exceeded": False}, + ): + response = await server.run(_make_run_request()) + + assert response.reward == 1.0 + assert response.context_length_exceeded_error == 1 + assert response.agent_timeout_error == 0 + assert response.response.output + assert "mask_sample" not in response.model_dump() + assert "_ng_failure_class" not in response.model_dump() + assert "_ng_failure_terminal" not in response.model_dump() + + async def test_harbor_memory_limit_routes_as_retryable_sandbox_failure(self): + server = _make_server() + trial_result = { + **DEFAULT_TRIAL_RESULT, + "verifier_result": None, + "agent_execution": {"started_at": "2026-01-01T00:00:00Z"}, + "verifier": None, + } + with _harbor_run_mocks( + trial_result=trial_result, + trajectory=DEFAULT_TRAJECTORY, + agent_error_flags={"context_length_exceeded": False, "memory_limit_exceeded": True}, + ): + response = await server.run(_make_run_request()) + + assert response.reward == 0.0 + assert response.mask_sample is True + assert response.sandbox_failed is True + assert response.memory_limit_exceeded_error == 1 + assert response.failure_reason == "Harbor sandbox exceeded its memory limit" + assert response.model_dump()["_ng_failure_class"] == "sandbox_failed" + assert "_ng_failure_terminal" not in response.model_dump() + + async def test_invalid_instance_id_is_terminal(self): + server = _make_server() + with _harbor_run_mocks(): + response = await server.run(_make_run_request(instance_id="not-an-instance-id")) + + assert response.reward == 0.0 + assert response.mask_sample is True + assert response.task_failed is True + assert response.model_dump()["_ng_failure_class"] == "task_failed" + assert response.model_dump()["_ng_failure_terminal"] is True + + async def test_invalid_job_configuration_is_terminal(self): + server = _make_server() + with ( + patch("responses_api_agents.harbor_agent.app.get_global_config_dict", return_value=_GLOBAL_CONFIG), + patch.object(server, "_build_job_config", side_effect=ValueError("invalid dataset source")), + ): + response = await server.run(_make_run_request()) + + assert response.reward == 0.0 + assert response.mask_sample is True + assert response.configuration_failed is True + assert response.model_dump()["_ng_failure_class"] == "configuration_failed" + assert response.model_dump()["_ng_failure_terminal"] is True @pytest.mark.parametrize( "model_name, expected", diff --git a/responses_api_agents/hermes_agent/README.md b/responses_api_agents/hermes_agent/README.md index 76c6de5090..f4cdc543d0 100644 --- a/responses_api_agents/hermes_agent/README.md +++ b/responses_api_agents/hermes_agent/README.md @@ -41,10 +41,7 @@ Key metrics for math_with_judge_hermes_agent: "mean/reward": 0.2, "mean/turns_used": 1.6, "mean/finished_naturally": 1.0, - "mean/library_reward": 0.2, - "mean/input_tokens": 0.0, - "mean/output_tokens": 0.0, - "mean/total_tokens": 0.0 + "mean/library_reward": 0.2 } Finished rollout collection! View results at: Fully materialized inputs: responses_api_agents/hermes_agent/data/example_math_rollouts_materialized_inputs.jsonl @@ -52,6 +49,10 @@ Rollouts: responses_api_agents/hermes_agent/data/example_math_rollouts.jsonl Aggregate metrics: responses_api_agents/hermes_agent/data/example_math_rollouts_aggregate_metrics.json ``` +Current runs also report nonzero `mean/input_tokens`, +`mean/output_tokens`, and `mean/total_tokens` from Hermes' cumulative session +counters. + Example training reward for small multi environment test is shown [here](https://github.com/NVIDIA-NeMo/Gym/pull/1033#issuecomment-4399509664). ## Description diff --git a/responses_api_agents/hermes_agent/app.py b/responses_api_agents/hermes_agent/app.py index 4d28b7d44c..03cebcec5e 100644 --- a/responses_api_agents/hermes_agent/app.py +++ b/responses_api_agents/hermes_agent/app.py @@ -57,6 +57,48 @@ from responses_api_agents.hermes_agent.observability import HermesAgentObserver +def _is_context_limit_error(message: str) -> bool: + normalized = message.lower() + return any( + marker in normalized + for marker in ( + "context length", + "context window", + "context_length_exceeded", + "maximum context", + "max context", + "too many tokens", + "token limit", + ) + ) + + +def _usage_from_result(result: dict[str, Any]) -> NeMoGymResponseUsage: + """Map Hermes' cumulative session counters to the Responses API usage schema.""" + + def token_count(key: str) -> int: + try: + return max(0, int(result.get(key) or 0)) + except (TypeError, ValueError): + return 0 + + cache_read_tokens = token_count("cache_read_tokens") + cache_write_tokens = token_count("cache_write_tokens") + input_tokens = token_count("prompt_tokens") + if input_tokens == 0: + input_tokens = token_count("input_tokens") + cache_read_tokens + cache_write_tokens + output_tokens = token_count("output_tokens") or token_count("completion_tokens") + total_tokens = token_count("total_tokens") or input_tokens + output_tokens + + return NeMoGymResponseUsage( + input_tokens=input_tokens, + input_tokens_details=NeMoGymResponseInputTokensDetails(cached_tokens=cache_read_tokens), + output_tokens=output_tokens, + output_tokens_details=NeMoGymResponseOutputTokensDetails(reasoning_tokens=token_count("reasoning_tokens")), + total_tokens=total_tokens, + ) + + def _trajectory_to_output_items(messages, n_input): output_items = [] for item in messages[n_input:]: @@ -401,22 +443,31 @@ def _patched_build_api_kwargs(api_messages): ) ) + agent_error = result.get("error") + context_limit_reached = bool(agent_error) and _is_context_limit_error(str(agent_error)) + max_turns_reached = bool( + not agent_error + and result.get("completed") is False + and int(result.get("api_calls") or 0) >= self.config.max_turns + ) + limit_reached = context_limit_reached or max_turns_reached + stop_reason = "context_limit" if context_limit_reached else "max_turns" return NeMoGymResponse( id=f"resp_{uuid4().hex}", created_at=int(time()), + status="incomplete" if limit_reached else ("failed" if agent_error else "completed"), + error=( + {"code": "server_error", "message": str(agent_error)} if agent_error and not limit_reached else None + ), + incomplete_details=({"reason": "max_output_tokens"} if limit_reached else None), + metadata=({"nemo_gym_stop_reason": stop_reason} if limit_reached else None), model=model_name, object="response", output=output_items, tool_choice=body.tool_choice, tools=body.tools, parallel_tool_calls=body.parallel_tool_calls, - usage=NeMoGymResponseUsage( - input_tokens=0, - input_tokens_details=NeMoGymResponseInputTokensDetails(cached_tokens=0), - output_tokens=0, - output_tokens_details=NeMoGymResponseOutputTokensDetails(reasoning_tokens=0), - total_tokens=0, - ), + usage=_usage_from_result(result), ) async def responses( diff --git a/responses_api_agents/hermes_agent/tests/test_app.py b/responses_api_agents/hermes_agent/tests/test_app.py index ae0aacd3ec..0ac820150c 100644 --- a/responses_api_agents/hermes_agent/tests/test_app.py +++ b/responses_api_agents/hermes_agent/tests/test_app.py @@ -35,6 +35,7 @@ ResourcesServerRef, _split_input_to_user_and_history, _trajectory_to_output_items, + _usage_from_result, ) from responses_api_agents.hermes_agent.observability import HermesAgentObserver @@ -273,6 +274,43 @@ def test_skips_non_dict_items(self) -> None: assert len(out) == 1 +class TestUsageFromResult: + def test_maps_hermes_session_counters(self) -> None: + usage = _usage_from_result( + { + "input_tokens": 100, + "cache_read_tokens": 20, + "cache_write_tokens": 5, + "prompt_tokens": 125, + "output_tokens": 40, + "reasoning_tokens": 10, + "total_tokens": 165, + } + ) + + assert usage.input_tokens == 125 + assert usage.input_tokens_details.cached_tokens == 20 + assert usage.output_tokens == 40 + assert usage.output_tokens_details.reasoning_tokens == 10 + assert usage.total_tokens == 165 + + def test_reconstructs_totals_when_aggregate_counters_are_missing(self) -> None: + usage = _usage_from_result( + { + "input_tokens": 100, + "cache_read_tokens": 20, + "cache_write_tokens": 5, + "completion_tokens": 40, + } + ) + + assert usage.input_tokens == 125 + assert usage.input_tokens_details.cached_tokens == 20 + assert usage.output_tokens == 40 + assert usage.output_tokens_details.reasoning_tokens == 0 + assert usage.total_tokens == 165 + + class TestRolloutCorrelation: def test_responses_applies_rollout_prefix(self, monkeypatch) -> None: from fastapi.testclient import TestClient @@ -296,7 +334,12 @@ def __init__(self, **kwargs) -> None: self.compression_enabled = True def run_conversation(self, *args, **kwargs) -> dict: - return {"messages": [{"role": "assistant", "content": "ok"}]} + return { + "messages": [{"role": "assistant", "content": "ok"}], + "prompt_tokens": 12, + "output_tokens": 3, + "total_tokens": 15, + } monkeypatch.setattr("run_agent.AIAgent", _StubAIAgent) client = TestClient(agent.setup_webserver()) @@ -307,6 +350,9 @@ def run_conversation(self, *args, **kwargs) -> dict: direct = asyncio.run(agent.responses(request=None, body=NeMoGymResponseCreateParamsNonStreaming(input="hi"))) assert seen["base_url"] == "http://h:1/v1" assert "_ng_agent_observations" not in direct.model_dump(mode="json") + assert direct.usage.input_tokens == 12 + assert direct.usage.output_tokens == 3 + assert direct.usage.total_tokens == 15 episode = asyncio.run( agent._create_episode( @@ -318,6 +364,80 @@ def run_conversation(self, *args, **kwargs) -> dict: assert episode.observations.source == "hermes" assert episode.observations.records[0].invocation_id == "root" + def test_partial_output_preserves_trajectory_and_marks_response_failed(self, monkeypatch) -> None: + import nemo_gym.base_responses_api_agent as base_agent + from nemo_gym.openai_utils import NeMoGymResponseCreateParamsNonStreaming + + monkeypatch.setattr(base_agent, "get_first_server_config_dict", lambda _gc, _name: {"host": "h", "port": 1}) + server_client = MagicMock(spec=ServerClient) + server_client.global_config_dict = {} + server_client._build_server_base_url = lambda _cfg: "http://h:1" + agent = HermesAgent(config=_config(), server_client=server_client) + monkeypatch.setattr(agent, "_ensure_sigterm_handler", lambda: None) + + class _StubAIAgent: + def __init__(self, **kwargs) -> None: + self._build_api_kwargs = lambda _messages: {} + self.compression_enabled = True + + def run_conversation(self, *args, **kwargs) -> dict: + return { + "messages": [{"role": "assistant", "content": "partial answer"}], + "error": "model stream disconnected", + "prompt_tokens": 12, + "output_tokens": 3, + "total_tokens": 15, + } + + monkeypatch.setattr("run_agent.AIAgent", _StubAIAgent) + response = asyncio.run(agent.responses(request=None, body=NeMoGymResponseCreateParamsNonStreaming(input="hi"))) + + assert response.output + assert response.status == "failed" + assert response.error is not None + assert "stream disconnected" in response.error.message + + @pytest.mark.parametrize( + ("result_update", "stop_reason"), + [ + ({"completed": False, "api_calls": 30}, "max_turns"), + ({"error": "maximum context length exceeded"}, "context_limit"), + ], + ) + def test_limit_output_is_a_scoreable_incomplete_outcome(self, monkeypatch, result_update, stop_reason) -> None: + import nemo_gym.base_responses_api_agent as base_agent + from nemo_gym.openai_utils import NeMoGymResponseCreateParamsNonStreaming + + monkeypatch.setattr(base_agent, "get_first_server_config_dict", lambda _gc, _name: {"host": "h", "port": 1}) + server_client = MagicMock(spec=ServerClient) + server_client.global_config_dict = {} + server_client._build_server_base_url = lambda _cfg: "http://h:1" + agent = HermesAgent(config=_config(max_turns=30), server_client=server_client) + monkeypatch.setattr(agent, "_ensure_sigterm_handler", lambda: None) + + class _StubAIAgent: + def __init__(self, **kwargs) -> None: + self._build_api_kwargs = lambda _messages: {} + self.compression_enabled = True + + def run_conversation(self, *args, **kwargs) -> dict: + return { + "messages": [{"role": "assistant", "content": "partial answer"}], + "prompt_tokens": 12, + "output_tokens": 3, + "total_tokens": 15, + **result_update, + } + + monkeypatch.setattr("run_agent.AIAgent", _StubAIAgent) + response = asyncio.run(agent.responses(request=None, body=NeMoGymResponseCreateParamsNonStreaming(input="hi"))) + + assert response.output + assert response.status == "incomplete" + assert response.error is None + assert response.incomplete_details.reason == "max_output_tokens" + assert response.metadata == {"nemo_gym_stop_reason": stop_reason} + class TestObservability: @pytest.mark.parametrize( diff --git a/responses_api_agents/legal_agent_bench_agent/.gitignore b/responses_api_agents/legal_agent_bench_agent/.gitignore new file mode 100644 index 0000000000..86e51f5891 --- /dev/null +++ b/responses_api_agents/legal_agent_bench_agent/.gitignore @@ -0,0 +1,3 @@ +.deps/ +results/ +__pycache__/ diff --git a/responses_api_agents/legal_agent_bench_agent/README.md b/responses_api_agents/legal_agent_bench_agent/README.md new file mode 100644 index 0000000000..f4e3a19362 --- /dev/null +++ b/responses_api_agents/legal_agent_bench_agent/README.md @@ -0,0 +1,176 @@ +# Legal Agent Bench Configurable Agent + +This task-driven Gym agent runs a configured Gym Responses API agent inside a +Legal Agent Bench (LAB) sandbox. It keeps the benchmark task set, skills, and +verifier fixed while allowing the agent harness and sandbox provider to vary. + +The default configuration selects LAB's direct Gym-native loop. Additional +configurations select Gym's Hermes, Claude Code, and Codex agents through: + +- `agent_server_module` +- `agent_server_class` +- `agent_config_class` +- `agent_kwargs` + +The runner also accepts the policy `model_server`, LAB runtime task and skill +paths, concurrency and layered timeout settings (including +`sandbox_staging_timeout_seconds` for large archive extraction and collection), `sandbox_provider`, an optional +`sandbox_image`, a model URL override, Docker-only network settings, +`opensandbox_request_fraction`, and a result directory. See `configs/` for +complete examples. +Those configurations include the LAB `resources_only.yaml` definition so Gym +starts one resource server and only the selected agent; the combined +compatibility config is reserved for Harbor. + +## Execution boundary + +For each rollout, the runner validates `instance_id`, resolves a +provider-compatible LAB image, provisions a portable Linux runtime for the +selected harness through that same sandbox provider, and starts an agent-only +sandbox. Docker can build the LAB image automatically as a local convenience; +every other provider receives the configured image reference unchanged. +Runtimes are immutable and content-addressed by their setup script, +requirements, configured CLI pin, LAB image and provider identity, Gym +packaging metadata, and installed `nemo_gym` source. An interprocess lock and +atomic publication prevent concurrent evaluations from rewriting a runtime +that is being built or used. + +Dependency provisioning exposes an explicit allowlist containing only the +installer and required Gym package files. It never mounts the repository, +`env.yaml`, task tests, judge credentials, or unrelated files. + +Agent source, documents, skills, runtime, and runner inputs are transferred +through Gym's sandbox file API. The sandbox has no writable host mounts. + +Only the task instructions and public skill manuals are supplied to the +configured agent. The native choice reproduces upstream LAB's model → function +tools → tool results loop using Gym's Responses API; it uses LAB's canonical +`bash`, `read`, `write`, `write_docx`, `edit`, `glob`, and `grep` tools. Its +results are downloaded to a private temporary directory, +the sandbox is destroyed, and links, devices, traversal paths, and other unsafe +archive entries are rejected before Gym creates host artifacts. A fresh +verifier-only sandbox then receives a separately staged, sanitized `lab-run` +tree, rubric tests, and verifier-only judge credentials. The verifier sees +deliverables at `lab-run/output`, preserving LAB's contract without sharing a +process namespace or writable filesystem with the agent. + +The default provider is Docker, but the common path uses only Gym's +`AsyncSandbox` operations: start, upload, execute, download, and stop. Docker, +ECS Fargate, Enroot, Apptainer, OpenSandbox, Daytona, and OpenShell can all be +selected with `sandbox_provider` without changing LAB code. The runtime +builder, agent, and verifier use the same selected provider. No common command +requests root, mounts a host path, invokes a container CLI, interprets the +provider's image format, or assumes the image's configured user. Transfer +archives are made readable to non-root image users, and all large runtime, +scratch, and transfer writes stay under the sandbox's writable `/sandbox` +tree rather than a provider's potentially bounded `/tmp`. +LAB's generated image also includes OpenShell's Docker-driver prerequisites: +`iproute2`, a high-UID `sandbox` identity, and a writable work directory. + +Docker's automatic image build and host-network translation are optional +provider conveniences. ECS uses Gym's provider-native SSH reverse tunnel to +expose the rollout-scoped policy proxy as `LAB_POLICY_MODEL_URL`, keeping model +credentials out of the sandbox. Enroot and Apptainer share the orchestrator's +host network. OpenSandbox, Daytona, and OpenShell need a +`sandbox_model_base_url` that is reachable from their remote sandbox whenever +Gym's derived policy-proxy URL is host-local. An explicit +`sandbox_model_base_url` is always used unchanged. A reachable credential-free +Gym proxy is preferred. For a directly authenticated endpoint, +`sandbox_model_api_key_env` may name a launcher environment variable whose +value is copied into the agent sandbox as `LAB_POLICY_API_KEY`. It is not +serialized into runner metadata or supplied to the runtime builder or verifier, +but the evaluated agent can read its own environment. Use this fallback only +with a narrowly scoped, short-lived key and rotate it after the run. Before +importing the selected harness, the in-sandbox runner +performs a bounded, proxy-aware HTTP connectivity check and writes +`runtime/runner_status.json`. Connectivity, harness, sandbox, and verifier +failures set `mask_sample`, skip the judge when applicable, and carry +`_ng_failure_class` so rollout collection sends them to the failure sidecar for +bounded retry. Deterministic task and harness-configuration failures also carry +`_ng_failure_terminal: true` and are not retried. `mask_sample` is a training +hint, not the routing signal. Max-turn and context-limit stops are valid +incomplete outcomes: the runner verifies their partial deliverables and keeps +them in the main rollout JSONL. For Hermes, the runner also disables its optional `/models` +pricing and context-metadata probes; Gym supplies the selected model explicitly, +and actual model-call access logging is enabled. + +For OpenSandbox, each LAB task's declared resources are the burst limits. By +default, `opensandbox_request_fraction: 0.25` sets CPU and memory scheduling +requests to 25% of those limits so concurrent evaluations can pack densely. +Disk and any requested GPU are not oversubscribed. The same split is applied to +the runtime builder, agent, and verifier sandboxes. This is the same ratio used +by Gym's +[Mini SWE Agent 2 config](../mini_swe_agent_2/configs/mini_swe_agent_2.yaml), +where 0.5 CPU and 2 GiB are requested against limits of 2 CPU and 8 GiB. Set +the value closer to `1.0` when the cluster needs stronger reservations, or to +`null` to let the OpenSandbox server use one resource map for both requests +and limits. + +Codex currently has the largest portable runtime archive. On a remote +OpenSandbox deployment, keep the provider's +`connection.request_timeout_s` at least as high as the runner's 900-second +staging timeout and reduce concurrency if parallel file transfers saturate the +OpenSandbox file API. These deadlines cover infrastructure transfer and +staging; they do not change the model's agent-phase deadline. + +Providers that enforce different network access by phase can use +`runtime_builder_provider_options`, `agent_sandbox_provider_options`, and +`verifier_sandbox_provider_options`. For OpenShell, use separate least-privilege +policies: allow only package registries in the runtime builder, only the Gym +policy proxy in the agent, and only the judge endpoint in the verifier. The LAB +image's global Node document libraries are exposed through an explicit +`NODE_PATH`; OpenShell's injected HTTP(S) proxy is trusted only inside its inner +runner so policy-model traffic passes through the gateway policy engine. The +builder's `registry.npmjs.org` endpoint must use `protocol: rest`, `access: +read-only`, `enforcement: enforce`, and `allow_encoded_slash: true`, because npm +uses encoded slashes for scoped packages. + +The checked-in configs decode the whole phase-option maps from +`NEMO_GYM_LAB_RUNTIME_BUILDER_PROVIDER_OPTIONS`, +`NEMO_GYM_LAB_AGENT_SANDBOX_PROVIDER_OPTIONS`, and +`NEMO_GYM_LAB_VERIFIER_SANDBOX_PROVIDER_OPTIONS`. This avoids inherited-config +merge restrictions when a benchmark variant needs to add provider-specific +keys such as `policy`; each setting defaults to `{}`. + +Artifacts are written below +`results/legal_agent_bench/_jobs///_/`. +For example, the default native loop uses +`results/legal_agent_bench/native_jobs`, while Hermes uses +`results/legal_agent_bench/hermes_jobs`. The model +segment is the configured `policy_model_name`, normalized into one safe path +segment. The task name is also normalized, and the run ID is an eight-character +unique suffix. A session directory is created only when its first rollout starts. +Each trial includes the inner Gym trajectory, harness logs, LAB run +configuration and metrics, completed output files, verifier artifacts, and a +top-level `run_summary.json`. The rollout response also includes direct paths +for the summary, trajectory, stdout, stderr, output directory, and verifier +report. + +Set `NEMO_GYM_LAB_RESULTS_DIR` to redirect the artifact root. This is useful +when Gym runs inside a Linux VM: use the VM's native filesystem instead of a +macOS-shared mount to avoid cross-OS permissions and ownership behavior. + +For setup, five copy-paste smoke commands, and result inspection, see the +[benchmark README](../../benchmarks/legal_agent_bench/README.md#test-the-various-harnesses). +Run `gym` from an activated repository environment rather than prefixing +server-starting commands with `uv run`. + +## Adding a harness + +A harness can be selected without changing this runner when it: + +1. is implemented as a Gym Responses API agent; +2. can run inside the LAB Linux sandbox image; +3. has a portable dependency script at + `responses_api_agents//scripts/_deps.sh`; and +4. can use the configured OpenAI-compatible policy endpoint. + +Add a standalone configuration under `configs/`, then add a benchmark variant +that inherits it. CLI versions belong in the harness configuration's +`agent_kwargs`; provisioning derives its exact package specification from that +pin and rejects unpinned Claude Code or Codex configurations. + +Docker is the zero-configuration local backend. See the benchmark README for +the provider matrix, image and networking requirements, lifecycle smoke test, +and one-task override examples. The separate Harbor compatibility variant does +not use this runner and is Docker-only. diff --git a/responses_api_agents/legal_agent_bench_agent/__init__.py b/responses_api_agents/legal_agent_bench_agent/__init__.py new file mode 100644 index 0000000000..353bc52e78 --- /dev/null +++ b/responses_api_agents/legal_agent_bench_agent/__init__.py @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Configurable Gym-agent runner for Legal Agent Bench.""" diff --git a/responses_api_agents/legal_agent_bench_agent/app.py b/responses_api_agents/legal_agent_bench_agent/app.py new file mode 100644 index 0000000000..430643c96f --- /dev/null +++ b/responses_api_agents/legal_agent_bench_agent/app.py @@ -0,0 +1,1945 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Run a config-selected Gym agent inside a Legal Agent Bench task sandbox.""" + +from __future__ import annotations + +import asyncio +import fcntl +import hashlib +import json +import math +import os +import re +import shutil +import stat +import sys +import tarfile +import tempfile +import time +from collections.abc import Mapping +from pathlib import Path, PurePosixPath +from typing import Any, Optional +from urllib.parse import urlsplit, urlunsplit +from uuid import uuid4 + + +# Server processes may run from a component-local working directory with an +# installed ``nemo_gym`` ahead of the checkout on sys.path. Derive the source +# root from this file and put it first so LAB's vendored runtime assets are +# resolved from the same checkout as the agent implementation on every host. +PACKAGE_DIR = Path(__file__).resolve().parent +PARENT_DIR = PACKAGE_DIR.parents[1] +if str(PARENT_DIR) not in sys.path: + sys.path.insert(0, str(PARENT_DIR)) + +from fastapi import Body, Request +from pydantic import ConfigDict, Field, PrivateAttr + +from nemo_gym.base_resources_server import BaseRunRequest, BaseVerifyResponse +from nemo_gym.base_responses_api_agent import BaseResponsesAPIAgentConfig, SimpleResponsesAPIAgent +from nemo_gym.config_types import ModelServerRef, ResourcesServerRef +from nemo_gym.global_config import get_first_server_config_dict +from nemo_gym.openai_utils import NeMoGymResponse, NeMoGymResponseCreateParamsNonStreaming +from nemo_gym.rollout_collection import NG_FAILURE_CLASS_KEY, NG_TERMINAL_KEY +from nemo_gym.sandbox import ( + AsyncSandbox, + SandboxResources, + SandboxSpec, + resolve_provider_config, + resolve_provider_metadata, +) +from nemo_gym.server_utils import apply_rollout_prefix +from resources_servers.legal_agent_bench.prepare import ( + DEFAULT_RUNTIME_TASKS_DIR, + DEFAULT_SKILLS_DIR, + REQUIRED_SKILLS, + resolve_repo_path, + validate_harness_skills, +) +from resources_servers.legal_agent_bench.vendor.harvey_labs.lab_harbor.tools import ( + get_all_tool_definitions, +) + + +PORTABLE_PYTHON_SH = PACKAGE_DIR / "setup_scripts" / "_portable_python.sh" +DATASET_ALIAS = "legal_agent_bench" +INITIAL_USER_PROMPT = "Please begin working on the task described in the system prompt." +NATIVE_AGENT_MODULE = "responses_api_agents.legal_agent_bench_native_agent.app" +AGENT_FAILURE_CLASS_METADATA_KEY = "nemo_gym_failure_class" +PROPAGATED_AGENT_FAILURE_CLASSES = frozenset({"agent_timed_out", "model_connection_failed"}) +LAB_SYSTEM_PROMPT = ( + PARENT_DIR / "resources_servers" / "legal_agent_bench" / "vendor" / "harvey_labs" / "harness" / "system-prompt.md" +).read_text(encoding="utf-8") +AGENT_CLI_PINS = { + "claude_code_agent": ("claude_code_version", "CLAUDE_SPEC", "@anthropic-ai/claude-code"), + "codex_agent": ("codex_version", "CODEX_SPEC", "@openai/codex"), +} +PINNED_NPM_VERSION = re.compile( + r"^(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)" + r"(?:-[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?" + r"(?:\+[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?$" +) +SANDBOX_ROOT = "/sandbox/nemo-gym-legal-agent-bench" +SANDBOX_AGENT_SOURCE = f"{SANDBOX_ROOT}/agent_source" +SANDBOX_AGENT_DEPS = f"{SANDBOX_ROOT}/agent_deps" +SANDBOX_RUNTIME = f"{SANDBOX_ROOT}/runtime" +SANDBOX_WORKSPACE = f"{SANDBOX_ROOT}/workspace" +SANDBOX_VDR = f"{SANDBOX_WORKSPACE}/vdr" +SANDBOX_OUTPUT = f"{SANDBOX_WORKSPACE}/output" +SANDBOX_SCRATCH = f"{SANDBOX_WORKSPACE}/scratch" +SANDBOX_SKILLS = f"{SANDBOX_WORKSPACE}/skills" +SANDBOX_VERIFIER = f"{SANDBOX_ROOT}/verifier" +SANDBOX_TESTS = f"{SANDBOX_VERIFIER}/tests" +SANDBOX_LOGS = f"{SANDBOX_VERIFIER}/logs" +LAB_SANDBOX_ENV = {"NODE_PATH": "/usr/local/lib/node_modules"} +CODEX_MODEL_CATALOG_PATH = f"{SANDBOX_RUNTIME}/codex_model_catalog.json" +CODEX_MODEL_CATALOG = { + "models": [ + { + "slug": "gym-policy-model", + "display_name": "Gym policy model", + "description": "Model selected by the configured Gym policy server.", + "supported_reasoning_levels": [], + "shell_type": "default", + "visibility": "none", + "supported_in_api": True, + "priority": 99, + "availability_nux": None, + "upgrade": None, + "base_instructions": "", + "supports_reasoning_summaries": True, + "support_verbosity": False, + "default_verbosity": None, + "apply_patch_tool_type": None, + "truncation_policy": {"mode": "bytes", "limit": 10_000}, + "supports_parallel_tool_calls": False, + "context_window": 272_000, + "max_context_window": 272_000, + "experimental_supported_tools": [], + } + ] +} + + +def codex_model_catalog(model_name: str) -> dict[str, Any]: + """Return Codex metadata for both proxied and direct policy routing. + + Codex indexes the catalog by the configured model slug. Gym's in-process + proxy uses ``gym-policy-model``, while a direct remote endpoint requires the + provider's real model name. Keep the proxy alias and add the selected model + as a second entry so one portable runtime works for both routes. + """ + catalog = json.loads(json.dumps(CODEX_MODEL_CATALOG)) + if model_name != catalog["models"][0]["slug"]: + direct_model = json.loads(json.dumps(catalog["models"][0])) + direct_model["slug"] = model_name + direct_model["display_name"] = model_name + catalog["models"].append(direct_model) + return catalog + + +GENERIC_HARNESS_PREAMBLE = """\ +You are an AI agent running in an automated Legal Agent Bench evaluation. + +## Workspace layout + +- Source documents are under `$VDR_DIR`. Treat them as read-only. +- Write every final deliverable under `$OUTPUT_DIR`. +- Use `$WORKSPACE_DIR` for scratch files. +- Skill manuals and their supporting assets are under `$SKILLS_DIR`. +- Do not search for task configuration, rubric, verifier, test, or judge files. They are intentionally + unavailable while you work. + +Use the terminal and filesystem tools provided by your agent harness. The image already contains the +document tooling required by the skills; do not install packages during the task. Read the skill manuals +included below before creating deliverables. +""" + +_RUNNER_SOURCE = r"""#!/usr/bin/env python3 +import asyncio +import inspect +import json +import os +import sys +from pathlib import Path +from types import SimpleNamespace +from urllib.error import HTTPError +from urllib.parse import urlsplit, urlunsplit +from urllib.request import Request, urlopen + +SANDBOX_ROOT = "/sandbox/nemo-gym-legal-agent-bench" +AGENT_SOURCE = f"{SANDBOX_ROOT}/agent_source" +AGENT_DEPS = f"{SANDBOX_ROOT}/agent_deps" +RUNTIME = f"{SANDBOX_ROOT}/runtime" +WORKSPACE = f"{SANDBOX_ROOT}/workspace" + +sys.path.insert(0, AGENT_SOURCE) +os.environ["PATH"] = os.environ.get("PATH", "") + os.pathsep + f"{AGENT_DEPS}/bin" +os.environ["VDR_DIR"] = f"{WORKSPACE}/vdr" +os.environ["OUTPUT_DIR"] = f"{WORKSPACE}/output" +os.environ["WORKSPACE_DIR"] = f"{WORKSPACE}/scratch" +os.environ["SKILLS_DIR"] = f"{WORKSPACE}/skills" +os.environ["HOME"] = f"{WORKSPACE}/scratch" +os.environ["TMPDIR"] = f"{WORKSPACE}/scratch" +# The inner runner receives its complete configuration below. Prevent Gym's +# shared aiohttp client from invoking Hydra's CLI config loader in the output +# directory when the native agent makes its first model request. +os.environ.setdefault("NEMO_GYM_CONFIG_DICT", "{}") +os.chdir(os.environ["OUTPUT_DIR"]) + +from nemo_gym.config_types import ModelServerRef, ResourcesServerRef +from nemo_gym.openai_utils import NeMoGymResponseCreateParamsNonStreaming +from nemo_gym.server_utils import ( + GlobalAIOHTTPAsyncClientConfig, + ServerClient, + is_global_aiohttp_client_setup, + set_global_aiohttp_client, +) + +runner = json.loads(Path(f"{RUNTIME}/runner.json").read_text()) +# ECS Fargate injects this value after resolving the host-side policy URL through +# its SSH reverse tunnel. Other providers use the URL persisted in runner.json. +model_url = os.environ.get("LAB_POLICY_MODEL_URL", runner["model_url"]).rstrip("/") +model_api_key = os.environ.get("LAB_POLICY_API_KEY") +model_url_root = model_url.removesuffix("/v1").rstrip("/") +model_url_v1 = model_url if model_url.endswith("/v1") else model_url + "/v1" +status_path = Path(f"{RUNTIME}/runner_status.json") + + +def write_status(*, ok, phase, error=None): + status_path.write_text(json.dumps({"ok": ok, "phase": phase, "error": error}, indent=2)) + + +try: + parsed_model_url = urlsplit(model_url) + if not parsed_model_url.hostname: + raise ValueError(f"Policy model URL has no hostname: {model_url!r}") + # Use a proxy-aware HTTP request rather than a raw TCP socket. Policy-enforcing + # providers such as OpenShell expose permitted egress through HTTP(S)_PROXY and + # intentionally block direct sockets. Strip only the terminal API-version segment + # so Gym policy proxies can answer their quiet 200 liveness route while preserving + # any rollout-scoped path prefix. + probe_path = parsed_model_url.path.removesuffix("/v1") or "/" + if not probe_path.endswith("/"): + probe_path += "/" + probe_url = urlunsplit(parsed_model_url._replace(path=probe_path, query="", fragment="")) + try: + probe_headers = {"Authorization": f"Bearer {model_api_key}"} if model_api_key else {} + with urlopen( + Request(probe_url, headers=probe_headers, method="GET"), + timeout=runner.get("model_connect_timeout_seconds", 10), + ): + pass + except HTTPError as exc: + # Any application response proves transport connectivity, but authentication + # or policy denials mean the agent cannot use the endpoint. + if exc.code in {401, 403}: + raise +except Exception as exc: + message = f"Policy model is unreachable from the LAB sandbox: {model_url} ({type(exc).__name__}: {exc})" + write_status(ok=False, phase="model_connectivity", error=message) + raise RuntimeError(message) from exc + +try: + module = __import__( + runner["agent_server_module"], + fromlist=[runner["agent_server_class"], runner["agent_config_class"]], + ) + agent_class = getattr(module, runner["agent_server_class"]) + config_class = getattr(module, runner["agent_config_class"]) + + if runner.get("disable_endpoint_metadata_probe"): + # Hermes probes /models for optional pricing and context metadata. + # Gym already supplies the selected model and its internal proxy does not + # expose model discovery, so skip only these optional metadata lookups. + from agent import model_metadata as hermes_model_metadata + from agent import usage_pricing as hermes_usage_pricing + + def empty_endpoint_model_metadata(*args, **kwargs): + return {} + + hermes_model_metadata.fetch_endpoint_model_metadata = empty_endpoint_model_metadata + # Current Hermes releases perform a second, optional local-server + # context probe when endpoint metadata is empty. Gym's rollout proxy is + # neither Ollama, LM Studio, llama.cpp, nor a public model catalog, so + # probing those routes only emits misleading 404s. Preserve Hermes's + # normal fallback context behavior without issuing discovery requests. + hermes_model_metadata._query_local_context_length = empty_endpoint_model_metadata + hermes_usage_pricing.fetch_endpoint_model_metadata = empty_endpoint_model_metadata + + client = ServerClient.model_construct( + global_config_dict={ + "global_aiohttp_trust_env": runner.get("http_proxy_from_environment", False), + "policy_model": { + "responses_api_models": { + "policy_model": {}, + } + } + } + ) + # ServerClient appends protocol paths such as /v1/responses itself, while + # SDK-style harnesses consume the versioned base URL below. Accept either a + # root or /v1 sandbox override without duplicating the API version. + client._build_server_base_url = lambda _cfg: model_url_root + + base = { + "host": "0.0.0.0", + "port": 0, + "name": "legal_agent_bench_inner_agent", + "entrypoint": "app.py", + "resources_server": ResourcesServerRef(name="legal_agent_bench", type="resources_servers"), + "model_server": ModelServerRef(name="policy_model", type="responses_api_models"), + } + kwargs = {key: value for key, value in base.items() if key in config_class.model_fields} + kwargs.update(runner.get("agent_kwargs") or {}) + if model_api_key and "model" in config_class.model_fields and not kwargs.get("model"): + # Gym model proxies substitute their configured model when a harness + # omits it or uses the proxy-local placeholder. A direct provider + # endpoint cannot do that, so give SDK/CLI harnesses the selected model. + kwargs["model"] = runner["model_name"] + config = config_class(**kwargs) + agent = agent_class(config=config, server_client=client) + + if model_api_key: + # Direct remote endpoints need the configured provider credential. The + # secret enters only through the agent sandbox environment: it is not + # serialized into runner.json or staged into the verifier sandbox. + original_post = client.post + + async def authenticated_post(*args, **kwargs): + headers = dict(kwargs.pop("headers", {}) or {}) + headers.setdefault("Authorization", f"Bearer {model_api_key}") + return await original_post(*args, headers=headers, **kwargs) + + object.__setattr__(client, "post", authenticated_post) + + if "anthropic_api_key" in config_class.model_fields: + object.__setattr__(config, "anthropic_api_key", model_api_key) + if "openai_api_key" in config_class.model_fields: + object.__setattr__(config, "openai_api_key", model_api_key) + if runner.get("disable_endpoint_metadata_probe"): + # Hermes constructs its OpenAI client inside responses() with a + # local-proxy sentinel. Replace only that constructor argument for + # this inner process so direct endpoints receive the real key. + import run_agent as hermes_run_agent + + original_ai_agent = hermes_run_agent.AIAgent + + class AuthenticatedAIAgent(original_ai_agent): + def __init__(self, *args, **kwargs): + kwargs["api_key"] = model_api_key + super().__init__(*args, **kwargs) + + hermes_run_agent.AIAgent = AuthenticatedAIAgent + + if hasattr(agent, "resolve_model_base_url"): + object.__setattr__(agent, "resolve_model_base_url", lambda *args, **kwargs: model_url_v1) + if hasattr(agent, "_resolve_model_base_url"): + object.__setattr__(agent, "_resolve_model_base_url", lambda *args, **kwargs: model_url_v1) + if hasattr(agent, "_resolve_base_url"): + # Claude Code appends /v1/messages to ANTHROPIC_BASE_URL itself. Give + # its private root resolver the unversioned URL even when the provider + # override was supplied with a terminal /v1 segment. + object.__setattr__(agent, "_resolve_base_url", lambda *args, **kwargs: model_url_root) + + body = NeMoGymResponseCreateParamsNonStreaming.model_validate(runner["responses_create_params"]) + if model_api_key and body.model is None: + body = body.model_copy(update={"model": runner["model_name"]}) + response_kwargs = {"body": body} + if "request" in inspect.signature(agent.responses).parameters: + response_kwargs["request"] = SimpleNamespace(path_params={}) + async def invoke_agent(): + # This runner calls the selected agent directly rather than starting its + # Gym webserver, so initialize the shared HTTP client that webserver + # startup would normally create. OpenShell injects policy-enforced proxy + # variables; other providers retain aiohttp's direct-connection default. + http_client = None + if not is_global_aiohttp_client_setup(): + http_client = set_global_aiohttp_client( + GlobalAIOHTTPAsyncClientConfig( + global_aiohttp_trust_env=runner.get("http_proxy_from_environment", False), + ) + ) + try: + return await agent.responses(**response_kwargs) + finally: + if http_client is not None: + await http_client.close() + + response = asyncio.run(invoke_agent()) + Path(f"{RUNTIME}/response.json").write_text(response.model_dump_json()) +except Exception as exc: + write_status(ok=False, phase="agent_execution", error=f"{type(exc).__name__}: {exc}") + raise +else: + write_status(ok=True, phase="complete") +""" + + +class LegalAgentBenchAgentConfig(BaseResponsesAPIAgentConfig): + resources_server: ResourcesServerRef + model_server: ModelServerRef + agent_server_module: str + agent_server_class: str + agent_config_class: str + agent_kwargs: dict[str, Any] = Field(default_factory=dict) + + runtime_tasks_dir: str = str(DEFAULT_RUNTIME_TASKS_DIR) + skills_dir: str = str(DEFAULT_SKILLS_DIR) + concurrency: int = Field(default=1, ge=1) + agent_timeout_seconds: int = Field(default=10800, ge=1) + model_connect_timeout_seconds: int = Field(default=10, ge=1) + verifier_timeout_seconds: int = Field(default=3600, ge=1) + runtime_build_timeout_seconds: int = Field(default=3600, ge=1) + sandbox_staging_timeout_seconds: int = Field(default=900, ge=1) + image_build_timeout_seconds: int = Field(default=3600, ge=1) + sandbox_ttl_seconds: int = Field(default=14400, ge=1) + sandbox_provider: str | dict[str, Any] = Field(default_factory=lambda: {"docker": {}}) + sandbox_image: Optional[str] = None + runtime_builder_provider_options: dict[str, Any] = Field(default_factory=dict) + agent_sandbox_provider_options: dict[str, Any] = Field(default_factory=dict) + verifier_sandbox_provider_options: dict[str, Any] = Field(default_factory=dict) + opensandbox_request_fraction: Optional[float] = Field(default=0.25, gt=0, le=1) + docker_network: Optional[str] = "host" + sandbox_model_base_url: Optional[str] = None + sandbox_model_api_key_env: Optional[str] = None + image_repository: str = "nemo-gym-legal-agent-bench" + results_dir: str = "results/legal_agent_bench" + + +class LegalAgentBenchRunRequest(BaseRunRequest): + model_config = ConfigDict(extra="allow") + instance_id: str + + +class LegalAgentBenchAgentResponse(BaseVerifyResponse): + model_config = ConfigDict(extra="allow") + instance_id: str + criteria_pass_rate: float = 0.0 + judge_error_count: int = 0 + verifier_error: int = 0 + mask_sample: bool = False + agent_failed: bool = False + model_connection_failed: bool = False + agent_timed_out: bool = False + verifier_failed: bool = False + verifier_timed_out: bool = False + sandbox_failed: bool = False + task_failed: bool = False + configuration_failed: bool = False + failure_reason: Optional[str] = None + artifact_dir: Optional[str] = None + run_summary_path: Optional[str] = None + agent_trace_path: Optional[str] = None + agent_stdout_path: Optional[str] = None + agent_stderr_path: Optional[str] = None + verifier_report_path: Optional[str] = None + output_dir: Optional[str] = None + + +class LegalAgentBenchTaskError(ValueError): + """The requested LAB task is unsafe, unknown, incomplete, or malformed.""" + + +class LegalAgentBenchConfigurationError(ValueError): + """The selected Gym agent or its LAB configuration is invalid.""" + + +class LegalAgentBenchArtifactError(RuntimeError): + """The sandbox returned an unsafe or malformed artifact tree.""" + + +def _provider_name(provider: Mapping[str, Any]) -> str: + names = list(provider) + if len(names) != 1: + raise LegalAgentBenchConfigurationError( + f"sandbox_provider must resolve to exactly one provider, got {names!r}" + ) + return str(names[0]) + + +def _create_archive(destination: Path, entries: list[tuple[Path, str]]) -> None: + """Archive trusted host inputs under explicit sandbox-relative names.""" + with tarfile.open(destination, "w:gz", dereference=False) as archive: + for source, arcname in entries: + if not source.exists(): + raise FileNotFoundError(source) + archive.add(source, arcname=arcname, recursive=True) + + +def _prepare_sandbox_upload(path: Path) -> None: + """Make a non-secret transfer archive readable by a non-root sandbox user. + + Some providers copy the host file's mode while creating the destination as + root. Temporary files start as mode 0600, which would make the shared + upload API unusable with an image whose default user is non-root. All LAB + transfer archives are created beneath private temporary/cache directories + and contain no model or judge credentials. + """ + path.chmod(0o644) + + +def _validate_archive_member(member: tarfile.TarInfo) -> None: + if member.name in {".", "./"} and member.isdir(): + return + path = PurePosixPath(member.name) + if path.is_absolute() or not path.parts or any(part in {"", ".", ".."} for part in path.parts): + raise LegalAgentBenchArtifactError(f"Unsafe sandbox artifact path: {member.name!r}") + if member.issym() or member.islnk(): + raise LegalAgentBenchArtifactError(f"Sandbox artifacts may not contain links: {member.name!r}") + if not (member.isdir() or member.isreg()): + raise LegalAgentBenchArtifactError( + f"Sandbox artifacts may contain only directories and regular files: {member.name!r}" + ) + + +def _extract_untrusted_archive(archive_path: Path, destination: Path) -> None: + """Materialize sandbox output without following links or restoring ownership.""" + destination.mkdir(parents=True, exist_ok=True) + with tarfile.open(archive_path, "r:gz") as archive: + members = archive.getmembers() + for member in members: + _validate_archive_member(member) + archive.extractall(destination, members=members, filter="data") + + +def _copy_downloaded_file(source: Path, destination: Path) -> None: + mode = source.lstat().st_mode + if not stat.S_ISREG(mode): + raise LegalAgentBenchArtifactError(f"Expected a regular downloaded file, got {source}") + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, destination) + + +def _validate_runtime_archive(archive_path: Path) -> None: + """Validate the trusted-installer output before another sandbox extracts it.""" + root = PurePosixPath("agent_deps") + with tarfile.open(archive_path, "r:gz") as archive: + members = archive.getmembers() + if not members: + raise LegalAgentBenchArtifactError("LAB runtime builder returned an empty archive") + for member in members: + path = PurePosixPath(member.name) + if path.is_absolute() or not path.parts or any(part in {"", ".", ".."} for part in path.parts): + raise LegalAgentBenchArtifactError(f"Unsafe LAB runtime archive path: {member.name!r}") + if path != root and root not in path.parents: + raise LegalAgentBenchArtifactError(f"LAB runtime archive entry is outside {root}: {member.name!r}") + if member.issym() or member.islnk(): + target = PurePosixPath(member.linkname) + if target.is_absolute(): + raise LegalAgentBenchArtifactError( + f"LAB runtime archive contains an absolute link: {member.name!r}" + ) + # Tar symbolic links are relative to the link's parent, while + # hard-link targets are archive-root-relative member names. + resolved = target if member.islnk() else path.parent.joinpath(target) + normalized: list[str] = [] + for part in resolved.parts: + if part == "..": + if not normalized: + raise LegalAgentBenchArtifactError( + f"LAB runtime archive link escapes its root: {member.name!r}" + ) + normalized.pop() + elif part not in {"", "."}: + normalized.append(part) + normalized_path = PurePosixPath(*normalized) + if normalized_path != root and root not in normalized_path.parents: + raise LegalAgentBenchArtifactError(f"LAB runtime archive link escapes its root: {member.name!r}") + elif not (member.isdir() or member.isreg()): + raise LegalAgentBenchArtifactError( + "LAB runtime archives may contain only directories, regular files, and internal links: " + f"{member.name!r}" + ) + + +async def _acquire_file_lock(path: Path) -> Any: + """Acquire an interprocess lock without blocking the async server loop.""" + path.parent.mkdir(parents=True, exist_ok=True) + lock_file = path.open("a+") + while True: + try: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + return lock_file + except BlockingIOError: + await asyncio.sleep(0.1) + + +def agent_key(agent_server_module: str) -> str: + parts = agent_server_module.split(".") + if len(parts) < 2 or parts[-1] != "app": + raise LegalAgentBenchConfigurationError(f"Agent module must end in '.app': {agent_server_module!r}") + key = parts[-2] + if not key.replace("_", "").isalnum(): + raise LegalAgentBenchConfigurationError(f"Invalid agent module key: {key!r}") + return key + + +def _results_segment(value: str, *, fallback: str) -> str: + segment = "".join( + character if character.isascii() and (character.isalnum() or character in "._-") else "-" + for character in value + ) + segment = segment.strip("._-") + return segment[:120] or fallback + + +def _results_session_dir( + results_root: Path, + *, + agent_server_module: str, + model_name: str, + timestamp: float, + session_id: str, +) -> Path: + key = agent_key(agent_server_module) + harness = "native" if key == "legal_agent_bench_native_agent" else key.removesuffix("_agent") + date = time.strftime("%Y%m%d", time.localtime(timestamp)) + clock = time.strftime("%H%M%S", time.localtime(timestamp)) + model = _results_segment(model_name, fallback="unknown_model") + return results_root / f"{harness}_jobs" / model / f"{date}-{clock}_{session_id}" + + +def resolve_agent_setup_script(agent_server_module: str) -> Path: + key = agent_key(agent_server_module) + script = PARENT_DIR / "responses_api_agents" / key / "scripts" / f"{key}_deps.sh" + if not script.is_file(): + raise LegalAgentBenchConfigurationError( + f"Configurable LAB agent {key!r} requires dependency setup script {script.relative_to(PARENT_DIR)}" + ) + return script + + +def _recipe_hash(paths: list[Path], *, values: list[str] | None = None) -> str: + digest = hashlib.sha256() + for path in paths: + digest.update(str(path).encode()) + if path.is_file(): + digest.update(path.read_bytes()) + elif path.is_dir(): + for child in sorted(item for item in path.rglob("*") if item.is_file()): + if "__pycache__" in child.parts or child.suffix in {".pyc", ".pyo"}: + continue + digest.update(child.relative_to(path).as_posix().encode()) + digest.update(child.read_bytes()) + for value in values or []: + digest.update(value.encode()) + return digest.hexdigest() + + +def agent_runtime_env(agent_server_module: str, agent_kwargs: dict[str, Any]) -> dict[str, str]: + key = agent_key(agent_server_module) + pin = AGENT_CLI_PINS.get(key) + if pin is None: + return {} + field, environment_variable, package = pin + version = agent_kwargs.get(field) + normalized_version = version.strip() if isinstance(version, str) else "" + if not PINNED_NPM_VERSION.fullmatch(normalized_version): + raise LegalAgentBenchConfigurationError( + f"Configurable LAB agent {key!r} requires agent_kwargs.{field} to be an exact npm version" + ) + return {environment_variable: f"{package}@{normalized_version}"} + + +def _runtime_recipe( + agent_server_module: str, + *, + agent_kwargs: dict[str, Any], + image: str, + provider: Mapping[str, Any], +) -> tuple[str, Path, dict[str, str]]: + key = agent_key(agent_server_module) + script = resolve_agent_setup_script(agent_server_module) + requirements = PARENT_DIR / "responses_api_agents" / key / "requirements.txt" + runtime_env = agent_runtime_env(agent_server_module, agent_kwargs) + recipe = _recipe_hash( + [ + PORTABLE_PYTHON_SH, + script, + requirements, + PARENT_DIR / "pyproject.toml", + PARENT_DIR / "README.md", + PARENT_DIR / "nemo_gym", + ], + values=[ + image, + json.dumps(runtime_env, sort_keys=True), + json.dumps(provider, sort_keys=True, default=str), + ], + ) + return recipe, script, runtime_env + + +def _create_runtime_builder_input(destination: Path, agent_server_module: str, script: Path) -> None: + key = agent_key(agent_server_module) + requirements = PARENT_DIR / "responses_api_agents" / key / "requirements.txt" + _create_archive( + destination, + [ + (PARENT_DIR / "pyproject.toml", "nemo_gym_mount/pyproject.toml"), + (PARENT_DIR / "README.md", "nemo_gym_mount/README.md"), + (PARENT_DIR / "nemo_gym", "nemo_gym_mount/nemo_gym"), + (requirements, f"nemo_gym_mount/responses_api_agents/{key}/requirements.txt"), + (script, f"nemo_gym_mount/responses_api_agents/{key}/scripts/{script.name}"), + ( + PORTABLE_PYTHON_SH, + "nemo_gym_mount/responses_api_agents/legal_agent_bench_agent/setup_scripts/_portable_python.sh", + ), + ], + ) + + +def _empty_response(model_name: str) -> NeMoGymResponse: + return NeMoGymResponse.model_validate( + { + "id": f"resp_{uuid4().hex}", + "created_at": int(time.time()), + "model": model_name or "policy_model", + "object": "response", + "output": [], + "parallel_tool_calls": False, + "tool_choice": "auto", + "tools": [], + } + ) + + +def sandbox_model_url( + model_url: str, + *, + docker_network: Optional[str], + platform_name: Optional[str] = None, +) -> str: + """Translate host-loopback model URLs into addresses reachable from a Docker sandbox.""" + parsed = urlsplit(model_url) + hostname = (parsed.hostname or "").lower() + platform_name = platform_name or sys.platform + docker_desktop = platform_name == "darwin" or platform_name.startswith("win") + uses_bridge = docker_network != "host" + if hostname not in {"localhost", "127.0.0.1", "0.0.0.0", "::1", "::"} or not (docker_desktop or uses_bridge): + return model_url + + userinfo = parsed.netloc.rsplit("@", 1)[0] + "@" if "@" in parsed.netloc else "" + port = f":{parsed.port}" if parsed.port is not None else "" + return urlunsplit(parsed._replace(netloc=f"{userinfo}host.docker.internal{port}")) + + +def host_tunnel_model_url(model_url: str) -> str: + """Translate wildcard listeners into an address usable on a shared host network or tunnel.""" + parsed = urlsplit(model_url) + if (parsed.hostname or "").lower() not in {"0.0.0.0", "::"}: + return model_url + userinfo = parsed.netloc.rsplit("@", 1)[0] + "@" if "@" in parsed.netloc else "" + port = f":{parsed.port}" if parsed.port is not None else "" + return urlunsplit(parsed._replace(netloc=f"{userinfo}127.0.0.1{port}")) + + +def _response_output_text(response: NeMoGymResponse) -> str: + parts: list[str] = [] + for item in response.output: + if getattr(item, "type", None) != "message": + continue + for content in getattr(item, "content", []) or []: + text = getattr(content, "text", None) + if text: + parts.append(str(text)) + return "\n".join(parts).strip() + + +def agent_response_failure(response: NeMoGymResponse, agent_server_module: str) -> Optional[str]: + """Return a harness-failure reason without treating normal task-quality failures as infrastructure.""" + if response.error is not None: + return f"Agent returned an error response: {response.error}" + # Max-turn and context-limit stops are valid incomplete model outcomes. The + # harnesses represent those with ``incomplete_details`` so LAB can still + # verify and score whatever artifacts the agent produced. + if response.incomplete_details is not None: + return None + if not response.output: + return "Agent produced an empty trajectory" + + key = agent_key(agent_server_module) + if key == "hermes_agent": + message_items = [item for item in response.output if getattr(item, "type", None) == "message"] + synthetic_messages = bool(message_items) and all( + getattr(item, "prompt_token_ids", None) == [0] and getattr(item, "generation_token_ids", None) == [0] + for item in message_items + ) + has_tool_activity = any(getattr(item, "type", None) != "message" for item in response.output) + if synthetic_messages and not has_tool_activity: + detail = _response_output_text(response) or "no assistant message" + return f"Hermes produced no model trajectory: {detail}" + + usage = response.usage + total_tokens = int(getattr(usage, "total_tokens", 0) or 0) if usage is not None else 0 + has_tool_activity = any(getattr(item, "type", None) != "message" for item in response.output) + if total_tokens == 0 and not has_tool_activity and not _response_output_text(response): + return f"{key} produced no model activity" + return None + + +def agent_response_failure_flags(response: NeMoGymResponse, agent_server_module: str) -> tuple[bool, bool]: + """Return structured model-connection and timeout flags from a failed native response.""" + if agent_server_module != NATIVE_AGENT_MODULE or response.error is None: + return False, False + metadata = response.metadata or {} + failure_class = metadata.get(AGENT_FAILURE_CLASS_METADATA_KEY) + if failure_class not in PROPAGATED_AGENT_FAILURE_CLASSES: + return False, False + return failure_class == "model_connection_failed", failure_class == "agent_timed_out" + + +def _task_name(instance_id: str) -> str: + alias, separator, task_name = instance_id.partition("::") + if separator != "::" or alias != DATASET_ALIAS or not task_name: + raise LegalAgentBenchTaskError(f"instance_id must be '{DATASET_ALIAS}::', got {instance_id!r}") + path = PurePosixPath(task_name) + if len(path.parts) != 1 or path.parts[0] in {"", ".", ".."}: + raise LegalAgentBenchTaskError(f"Unsafe Legal Agent Bench task name: {task_name!r}") + return task_name + + +def resolve_task_dir(runtime_tasks_dir: str | Path, instance_id: str) -> Path: + root = resolve_repo_path(runtime_tasks_dir) + task_dir = (root / _task_name(instance_id)).resolve() + if task_dir.parent != root or not task_dir.is_dir(): + raise LegalAgentBenchTaskError(f"Legal Agent Bench runtime task not found: {task_dir}") + required = ("instruction.md", "task.json", "task.toml", "documents", "environment", "tests") + missing = [name for name in required if not (task_dir / name).exists()] + if missing: + raise LegalAgentBenchTaskError( + f"Legal Agent Bench task {task_dir.name} is incomplete; missing: {', '.join(missing)}" + ) + return task_dir + + +def _load_skill_prompt(skills_dir: Path) -> str: + try: + validate_harness_skills(skills_dir) + sections = [] + for name in REQUIRED_SKILLS: + skill_path = skills_dir / name / "SKILL.md" + sections.append(f"\n\n## Skill: {name}\n\n{skill_path.read_text(encoding='utf-8')}") + except (FileNotFoundError, ValueError) as exc: + raise LegalAgentBenchConfigurationError(f"Invalid LAB skills configuration: {exc}") from exc + return "".join(sections) + + +def native_tool_definitions() -> list[dict[str, Any]]: + """Translate LAB's canonical tools to OpenAI Responses function tools.""" + return [ + { + "type": "function", + "name": tool["name"], + "description": tool["description"], + "parameters": tool["parameters"], + "strict": False, + } + for tool in get_all_tool_definitions() + ] + + +def compose_agent_input( + task_dir: Path, + skills_dir: Path, + params: NeMoGymResponseCreateParamsNonStreaming, + *, + native: bool = False, +) -> NeMoGymResponseCreateParamsNonStreaming: + try: + task = json.loads((task_dir / "task.json").read_text(encoding="utf-8")) + title = task["title"] + instructions = task["instructions"] + except (OSError, KeyError, TypeError, json.JSONDecodeError) as exc: + raise LegalAgentBenchTaskError(f"Invalid LAB task configuration in {task_dir}: {exc}") from exc + preamble = LAB_SYSTEM_PROMPT if native else GENERIC_HARNESS_PREAMBLE + system_prompt = preamble + _load_skill_prompt(skills_dir) + "\n\n## Task\n\n" + f"# {title}\n\n{instructions}" + update: dict[str, Any] = { + "input": [ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": INITIAL_USER_PROMPT}, + ] + } + if native: + update.update( + { + "tools": native_tool_definitions(), + "tool_choice": "auto", + "parallel_tool_calls": False, + } + ) + payload = params.model_dump(mode="json") + payload.update(update) + return NeMoGymResponseCreateParamsNonStreaming.model_validate(payload) + + +def _normalized_reward_data(reward_data: Mapping[str, Any]) -> dict[str, Any]: + """Validate verifier metrics before they can reach response construction or host artifacts.""" + + def metric(name: str) -> float: + value = reward_data.get(name, 0.0) + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise LegalAgentBenchArtifactError(f"Invalid verifier {name}: expected a number") + normalized = float(value) + if not math.isfinite(normalized) or not 0.0 <= normalized <= 1.0: + raise LegalAgentBenchArtifactError(f"Invalid verifier {name}: expected a finite value in [0, 1]") + return normalized + + def count(name: str) -> int: + value = reward_data.get(name, 0) + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise LegalAgentBenchArtifactError(f"Invalid verifier {name}: expected a non-negative integer") + return value + + normalized = dict(reward_data) + normalized.update( + reward=metric("reward"), + criteria_pass_rate=metric("criteria_pass_rate"), + verifier_error=count("verifier_error"), + judge_error_count=count("judge_error_count"), + ) + return normalized + + +def _task_toml(task_dir: Path) -> dict[str, Any]: + import tomllib + + try: + with (task_dir / "task.toml").open("rb") as handle: + return tomllib.load(handle) + except (OSError, tomllib.TOMLDecodeError) as exc: + raise LegalAgentBenchTaskError(f"Invalid LAB task.toml in {task_dir}: {exc}") from exc + + +def _verifier_env(task_dir: Path) -> dict[str, str]: + return {str(key): str(value) for key, value in (_task_toml(task_dir).get("verifier", {}).get("env") or {}).items()} + + +def _sandbox_resources(task_dir: Path) -> SandboxResources: + environment = _task_toml(task_dir).get("environment") or {} + return SandboxResources( + cpu=float(environment["cpus"]) if environment.get("cpus") is not None else None, + memory_mib=int(environment["memory_mb"]) if environment.get("memory_mb") is not None else None, + disk_gib=( + max(1, int(environment["storage_mb"]) // 1024) if environment.get("storage_mb") is not None else None + ), + gpu=int(environment.get("gpus") or 0), + ) + + +def _fractional_resource_requests(resources: SandboxResources, fraction: float) -> dict[str, Any]: + """Keep Kubernetes requests below LAB limits while preserving non-shareable resources.""" + requests: dict[str, Any] = {} + if resources.cpu is not None: + requests["cpu"] = resources.cpu * fraction + if resources.memory_mib is not None: + requests["memory_mib"] = max(1, math.ceil(resources.memory_mib * fraction)) + if resources.disk_gib is not None: + requests["disk_gib"] = resources.disk_gib + if resources.gpu: + requests["gpu"] = resources.gpu + if resources.gpu and resources.gpu_type is not None: + requests["gpu_type"] = resources.gpu_type + return requests + + +def _environment_hash(environment_dir: Path) -> str: + digest = hashlib.sha256() + for path in sorted(item for item in environment_dir.rglob("*") if item.is_file()): + digest.update(path.relative_to(environment_dir).as_posix().encode()) + digest.update(path.read_bytes()) + return digest.hexdigest() + + +async def _run_process(args: list[str], *, cwd: Path, timeout: int) -> tuple[int, str, str]: + process = await asyncio.create_subprocess_exec( + *args, + cwd=cwd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + try: + stdout, stderr = await asyncio.wait_for(process.communicate(), timeout=timeout) + except asyncio.TimeoutError: + process.kill() + await process.communicate() + raise TimeoutError(f"Command timed out after {timeout}s: {args}") + return process.returncode or 0, stdout.decode(errors="replace"), stderr.decode(errors="replace") + + +class LegalAgentBenchAgent(SimpleResponsesAPIAgent): + config: LegalAgentBenchAgentConfig + model_config = ConfigDict(arbitrary_types_allowed=True) + + _sem: asyncio.Semaphore = PrivateAttr() + _image_lock: asyncio.Lock = PrivateAttr() + _runtime_lock: asyncio.Lock = PrivateAttr() + _session_results_dir: Path = PrivateAttr() + _runtime_archives: dict[str, Path] = PrivateAttr(default_factory=dict) + + def model_post_init(self, context: Any) -> None: + self._sem = asyncio.Semaphore(self.config.concurrency) + self._image_lock = asyncio.Lock() + self._runtime_lock = asyncio.Lock() + results_root = resolve_repo_path(self.config.results_dir) + timestamp = time.time() + self._session_results_dir = _results_session_dir( + results_root, + agent_server_module=self.config.agent_server_module, + model_name=self._model_name(), + timestamp=timestamp, + session_id=uuid4().hex[:8], + ) + super().model_post_init(context) + + async def responses( + self, + body: NeMoGymResponseCreateParamsNonStreaming = Body(), + ) -> NeMoGymResponse: + raise NotImplementedError("LegalAgentBenchAgent is task-driven through /run, not /v1/responses") + + async def _ensure_runtime(self, image: str) -> Path: + async with self._runtime_lock: + provider = self._provider_config() + recipe, script, runtime_env = _runtime_recipe( + self.config.agent_server_module, + agent_kwargs=self.config.agent_kwargs, + image=image, + provider=provider, + ) + if recipe in self._runtime_archives: + return self._runtime_archives[recipe] + + key = agent_key(self.config.agent_server_module) + cache_root = PACKAGE_DIR / ".deps" / key + cache_root.mkdir(parents=True, exist_ok=True) + archive_path = cache_root / f"{recipe}.tar.gz" + if archive_path.is_file(): + _validate_runtime_archive(archive_path) + self._runtime_archives[recipe] = archive_path + return archive_path + + lock_path = PACKAGE_DIR / ".deps" / ".locks" / f"{key}-{recipe}.lock" + lock_file = await _acquire_file_lock(lock_path) + try: + if archive_path.is_file(): + _validate_runtime_archive(archive_path) + self._runtime_archives[recipe] = archive_path + return archive_path + + with tempfile.NamedTemporaryFile(suffix=".tar.gz", delete=False) as temporary: + builder_input = Path(temporary.name) + with tempfile.NamedTemporaryFile( + suffix=".tar.gz", + prefix=f".{recipe}-", + dir=cache_root, + delete=False, + ) as temporary: + downloaded_runtime = Path(temporary.name) + builder_sandbox: Optional[AsyncSandbox] = None + try: + _create_runtime_builder_input( + builder_input, + self.config.agent_server_module, + script, + ) + _prepare_sandbox_upload(builder_input) + builder_resources = SandboxResources(cpu=1, memory_mib=4096, disk_gib=10) + builder_options = self._resource_provider_options(provider, builder_resources) + builder_options.update(self.config.runtime_builder_provider_options) + builder_sandbox = AsyncSandbox( + provider, + SandboxSpec( + image=image, + ttl_s=self.config.sandbox_ttl_seconds, + workdir="/tmp", + env=LAB_SANDBOX_ENV, + resources=builder_resources, + metadata=self._sandbox_metadata(), + provider_options=builder_options, + ), + ) + await builder_sandbox.start() + root_result = await builder_sandbox.exec(f"mkdir -p {SANDBOX_ROOT}", timeout_s=300) + if root_result.return_code != 0 or root_result.error_type is not None: + raise RuntimeError( + root_result.stderr or root_result.stdout or "Failed to create LAB sandbox root" + ) + await builder_sandbox.upload(builder_input, f"{SANDBOX_ROOT}/runtime-builder-input.tar.gz") + build_root = f"{SANDBOX_ROOT}/runtime-builder" + nemo_gym_root = f"{build_root}/nemo_gym_mount" + deps_root = f"{build_root}/agent_deps" + environment = { + "PORTABLE_PYTHON_SH": ( + f"{nemo_gym_root}/responses_api_agents/legal_agent_bench_agent/" + "setup_scripts/_portable_python.sh" + ), + "DEPS_DIR": deps_root, + "NEMO_GYM_ROOT": nemo_gym_root, + "TMPDIR": f"{build_root}/tmp", + **runtime_env, + } + result = await builder_sandbox.exec( + f"rm -rf {build_root} && mkdir -p {build_root}/home {build_root}/tmp {deps_root} && " + f"tar -xzf {SANDBOX_ROOT}/runtime-builder-input.tar.gz -C {build_root} && " + f"export HOME={build_root}/home && " + f"bash {nemo_gym_root}/responses_api_agents/{key}/scripts/{script.name} && " + f"tar -czf {SANDBOX_ROOT}/runtime-builder-output.tar.gz -C {build_root} agent_deps", + cwd="/tmp", + env=environment, + timeout_s=self.config.runtime_build_timeout_seconds, + ) + if result.return_code != 0: + detail = result.stderr or result.stdout or "LAB runtime builder failed" + raise RuntimeError(detail[-4000:]) + await builder_sandbox.download( + f"{SANDBOX_ROOT}/runtime-builder-output.tar.gz", + downloaded_runtime, + ) + _validate_runtime_archive(downloaded_runtime) + _prepare_sandbox_upload(downloaded_runtime) + downloaded_runtime.replace(archive_path) + finally: + if builder_sandbox is not None: + await builder_sandbox.stop() + builder_input.unlink(missing_ok=True) + downloaded_runtime.unlink(missing_ok=True) + finally: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN) + lock_file.close() + + self._runtime_archives[recipe] = archive_path + return archive_path + + async def _ensure_image(self, task_dir: Path) -> str: + if self.config.sandbox_image: + return self.config.sandbox_image + if _provider_name(self._provider_config()) != "docker": + raise LegalAgentBenchConfigurationError( + "Non-Docker LAB sandboxes require a provider-compatible sandbox_image" + ) + environment_dir = task_dir / "environment" + image = f"{self.config.image_repository}:{_environment_hash(environment_dir)[:16]}" + async with self._image_lock: + docker = shutil.which("docker") + if not docker: + raise FileNotFoundError( + "Docker CLI is required to auto-build the default Legal Agent Bench image; " + "set sandbox_image to use an existing image" + ) + inspect, _stdout, _stderr = await _run_process( + [docker, "image", "inspect", image], + cwd=environment_dir, + timeout=60, + ) + if inspect == 0: + return image + code, _stdout, stderr = await _run_process( + [docker, "build", "--tag", image, "."], + cwd=environment_dir, + timeout=self.config.image_build_timeout_seconds, + ) + if code != 0: + raise RuntimeError(f"Legal Agent Bench image build failed: {stderr[-2000:]}") + return image + + def _model_url(self, body: LegalAgentBenchRunRequest) -> str: + if self.config.sandbox_model_base_url: + return self.config.sandbox_model_base_url + provider_name = _provider_name(self._provider_config()) + try: + model_config = get_first_server_config_dict( + self.server_client.global_config_dict, + self.config.model_server.name, + ) + base_url = self.server_client._build_server_base_url(model_config) + except (AttributeError, KeyError, TypeError, ValueError) as exc: + raise LegalAgentBenchConfigurationError( + f"Unable to resolve policy model server {self.config.model_server.name!r}: {exc}" + ) from exc + rollout_id = self.rollout_id_from_run(body) + prefixed_url = apply_rollout_prefix(base_url, rollout_id) if rollout_id else base_url + if provider_name == "docker": + return sandbox_model_url(prefixed_url, docker_network=self.config.docker_network) + if provider_name == "ecs_fargate": + return host_tunnel_model_url(prefixed_url) + if provider_name in {"apptainer", "enroot"}: + return host_tunnel_model_url(prefixed_url) + + hostname = (urlsplit(prefixed_url).hostname or "").lower() + if hostname in {"localhost", "127.0.0.1", "0.0.0.0", "::1", "::"}: + raise LegalAgentBenchConfigurationError( + f"LAB sandbox provider {provider_name!r} cannot use the host-local policy URL; " + "set sandbox_model_base_url to a policy proxy reachable from that provider" + ) + return prefixed_url + + def _resource_provider_options( + self, + provider: Mapping[str, Any], + resources: SandboxResources, + ) -> dict[str, Any]: + fraction = self.config.opensandbox_request_fraction + if _provider_name(provider) != "opensandbox" or fraction is None: + return {} + return {"resource_requests": _fractional_resource_requests(resources, fraction)} + + def _agent_provider_options( + self, + provider: Mapping[str, Any], + model_url: str, + resources: SandboxResources, + ) -> dict[str, Any]: + """Return provider-specific routing without exposing policy credentials to the sandbox.""" + options = self._resource_provider_options(provider, resources) + options.update(self.config.agent_sandbox_provider_options) + if _provider_name(provider) == "ecs_fargate" and not self.config.sandbox_model_base_url: + outside_endpoints = list(options.get("outside_endpoints") or []) + outside_endpoints.append( + { + "url": model_url, + "env_var": "LAB_POLICY_MODEL_URL", + } + ) + options["outside_endpoints"] = outside_endpoints + return options + + def _model_name(self) -> str: + global_config = getattr(getattr(self, "server_client", None), "global_config_dict", None) + configured_name = global_config.get("policy_model_name") if isinstance(global_config, Mapping) else None + return str(configured_name or self.config.model_server.name) + + @staticmethod + def _paths_for_root(root: Path, *, create: bool = False) -> dict[str, Path]: + paths = { + "root": root, + "runtime": root / "runtime", + "agent_source": root / "agent_source", + "agent": root / "agent", + "lab_run": root / "agent" / "artifacts" / "lab-run", + "output": root / "agent" / "artifacts" / "lab-run" / "output", + "workspace": root / "workspace", + "verifier": root / "verifier", + } + if create: + for path in paths.values(): + path.mkdir(parents=True, exist_ok=True) + return paths + + def _run_root(self, task_name: str) -> Path: + task_segment = _results_segment(task_name, fallback="unknown_task") + return self._session_results_dir / f"{task_segment}_{uuid4().hex[:8]}" + + def _run_dirs(self, task_name: str) -> dict[str, Path]: + root = self._run_root(task_name) + paths = self._paths_for_root(root, create=True) + print(f"LAB rollout artifacts: {root}", flush=True) + return paths + + def _publish_staged_run(self, staged: dict[str, Path], final_root: Path) -> dict[str, Path]: + final_root.parent.mkdir(parents=True, exist_ok=True) + shutil.copytree(staged["root"], final_root) + print(f"LAB rollout artifacts: {final_root}", flush=True) + return self._paths_for_root(final_root) + + def _stage_agent_source(self, paths: dict[str, Path]) -> None: + key = agent_key(self.config.agent_server_module) + source = PARENT_DIR / "responses_api_agents" / key + if not source.is_dir(): + raise LegalAgentBenchConfigurationError(f"Configured Gym agent source not found: {source}") + destination = paths["agent_source"] / "responses_api_agents" / key + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copytree( + source, + destination, + ignore=shutil.ignore_patterns( + "__pycache__", + ".pytest_cache", + ".venv", + ".deps", + ".claude_node", + ".codex_node", + "configs", + "data", + "scripts", + "tests", + ), + ) + + def _write_runner_config( + self, + paths: dict[str, Path], + params: NeMoGymResponseCreateParamsNonStreaming, + model_url: str, + ) -> None: + (paths["runtime"] / "agent_runner.py").write_text(_RUNNER_SOURCE) + agent_kwargs = dict(self.config.agent_kwargs) + if agent_key(self.config.agent_server_module) == "codex_agent": + extra_config = dict(agent_kwargs.get("extra_config") or {}) + if "model_catalog_json" not in extra_config: + catalog = codex_model_catalog(self._model_name()) + (paths["runtime"] / "codex_model_catalog.json").write_text(json.dumps(catalog, indent=2)) + extra_config["model_catalog_json"] = CODEX_MODEL_CATALOG_PATH + agent_kwargs["extra_config"] = extra_config + runner = { + "agent_server_module": self.config.agent_server_module, + "agent_server_class": self.config.agent_server_class, + "agent_config_class": self.config.agent_config_class, + "agent_kwargs": agent_kwargs, + "model_name": self._model_name(), + "model_url": model_url, + "model_connect_timeout_seconds": self.config.model_connect_timeout_seconds, + "http_proxy_from_environment": _provider_name(self._provider_config()) == "openshell", + "disable_endpoint_metadata_probe": agent_key(self.config.agent_server_module) == "hermes_agent", + "responses_create_params": params.model_dump(mode="json", exclude_none=True), + } + (paths["runtime"] / "runner.json").write_text(json.dumps(runner, indent=2)) + + def _provider_config(self) -> dict[str, Any]: + global_config = getattr(getattr(self, "server_client", None), "global_config_dict", None) + try: + provider = resolve_provider_config(self.config.sandbox_provider, global_config) + except (TypeError, ValueError) as exc: + raise LegalAgentBenchConfigurationError(f"Invalid sandbox provider configuration: {exc}") from exc + if _provider_name(provider) != "docker": + return provider + + docker = dict(provider.get("docker") or {}) + create = dict(docker.get("create") or {}) + create.setdefault("network", self.config.docker_network) + create.setdefault("pids_limit", 4096) + if sys.platform == "linux" and self.config.docker_network != "host": + extra_run_args = list(create.get("extra_run_args") or []) + host_gateway = ["--add-host", "host.docker.internal:host-gateway"] + if host_gateway[0] not in extra_run_args: + extra_run_args.extend(host_gateway) + create["extra_run_args"] = extra_run_args + execution = dict(docker.get("exec") or {}) + execution.setdefault("default_timeout_s", self.config.agent_timeout_seconds) + execution.setdefault("concurrency", 8) + docker.update({"create": create, "exec": execution}) + return {"docker": docker} + + def _sandbox_metadata(self) -> dict[str, Any]: + global_config = getattr(getattr(self, "server_client", None), "global_config_dict", None) + try: + return resolve_provider_metadata(self.config.sandbox_provider, global_config) + except (TypeError, ValueError) as exc: + raise LegalAgentBenchConfigurationError(f"Invalid sandbox provider metadata: {exc}") from exc + + def _agent_sandbox( + self, + *, + image: str, + task_dir: Path, + model_url: str, + ) -> AsyncSandbox: + provider = self._provider_config() + resources = _sandbox_resources(task_dir) + sandbox_env = dict(LAB_SANDBOX_ENV) + if self.config.sandbox_model_api_key_env: + model_api_key = os.environ.get(self.config.sandbox_model_api_key_env) + if not model_api_key: + raise LegalAgentBenchConfigurationError("Configured sandbox_model_api_key_env is unset or empty") + sandbox_env["LAB_POLICY_API_KEY"] = model_api_key + return AsyncSandbox( + provider, + SandboxSpec( + image=image, + ttl_s=self.config.sandbox_ttl_seconds, + workdir="/tmp", + env=sandbox_env, + resources=resources, + metadata=self._sandbox_metadata(), + provider_options=self._agent_provider_options(provider, model_url, resources), + ), + ) + + def _verifier_sandbox( + self, + *, + image: str, + task_dir: Path, + ) -> AsyncSandbox: + provider = self._provider_config() + resources = _sandbox_resources(task_dir) + provider_options = self._resource_provider_options(provider, resources) + provider_options.update(self.config.verifier_sandbox_provider_options) + return AsyncSandbox( + provider, + SandboxSpec( + image=image, + ttl_s=self.config.sandbox_ttl_seconds, + workdir="/tmp", + env=LAB_SANDBOX_ENV, + resources=resources, + metadata=self._sandbox_metadata(), + provider_options=provider_options, + ), + ) + + async def _stage_agent_sandbox( + self, + sandbox: AsyncSandbox, + *, + task_dir: Path, + skills_dir: Path, + runtime_archive: Path, + paths: dict[str, Path], + ) -> None: + with tempfile.NamedTemporaryFile(suffix=".tar.gz", delete=False) as temporary: + archive_path = Path(temporary.name) + try: + _create_archive( + archive_path, + [ + (paths["agent_source"], "agent_source"), + (task_dir / "documents", "workspace/vdr"), + (skills_dir, "workspace/skills"), + (paths["runtime"], "runtime"), + ], + ) + _prepare_sandbox_upload(archive_path) + _prepare_sandbox_upload(runtime_archive) + root_result = await sandbox.exec( + f"mkdir -p {SANDBOX_ROOT}", timeout_s=self.config.sandbox_staging_timeout_seconds + ) + if root_result.return_code != 0 or root_result.error_type is not None: + raise RuntimeError(root_result.stderr or root_result.stdout or "Failed to create LAB sandbox root") + await sandbox.upload(archive_path, f"{SANDBOX_ROOT}/agent-input.tar.gz") + await sandbox.upload(runtime_archive, f"{SANDBOX_ROOT}/runtime.tar.gz") + finally: + archive_path.unlink(missing_ok=True) + + command = ( + f"mkdir -p {SANDBOX_ROOT} {SANDBOX_OUTPUT} {SANDBOX_SCRATCH} && " + f"tar -xzf {SANDBOX_ROOT}/agent-input.tar.gz -C {SANDBOX_ROOT} && " + f"tar -xzf {SANDBOX_ROOT}/runtime.tar.gz -C {SANDBOX_ROOT} && " + f"chmod -R a+rX,a-w {SANDBOX_AGENT_SOURCE} {SANDBOX_AGENT_DEPS} {SANDBOX_VDR} {SANDBOX_SKILLS}" + ) + result = await sandbox.exec(command, timeout_s=self.config.sandbox_staging_timeout_seconds) + if result.return_code != 0: + raise RuntimeError(result.stderr or result.stdout or "Failed to stage LAB agent sandbox") + + async def _collect_agent_sandbox(self, sandbox: AsyncSandbox, download_dir: Path) -> dict[str, Path]: + downloads: dict[str, Path] = {} + for filename in ("response.json", "runner_status.json"): + destination = download_dir / filename + try: + await sandbox.download(f"{SANDBOX_RUNTIME}/{filename}", destination) + except Exception: + continue + downloads[filename] = destination + + archive_path = download_dir / "output.tar.gz" + archive_result = await sandbox.exec( + f"tar -czf {SANDBOX_ROOT}/output.tar.gz -C {SANDBOX_OUTPUT} .", + timeout_s=self.config.sandbox_staging_timeout_seconds, + ) + if archive_result.return_code != 0: + raise RuntimeError(archive_result.stderr or archive_result.stdout or "Failed to collect LAB output") + await sandbox.download(f"{SANDBOX_ROOT}/output.tar.gz", archive_path) + downloads["output.tar.gz"] = archive_path + return downloads + + @staticmethod + def _materialize_agent_downloads( + paths: dict[str, Path], + downloads: dict[str, Path], + *, + stdout: str, + stderr: str, + ) -> None: + (paths["agent"] / "stdout.log").write_text(stdout) + (paths["agent"] / "stderr.log").write_text(stderr) + for filename in ("response.json", "runner_status.json"): + source = downloads.get(filename) + if source is not None: + _copy_downloaded_file(source, paths["runtime"] / filename) + output_archive = downloads.get("output.tar.gz") + if output_archive is None: + raise LegalAgentBenchArtifactError("Agent sandbox did not return an output archive") + _extract_untrusted_archive(output_archive, paths["output"]) + + async def _stage_and_run_verifier( + self, + sandbox: AsyncSandbox, + task_dir: Path, + paths: dict[str, Path], + ) -> tuple[dict[str, bytes], bool, Optional[str]]: + with tempfile.NamedTemporaryFile(suffix=".tar.gz", delete=False) as temporary: + archive_path = Path(temporary.name) + try: + _create_archive( + archive_path, + [ + (paths["lab_run"], "logs/agent/artifacts/lab-run"), + (task_dir / "tests", "tests"), + ], + ) + _prepare_sandbox_upload(archive_path) + root_result = await sandbox.exec( + f"mkdir -p {SANDBOX_ROOT}", timeout_s=self.config.sandbox_staging_timeout_seconds + ) + if root_result.return_code != 0 or root_result.error_type is not None: + reason = root_result.stderr or root_result.stdout or "Failed to create LAB verifier sandbox root" + return {}, root_result.error_type == "timeout", reason[-2000:] + await sandbox.upload(archive_path, f"{SANDBOX_ROOT}/verifier-input.tar.gz") + finally: + archive_path.unlink(missing_ok=True) + + stage_command = ( + f"rm -rf {SANDBOX_VERIFIER} && mkdir -p {SANDBOX_VERIFIER} && " + f"tar -xzf {SANDBOX_ROOT}/verifier-input.tar.gz -C {SANDBOX_VERIFIER} && " + f"mkdir -p {SANDBOX_LOGS}/verifier && " + f"chmod -R a-w {SANDBOX_LOGS}/agent" + ) + stage_result = await sandbox.exec( + stage_command, + cwd="/tmp", + timeout_s=self.config.sandbox_staging_timeout_seconds, + ) + if stage_result.return_code != 0 or stage_result.error_type is not None: + timed_out = stage_result.error_type == "timeout" + reason = stage_result.stderr or stage_result.stdout or "Failed to stage LAB verifier sandbox" + return {}, timed_out, reason[-2000:] + + verifier_env = { + **_verifier_env(task_dir), + "LAB_TESTS_DIR": SANDBOX_TESTS, + "LAB_LOGS_DIR": SANDBOX_LOGS, + } + result = await sandbox.exec( + f"bash {SANDBOX_TESTS}/test.sh", + cwd=f"{SANDBOX_LOGS}/agent/artifacts/lab-run/output", + env=verifier_env, + timeout_s=self.config.verifier_timeout_seconds, + ) + + verifier_filenames = ("reward.json", "scores.json", "transcript.jsonl", "report.html", "error.json") + manifest_command = ( + "for filename in " + + " ".join(verifier_filenames) + + f'; do test -f "{SANDBOX_LOGS}/verifier/$filename" && printf "%s\\n" "$filename"; done; true' + ) + manifest_result = await sandbox.exec(manifest_command, cwd="/tmp", timeout_s=60) + if manifest_result.return_code != 0 or manifest_result.error_type is not None: + reason = manifest_result.stderr or manifest_result.stdout or "Failed to list LAB verifier artifacts" + return {}, manifest_result.error_type == "timeout", reason[-2000:] + available = set(manifest_result.stdout.splitlines()) & set(verifier_filenames) + + downloaded: dict[str, bytes] = {} + with tempfile.TemporaryDirectory(prefix="legal-agent-bench-verifier-") as temporary_dir: + temporary_path = Path(temporary_dir) + for filename in verifier_filenames: + if filename not in available: + continue + destination = temporary_path / filename + await sandbox.download(f"{SANDBOX_LOGS}/verifier/{filename}", destination) + if not stat.S_ISREG(destination.lstat().st_mode): + return {}, result.error_type == "timeout", f"Verifier returned unsafe {filename}" + downloaded[filename] = destination.read_bytes() + + timed_out = result.error_type == "timeout" + if "reward.json" not in downloaded: + reason = result.stderr or result.stdout or "LAB verifier did not produce reward.json" + return {}, timed_out, reason[-2000:] + return downloaded, timed_out, None + + @staticmethod + def _materialize_verifier_downloads(paths: dict[str, Path], downloaded: dict[str, bytes]) -> dict[str, Any]: + try: + reward_data = json.loads(downloaded["reward.json"].decode("utf-8")) + except (KeyError, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise LegalAgentBenchArtifactError(f"Invalid verifier reward.json: {exc}") from exc + if not isinstance(reward_data, dict): + raise LegalAgentBenchArtifactError("Invalid verifier reward.json: expected an object") + normalized = _normalized_reward_data(reward_data) + for filename, contents in downloaded.items(): + (paths["verifier"] / filename).write_bytes(contents) + return normalized + + @staticmethod + def _runner_status(paths: dict[str, Path]) -> dict[str, Any]: + status_path = paths["runtime"] / "runner_status.json" + if not status_path.is_file(): + return {} + try: + status = json.loads(status_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + return {"ok": False, "phase": "agent_execution", "error": f"Invalid runner status: {exc}"} + return ( + status + if isinstance(status, dict) + else { + "ok": False, + "phase": "agent_execution", + "error": "Invalid runner status: expected an object", + } + ) + + def _artifacts( + self, + paths: dict[str, Path], + *, + task_name: str, + model_name: str, + agent_elapsed: float, + response: NeMoGymResponse, + failure_reason: Optional[str], + ) -> None: + config = { + "agent_id": agent_key(self.config.agent_server_module), + "agent_config_id": self.config.name, + "model": model_name, + "task": task_name, + "run_id": paths["root"].name, + "tool_runtime": "gym-agent-in-sandbox", + "agent_server_module": self.config.agent_server_module, + "agent_server_class": self.config.agent_server_class, + "skills": list(REQUIRED_SKILLS), + } + metrics = { + "model": model_name, + "task": task_name, + "run_id": paths["root"].name, + "wall_clock_seconds": round(agent_elapsed, 3), + "agent_error": failure_reason, + } + (paths["lab_run"] / "config.json").write_text(json.dumps(config, indent=2)) + (paths["lab_run"] / "metrics.json").write_text(json.dumps(metrics, indent=2)) + (paths["agent"] / "trajectory.json").write_text(response.model_dump_json(indent=2)) + + def _write_run_summary(self, result: LegalAgentBenchAgentResponse) -> None: + if not result.artifact_dir: + return + summary_path = Path(result.artifact_dir) / "run_summary.json" + output_dir = Path(result.output_dir) if result.output_dir else None + output_files = ( + sorted(str(path.relative_to(output_dir)) for path in output_dir.rglob("*") if path.is_file()) + if output_dir and output_dir.is_dir() + else [] + ) + summary = { + "instance_id": result.instance_id, + "reward": result.reward, + "criteria_pass_rate": result.criteria_pass_rate, + "mask_sample": result.mask_sample, + "failure_reason": result.failure_reason, + "flags": { + "agent_failed": result.agent_failed, + "model_connection_failed": result.model_connection_failed, + "agent_timed_out": result.agent_timed_out, + "verifier_failed": result.verifier_failed, + "verifier_timed_out": result.verifier_timed_out, + "sandbox_failed": result.sandbox_failed, + "task_failed": result.task_failed, + "configuration_failed": result.configuration_failed, + "judge_error_count": result.judge_error_count, + "verifier_error": result.verifier_error, + }, + "paths": { + "artifact_dir": result.artifact_dir, + "agent_trace": result.agent_trace_path, + "agent_stdout": result.agent_stdout_path, + "agent_stderr": result.agent_stderr_path, + "verifier_report": result.verifier_report_path, + "output_dir": result.output_dir, + }, + "output_files": output_files, + } + summary_path.write_text(json.dumps(summary, indent=2)) + if result.mask_sample: + print( + f"LAB rollout failed: {result.failure_reason or 'unreliable result'}; artifacts={result.artifact_dir}", + flush=True, + ) + else: + print( + f"LAB rollout complete: reward={result.reward} " + f"criteria_pass_rate={result.criteria_pass_rate} artifacts={result.artifact_dir}", + flush=True, + ) + + def _response( + self, + *, + body: LegalAgentBenchRunRequest, + params: NeMoGymResponseCreateParamsNonStreaming, + response: NeMoGymResponse, + reward_data: dict[str, Any], + paths: Optional[dict[str, Path]], + agent_failed: bool = False, + model_connection_failed: bool = False, + agent_timed_out: bool = False, + verifier_failed: bool = False, + verifier_timed_out: bool = False, + sandbox_failed: bool = False, + task_failed: bool = False, + configuration_failed: bool = False, + failure_reason: Optional[str] = None, + ) -> LegalAgentBenchAgentResponse: + try: + normalized_reward = _normalized_reward_data(reward_data) + except LegalAgentBenchArtifactError as exc: + normalized_reward = _normalized_reward_data({}) + verifier_failed = True + failure_reason = failure_reason or str(exc) + verifier_error = normalized_reward["verifier_error"] + judge_errors = normalized_reward["judge_error_count"] + verifier_failed = verifier_failed or bool(verifier_error or judge_errors) + if failure_reason is None and judge_errors: + suffix = "error" if judge_errors == 1 else "errors" + failure_reason = f"Verifier reported {judge_errors} judge {suffix}" + elif failure_reason is None and verifier_error: + failure_reason = "Verifier reported an internal error" + unreliable = bool( + agent_failed + or model_connection_failed + or agent_timed_out + or verifier_failed + or verifier_timed_out + or sandbox_failed + or task_failed + or configuration_failed + or verifier_error + or judge_errors + ) + failure_class: Optional[str] = None + failure_terminal = False + if task_failed: + failure_class = "task_failed" + failure_terminal = True + elif configuration_failed: + failure_class = "configuration_failed" + failure_terminal = True + elif model_connection_failed: + failure_class = "model_connection_failed" + elif sandbox_failed: + failure_class = "sandbox_failed" + elif verifier_failed or verifier_timed_out or verifier_error or judge_errors: + failure_class = "verifier_failed" + elif agent_timed_out: + failure_class = "agent_timed_out" + elif agent_failed: + failure_class = "agent_failed" + + routing: dict[str, Any] = {} + if failure_class is not None: + routing[NG_FAILURE_CLASS_KEY] = failure_class + if failure_terminal: + routing[NG_TERMINAL_KEY] = True + return LegalAgentBenchAgentResponse( + responses_create_params=params, + response=response, + reward=normalized_reward["reward"], + instance_id=body.instance_id, + criteria_pass_rate=normalized_reward["criteria_pass_rate"], + judge_error_count=judge_errors, + verifier_error=verifier_error, + mask_sample=unreliable, + agent_failed=agent_failed, + model_connection_failed=model_connection_failed, + agent_timed_out=agent_timed_out, + verifier_failed=verifier_failed, + verifier_timed_out=verifier_timed_out, + sandbox_failed=sandbox_failed, + task_failed=task_failed, + configuration_failed=configuration_failed, + failure_reason=failure_reason, + artifact_dir=str(paths["root"]) if paths else None, + run_summary_path=str(paths["root"] / "run_summary.json") if paths else None, + agent_trace_path=str(paths["agent"] / "trajectory.json") if paths else None, + agent_stdout_path=str(paths["agent"] / "stdout.log") if paths else None, + agent_stderr_path=str(paths["agent"] / "stderr.log") if paths else None, + verifier_report_path=( + str(paths["verifier"] / "report.html") + if paths and (paths["verifier"] / "report.html").is_file() + else None + ), + output_dir=str(paths["output"]) if paths else None, + **routing, + ) + + async def run(self, request: Request, body: LegalAgentBenchRunRequest) -> LegalAgentBenchAgentResponse: + async with self._sem: + model_name = self.config.model_server.name + params = body.responses_create_params + response = _empty_response(model_name) + paths: Optional[dict[str, Path]] = None + sandbox: Optional[AsyncSandbox] = None + agent_failed = agent_timed_out = verifier_failed = verifier_timed_out = sandbox_failed = False + model_connection_failed = False + task_failed = configuration_failed = False + failure_reason: Optional[str] = None + reward_data: dict[str, Any] = {} + verifier_sandbox: Optional[AsyncSandbox] = None + staged_paths: Optional[dict[str, Path]] = None + final_root: Optional[Path] = None + staging_temp: Optional[tempfile.TemporaryDirectory[str]] = None + + try: + model_name = self._model_name() + response = _empty_response(model_name) + task_dir = resolve_task_dir(self.config.runtime_tasks_dir, body.instance_id) + skills_dir = resolve_repo_path(self.config.skills_dir) + if self.config.agent_server_module == NATIVE_AGENT_MODULE: + params = compose_agent_input(task_dir, skills_dir, params, native=True) + else: + params = compose_agent_input(task_dir, skills_dir, params) + final_root = self._run_root(task_dir.name) + staging_temp = tempfile.TemporaryDirectory(prefix="legal-agent-bench-stage-") + staged_paths = self._paths_for_root(Path(staging_temp.name), create=True) + image = await self._ensure_image(task_dir) + runtime_archive = await self._ensure_runtime(image) + self._stage_agent_source(staged_paths) + model_url = self._model_url(body) + self._write_runner_config(staged_paths, params, model_url) + + sandbox = self._agent_sandbox( + image=image, + task_dir=task_dir, + model_url=model_url, + ) + await sandbox.start() + await self._stage_agent_sandbox( + sandbox, + task_dir=task_dir, + skills_dir=skills_dir, + runtime_archive=runtime_archive, + paths=staged_paths, + ) + with tempfile.TemporaryDirectory(prefix="legal-agent-bench-agent-") as download_raw: + started = time.time() + agent_result = await sandbox.exec( + f"{SANDBOX_AGENT_DEPS}/bin/python {SANDBOX_RUNTIME}/agent_runner.py", + cwd=SANDBOX_OUTPUT, + timeout_s=self.config.agent_timeout_seconds, + ) + agent_elapsed = time.time() - started + downloads = await self._collect_agent_sandbox(sandbox, Path(download_raw)) + await sandbox.stop() + sandbox = None + self._materialize_agent_downloads( + staged_paths, + downloads, + stdout=agent_result.stdout or "", + stderr=agent_result.stderr or "", + ) + paths = self._publish_staged_run(staged_paths, final_root) + staging_temp.cleanup() + staging_temp = None + agent_timed_out = agent_result.error_type == "timeout" + runner_status = self._runner_status(paths) + if runner_status.get("ok") is False: + agent_failed = True + model_connection_failed = runner_status.get("phase") == "model_connectivity" + failure_reason = str(runner_status.get("error") or "Agent runner failed")[-2000:] + if agent_result.return_code != 0: + agent_failed = True + failure_reason = ( + failure_reason or (agent_result.stderr or agent_result.stdout or "Agent runner failed")[-2000:] + ) + if agent_timed_out: + agent_failed = True + failure_reason = failure_reason or f"Agent timed out after {self.config.agent_timeout_seconds}s" + + response_path = paths["runtime"] / "response.json" + if response_path.is_file(): + try: + response = NeMoGymResponse.model_validate_json(response_path.read_text()) + except ValueError as exc: + agent_failed = True + failure_reason = failure_reason or f"Invalid agent response: {exc}" + else: + agent_failed = True + failure_reason = failure_reason or "Agent did not produce response.json" + response_model_connection_failed, response_agent_timed_out = agent_response_failure_flags( + response, self.config.agent_server_module + ) + model_connection_failed = model_connection_failed or response_model_connection_failed + agent_timed_out = agent_timed_out or response_agent_timed_out + if not agent_failed: + response_failure = agent_response_failure(response, self.config.agent_server_module) + if response_failure: + agent_failed = True + failure_reason = response_failure + self._artifacts( + paths, + task_name=task_dir.name, + model_name=model_name, + agent_elapsed=agent_elapsed, + response=response, + failure_reason=failure_reason, + ) + + if not agent_failed: + verifier_sandbox = self._verifier_sandbox(image=image, task_dir=task_dir) + await verifier_sandbox.start() + verifier_downloads, verifier_timed_out, verifier_failure = await self._stage_and_run_verifier( + verifier_sandbox, task_dir, paths + ) + await verifier_sandbox.stop() + verifier_sandbox = None + if verifier_failure is None: + try: + reward_data = self._materialize_verifier_downloads(paths, verifier_downloads) + except LegalAgentBenchArtifactError as exc: + verifier_failure = str(exc) + verifier_failed = verifier_failure is not None + failure_reason = verifier_failure or failure_reason + except LegalAgentBenchTaskError as exc: + task_failed = True + failure_reason = f"{type(exc).__name__}: {exc}" + except LegalAgentBenchConfigurationError as exc: + configuration_failed = True + failure_reason = f"{type(exc).__name__}: {exc}" + except Exception as exc: + sandbox_failed = True + failure_reason = f"{type(exc).__name__}: {exc}" + finally: + if sandbox is not None: + try: + await sandbox.stop() + except Exception as exc: + sandbox_failed = True + failure_reason = failure_reason or f"Sandbox cleanup failed: {exc}" + if verifier_sandbox is not None: + try: + await verifier_sandbox.stop() + except Exception as exc: + sandbox_failed = True + failure_reason = failure_reason or f"Verifier sandbox cleanup failed: {exc}" + if staging_temp is not None: + if staged_paths is not None and final_root is not None: + try: + paths = self._publish_staged_run(staged_paths, final_root) + except Exception as exc: + sandbox_failed = True + failure_reason = failure_reason or f"Artifact publication failed: {exc}" + staging_temp.cleanup() + + result = self._response( + body=body, + params=params, + response=response, + reward_data=reward_data, + paths=paths, + agent_failed=agent_failed, + model_connection_failed=model_connection_failed, + agent_timed_out=agent_timed_out, + verifier_failed=verifier_failed, + verifier_timed_out=verifier_timed_out, + sandbox_failed=sandbox_failed, + task_failed=task_failed, + configuration_failed=configuration_failed, + failure_reason=failure_reason, + ) + self._write_run_summary(result) + return result + + +if __name__ == "__main__": + LegalAgentBenchAgent.run_webserver() diff --git a/responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_claude_code.yaml b/responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_claude_code.yaml new file mode 100644 index 0000000000..25ba73a8ba --- /dev/null +++ b/responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_claude_code.yaml @@ -0,0 +1,50 @@ +config_paths: + - resources_servers/legal_agent_bench/configs/resources_only.yaml + +legal_agent_bench_claude_code_agent: + responses_api_agents: + legal_agent_bench_agent: + entrypoint: app.py + domain: agent + description: Legal Agent Bench with a config-selected Claude Code agent harness. + value: Evaluate legal-document agents independently of Harbor orchestration. + resources_server: + type: resources_servers + name: legal_agent_bench + model_server: + type: responses_api_models + name: policy_model + runtime_tasks_dir: ${legal_agent_bench.resources_servers.legal_agent_bench.harbor_tasks_dir} + skills_dir: ${legal_agent_bench.resources_servers.legal_agent_bench.harness_skills_dir} + concurrency: 1 + agent_timeout_seconds: 10800 + verifier_timeout_seconds: 3600 + sandbox_staging_timeout_seconds: 900 + results_dir: ${oc.env:NEMO_GYM_LAB_RESULTS_DIR,results/legal_agent_bench} + sandbox_provider: + docker: {} + sandbox_image: ${oc.env:NEMO_GYM_LAB_SANDBOX_IMAGE,null} + runtime_builder_provider_options: ${oc.decode:${oc.env:NEMO_GYM_LAB_RUNTIME_BUILDER_PROVIDER_OPTIONS,{}}} + agent_sandbox_provider_options: ${oc.decode:${oc.env:NEMO_GYM_LAB_AGENT_SANDBOX_PROVIDER_OPTIONS,{}}} + verifier_sandbox_provider_options: ${oc.decode:${oc.env:NEMO_GYM_LAB_VERIFIER_SANDBOX_PROVIDER_OPTIONS,{}}} + opensandbox_request_fraction: 0.25 + sandbox_model_base_url: ${oc.env:NEMO_GYM_SANDBOX_MODEL_BASE_URL,null} + sandbox_model_api_key_env: ${oc.env:NEMO_GYM_LAB_SANDBOX_MODEL_API_KEY_ENV,null} + agent_server_module: responses_api_agents.claude_code_agent.app + agent_server_class: ClaudeCodeAgent + agent_config_class: ClaudeCodeAgentConfig + agent_kwargs: + model: ${policy_model_name} + max_turns: null + timeout: 10800 + bare: true + claude_code_version: 2.1.211 + datasets: + - name: example + type: example + jsonl_fpath: resources_servers/legal_agent_bench/data/example.jsonl + license: MIT + - name: legal_agent_bench + type: validation + jsonl_fpath: resources_servers/legal_agent_bench/data/generated/all.jsonl + license: MIT diff --git a/responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_codex.yaml b/responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_codex.yaml new file mode 100644 index 0000000000..2b5a7bd545 --- /dev/null +++ b/responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_codex.yaml @@ -0,0 +1,50 @@ +config_paths: + - resources_servers/legal_agent_bench/configs/resources_only.yaml + +legal_agent_bench_codex_agent: + responses_api_agents: + legal_agent_bench_agent: + entrypoint: app.py + domain: agent + description: Legal Agent Bench with a config-selected Codex agent harness. + value: Evaluate legal-document agents independently of Harbor orchestration. + resources_server: + type: resources_servers + name: legal_agent_bench + model_server: + type: responses_api_models + name: policy_model + runtime_tasks_dir: ${legal_agent_bench.resources_servers.legal_agent_bench.harbor_tasks_dir} + skills_dir: ${legal_agent_bench.resources_servers.legal_agent_bench.harness_skills_dir} + concurrency: 1 + agent_timeout_seconds: 10800 + verifier_timeout_seconds: 3600 + sandbox_staging_timeout_seconds: 900 + results_dir: ${oc.env:NEMO_GYM_LAB_RESULTS_DIR,results/legal_agent_bench} + sandbox_provider: + docker: {} + sandbox_image: ${oc.env:NEMO_GYM_LAB_SANDBOX_IMAGE,null} + runtime_builder_provider_options: ${oc.decode:${oc.env:NEMO_GYM_LAB_RUNTIME_BUILDER_PROVIDER_OPTIONS,{}}} + agent_sandbox_provider_options: ${oc.decode:${oc.env:NEMO_GYM_LAB_AGENT_SANDBOX_PROVIDER_OPTIONS,{}}} + verifier_sandbox_provider_options: ${oc.decode:${oc.env:NEMO_GYM_LAB_VERIFIER_SANDBOX_PROVIDER_OPTIONS,{}}} + opensandbox_request_fraction: 0.25 + sandbox_model_base_url: ${oc.env:NEMO_GYM_SANDBOX_MODEL_BASE_URL,null} + sandbox_model_api_key_env: ${oc.env:NEMO_GYM_LAB_SANDBOX_MODEL_API_KEY_ENV,null} + agent_server_module: responses_api_agents.codex_agent.app + agent_server_class: CodexAgent + agent_config_class: CodexAgentConfig + agent_kwargs: + model: null + sandbox_mode: danger-full-access + timeout: 10800 + codex_version: 0.144.4 + cwd: /sandbox/nemo-gym-legal-agent-bench/workspace/output + datasets: + - name: example + type: example + jsonl_fpath: resources_servers/legal_agent_bench/data/example.jsonl + license: MIT + - name: legal_agent_bench + type: validation + jsonl_fpath: resources_servers/legal_agent_bench/data/generated/all.jsonl + license: MIT diff --git a/responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_hermes.yaml b/responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_hermes.yaml new file mode 100644 index 0000000000..851c18a6ed --- /dev/null +++ b/responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_hermes.yaml @@ -0,0 +1,50 @@ +config_paths: + - resources_servers/legal_agent_bench/configs/resources_only.yaml + +legal_agent_bench_hermes_agent: + responses_api_agents: + legal_agent_bench_agent: + entrypoint: app.py + domain: agent + description: Legal Agent Bench with a config-selected Hermes agent harness. + value: Evaluate legal-document agents independently of Harbor orchestration. + resources_server: + type: resources_servers + name: legal_agent_bench + model_server: + type: responses_api_models + name: policy_model + runtime_tasks_dir: ${legal_agent_bench.resources_servers.legal_agent_bench.harbor_tasks_dir} + skills_dir: ${legal_agent_bench.resources_servers.legal_agent_bench.harness_skills_dir} + concurrency: 1 + agent_timeout_seconds: 10800 + verifier_timeout_seconds: 3600 + sandbox_staging_timeout_seconds: 900 + results_dir: ${oc.env:NEMO_GYM_LAB_RESULTS_DIR,results/legal_agent_bench} + sandbox_provider: + docker: {} + sandbox_image: ${oc.env:NEMO_GYM_LAB_SANDBOX_IMAGE,null} + runtime_builder_provider_options: ${oc.decode:${oc.env:NEMO_GYM_LAB_RUNTIME_BUILDER_PROVIDER_OPTIONS,{}}} + agent_sandbox_provider_options: ${oc.decode:${oc.env:NEMO_GYM_LAB_AGENT_SANDBOX_PROVIDER_OPTIONS,{}}} + verifier_sandbox_provider_options: ${oc.decode:${oc.env:NEMO_GYM_LAB_VERIFIER_SANDBOX_PROVIDER_OPTIONS,{}}} + opensandbox_request_fraction: 0.25 + sandbox_model_base_url: ${oc.env:NEMO_GYM_SANDBOX_MODEL_BASE_URL,null} + sandbox_model_api_key_env: ${oc.env:NEMO_GYM_LAB_SANDBOX_MODEL_API_KEY_ENV,null} + agent_server_module: responses_api_agents.hermes_agent.app + agent_server_class: HermesAgent + agent_config_class: HermesAgentConfig + agent_kwargs: + max_turns: 90 + terminal_backend: local + terminal_timeout: 180 + compression_enabled: true + compression_threshold: 0.85 + datasets: + - name: example + type: example + jsonl_fpath: resources_servers/legal_agent_bench/data/example.jsonl + license: MIT + - name: legal_agent_bench + type: validation + jsonl_fpath: resources_servers/legal_agent_bench/data/generated/all.jsonl + license: MIT diff --git a/responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_native.yaml b/responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_native.yaml new file mode 100644 index 0000000000..f75bba0ed6 --- /dev/null +++ b/responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_native.yaml @@ -0,0 +1,50 @@ +config_paths: + - resources_servers/legal_agent_bench/configs/resources_only.yaml + +legal_agent_bench_native_agent: + responses_api_agents: + legal_agent_bench_agent: + entrypoint: app.py + domain: agent + description: Legal Agent Bench with its direct Gym-native tool loop. + value: Evaluate legal-document agents without an external agent harness. + resources_server: + type: resources_servers + name: legal_agent_bench + model_server: + type: responses_api_models + name: policy_model + runtime_tasks_dir: ${legal_agent_bench.resources_servers.legal_agent_bench.harbor_tasks_dir} + skills_dir: ${legal_agent_bench.resources_servers.legal_agent_bench.harness_skills_dir} + concurrency: 1 + agent_timeout_seconds: 10800 + verifier_timeout_seconds: 3600 + sandbox_staging_timeout_seconds: 900 + results_dir: ${oc.env:NEMO_GYM_LAB_RESULTS_DIR,results/legal_agent_bench} + sandbox_provider: + docker: {} + sandbox_image: ${oc.env:NEMO_GYM_LAB_SANDBOX_IMAGE,null} + runtime_builder_provider_options: ${oc.decode:${oc.env:NEMO_GYM_LAB_RUNTIME_BUILDER_PROVIDER_OPTIONS,{}}} + agent_sandbox_provider_options: ${oc.decode:${oc.env:NEMO_GYM_LAB_AGENT_SANDBOX_PROVIDER_OPTIONS,{}}} + verifier_sandbox_provider_options: ${oc.decode:${oc.env:NEMO_GYM_LAB_VERIFIER_SANDBOX_PROVIDER_OPTIONS,{}}} + opensandbox_request_fraction: 0.25 + sandbox_model_base_url: ${oc.env:NEMO_GYM_SANDBOX_MODEL_BASE_URL,null} + sandbox_model_api_key_env: ${oc.env:NEMO_GYM_LAB_SANDBOX_MODEL_API_KEY_ENV,null} + agent_server_module: responses_api_agents.legal_agent_bench_native_agent.app + agent_server_class: LegalAgentBenchNativeAgent + agent_config_class: LegalAgentBenchNativeAgentConfig + agent_kwargs: + max_turns: 60 + shell_timeout: 60 + preflight_timeout_seconds: 120 + model_timeout_seconds: 1800 + max_output_chars: 16384 + datasets: + - name: example + type: example + jsonl_fpath: resources_servers/legal_agent_bench/data/example.jsonl + license: MIT + - name: legal_agent_bench + type: validation + jsonl_fpath: resources_servers/legal_agent_bench/data/generated/all.jsonl + license: MIT diff --git a/responses_api_agents/legal_agent_bench_agent/requirements.txt b/responses_api_agents/legal_agent_bench_agent/requirements.txt new file mode 100644 index 0000000000..eae22a6b0c --- /dev/null +++ b/responses_api_agents/legal_agent_bench_agent/requirements.txt @@ -0,0 +1 @@ +-e nemo-gym[dev,sandbox] @ ../../ diff --git a/responses_api_agents/legal_agent_bench_agent/scripts/smoke_provider.py b/responses_api_agents/legal_agent_bench_agent/scripts/smoke_provider.py new file mode 100644 index 0000000000..d915eaa6a1 --- /dev/null +++ b/responses_api_agents/legal_agent_bench_agent/scripts/smoke_provider.py @@ -0,0 +1,113 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Exercise the shared sandbox contract used by Legal Agent Bench.""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import shlex +import tempfile +from pathlib import Path +from uuid import uuid4 + +from omegaconf import OmegaConf + +from nemo_gym.sandbox import AsyncSandbox, SandboxSpec +from nemo_gym.sandbox.config import resolve_provider_config + + +def _provider(args: argparse.Namespace) -> dict: + if args.config is None: + return {args.provider: {}} + config = OmegaConf.load(args.config) + resolved = OmegaConf.to_container(config, resolve=True) + if not isinstance(resolved, dict): + raise ValueError(f"Sandbox config must resolve to a mapping: {args.config}") + return resolve_provider_config(args.sandbox_name, resolved) + + +async def _smoke(args: argparse.Namespace) -> dict[str, object]: + provider = _provider(args) + provider_name = next(iter(provider)) + token = uuid4().hex + remote_root = f"{args.workdir.rstrip('/')}/nemo-gym-lab-provider-smoke-{token[:8]}" + input_path = f"{remote_root}/input.txt" + output_path = f"{remote_root}/output.txt" + sandbox = AsyncSandbox( + provider, + SandboxSpec( + image=args.image, + ttl_s=args.ttl, + ready_timeout_s=args.ready_timeout, + workdir=args.workdir, + metadata={"benchmark": "legal-agent-bench", "purpose": "provider-smoke"}, + ), + ) + with tempfile.TemporaryDirectory(prefix="legal-agent-bench-provider-smoke-") as temporary: + temporary_root = Path(temporary) + source = temporary_root / "input.txt" + downloaded = temporary_root / "output.txt" + source.write_text(token) + try: + await sandbox.start() + await sandbox.upload(source, input_path) + command = ( + f'test "$(cat {shlex.quote(input_path)})" = {shlex.quote(token)} && ' + f"printf %s {shlex.quote(token)} > {shlex.quote(output_path)}" + ) + result = await sandbox.exec(command, timeout_s=args.timeout) + if result.return_code != 0 or result.error_type is not None: + raise RuntimeError( + f"sandbox exec failed: return_code={result.return_code}, " + f"error_type={result.error_type!r}, stderr={result.stderr!r}" + ) + await sandbox.download(output_path, downloaded) + if downloaded.read_text() != token: + raise RuntimeError("sandbox upload/exec/download round trip changed the payload") + finally: + await sandbox.stop() + return { + "provider": provider_name, + "image": args.image, + "start": "passed", + "exec": "passed", + "upload_download": "passed", + "cleanup": "passed", + } + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + selection = parser.add_mutually_exclusive_group(required=True) + selection.add_argument("--provider", help="Provider name with its default configuration") + selection.add_argument("--config", type=Path, help="Gym provider YAML containing a named sandbox block") + parser.add_argument("--sandbox-name", default="sandbox", help="Named sandbox block in --config") + parser.add_argument("--image", required=True, help="Provider-compatible LAB image reference") + parser.add_argument("--workdir", default="/tmp", help="Writable directory in the sandbox image") + parser.add_argument("--timeout", type=float, default=300, help="Command timeout in seconds") + parser.add_argument("--ready-timeout", type=float, default=900, help="Sandbox readiness timeout in seconds") + parser.add_argument("--ttl", type=float, default=1200, help="Sandbox lifetime in seconds") + return parser + + +def main() -> None: + result = asyncio.run(_smoke(_parser().parse_args())) + print(json.dumps(result, indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/responses_api_agents/legal_agent_bench_agent/setup_scripts/_portable_python.sh b/responses_api_agents/legal_agent_bench_agent/setup_scripts/_portable_python.sh new file mode 100755 index 0000000000..2319f9da62 --- /dev/null +++ b/responses_api_agents/legal_agent_bench_agent/setup_scripts/_portable_python.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# Shared helper for a relocatable CPython under $DEPS_DIR. +set -euo pipefail + +export PYTHONNOUSERSITE=1 + +PORTABLE_PYTHON_VERSION="${PORTABLE_PYTHON_VERSION:-3.12.8}" +PORTABLE_PYTHON_RELEASE="${PORTABLE_PYTHON_RELEASE:-20241219}" +if [ -z "${PORTABLE_PYTHON_ARCH:-}" ]; then + case "$(uname -m)" in + x86_64) PORTABLE_PYTHON_ARCH="x86_64-unknown-linux-gnu" ;; + aarch64|arm64) PORTABLE_PYTHON_ARCH="aarch64-unknown-linux-gnu" ;; + *) + echo "Unsupported portable Python architecture: $(uname -m)" >&2 + exit 1 + ;; + esac +fi + +install_portable_python() { + if [ -x "$DEPS_DIR/bin/python3" ]; then + return 0 + fi + local url="https://github.com/astral-sh/python-build-standalone/releases/download/${PORTABLE_PYTHON_RELEASE}/cpython-${PORTABLE_PYTHON_VERSION}+${PORTABLE_PYTHON_RELEASE}-${PORTABLE_PYTHON_ARCH}-install_only.tar.gz" + curl -fsSL "$url" | tar xz -C "$DEPS_DIR" --strip-components=1 + "$DEPS_DIR/bin/python3" -m pip install --upgrade pip +} + +install_nemo_gym_deps() { + local build_root + build_root="$(mktemp -d)" + mkdir -p "$build_root/cache" + cp "$NEMO_GYM_ROOT/pyproject.toml" "$NEMO_GYM_ROOT/README.md" "$build_root/" + cp -a "$NEMO_GYM_ROOT/nemo_gym" "$build_root/" + "$DEPS_DIR/bin/python3" -m pip install "$build_root" + rm -rf "$build_root" +} diff --git a/responses_api_agents/legal_agent_bench_agent/tests/__init__.py b/responses_api_agents/legal_agent_bench_agent/tests/__init__.py new file mode 100644 index 0000000000..52a7a9daf0 --- /dev/null +++ b/responses_api_agents/legal_agent_bench_agent/tests/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/responses_api_agents/legal_agent_bench_agent/tests/test_app.py b/responses_api_agents/legal_agent_bench_agent/tests/test_app.py new file mode 100644 index 0000000000..57f7b33f7c --- /dev/null +++ b/responses_api_agents/legal_agent_bench_agent/tests/test_app.py @@ -0,0 +1,2362 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Tests for the configurable Legal Agent Bench Gym-agent runner.""" + +from __future__ import annotations + +import io +import json +import os +import sys +import tarfile +from pathlib import Path +from types import SimpleNamespace + +import pytest +from omegaconf import OmegaConf + +from nemo_gym.config_types import ModelServerRef, ResourcesServerRef +from nemo_gym.openai_utils import NeMoGymResponseCreateParamsNonStreaming +from responses_api_agents.legal_agent_bench_agent import app +from responses_api_agents.legal_agent_bench_agent.scripts import smoke_provider + + +class _RuntimeBuilderSandbox: + instances = [] + + def __init__(self, provider, spec): + self.provider = provider + self.spec = spec + self.upload_members = set() + self.exec_calls = [] + self.stopped = False + self.__class__.instances.append(self) + + async def start(self): + return self + + async def upload(self, source, destination): + assert destination == f"{app.SANDBOX_ROOT}/runtime-builder-input.tar.gz" + assert Path(source).stat().st_mode & 0o777 == 0o644 + with tarfile.open(source, "r:gz") as archive: + self.upload_members = {member.name for member in archive.getmembers()} + + async def exec(self, command, **kwargs): + assert "user" not in kwargs + self.exec_calls.append((command, kwargs)) + return SimpleNamespace(return_code=0, error_type=None, stdout="", stderr="") + + async def download(self, source, destination): + assert source == f"{app.SANDBOX_ROOT}/runtime-builder-output.tar.gz" + payload = b"#!/bin/sh\n" + with tarfile.open(destination, "w:gz") as archive: + root = tarfile.TarInfo("agent_deps") + root.type = tarfile.DIRTYPE + archive.addfile(root) + executable = tarfile.TarInfo("agent_deps/bin/python") + executable.mode = 0o755 + executable.size = len(payload) + archive.addfile(executable, io.BytesIO(payload)) + + async def stop(self): + self.stopped = True + + +def _config(**overrides) -> app.LegalAgentBenchAgentConfig: + values = { + "host": "0.0.0.0", + "port": 10000, + "name": "lab_test_agent", + "entrypoint": "app.py", + "resources_server": ResourcesServerRef(name="lab", type="resources_servers"), + "model_server": ModelServerRef(name="policy_model", type="responses_api_models"), + "agent_server_module": "responses_api_agents.hermes_agent.app", + "agent_server_class": "HermesAgent", + "agent_config_class": "HermesAgentConfig", + } + values.update(overrides) + return app.LegalAgentBenchAgentConfig(**values) + + +def _task_tree(tmp_path: Path, task_name: str = "area__task") -> tuple[Path, Path]: + root = tmp_path / "tasks" + task = root / task_name + for directory in ("documents", "environment/harness", "tests"): + (task / directory).mkdir(parents=True, exist_ok=True) + (task / "instruction.md").write_text("Do the task") + (task / "task.json").write_text( + json.dumps({"title": "Legal task", "instructions": "Write a memo.", "criteria": [{"id": "1"}]}) + ) + (task / "task.toml").write_text( + """ +version = "1.0" +[agent] +timeout_sec = 30 +[verifier] +timeout_sec = 60 +[verifier.env] +LAB_JUDGE_API_KEY = "secret" # pragma: allowlist secret +LAB_JUDGE_MODEL = "judge" +[environment] +cpus = 2 +memory_mb = 4096 +storage_mb = 10240 +""" + ) + (task / "environment" / "Dockerfile").write_text("FROM python:3.12-slim\nWORKDIR /workspace/output\n") + (task / "environment" / "harness" / "runner.py").write_text("print('ok')\n") + (task / "tests" / "test.sh").write_text("#!/bin/bash\n") + return root, task + + +def _skills(tmp_path: Path) -> Path: + root = tmp_path / "skills" + for name in app.REQUIRED_SKILLS: + (root / name).mkdir(parents=True) + (root / name / "SKILL.md").write_text(f"# {name}\nUse {name}.") + return root + + +def _runtime_sources(monkeypatch, tmp_path: Path, key: str, script_text: str = "installer-v1"): + package_dir = tmp_path / "legal_agent_bench_agent" + package_dir.mkdir() + portable = package_dir / "_portable_python.sh" + portable.write_text("portable-v1") + script = tmp_path / f"{key}_deps.sh" + script.write_text(script_text) + agent_dir = tmp_path / "responses_api_agents" / key + agent_dir.mkdir(parents=True) + (agent_dir / "requirements.txt").write_text("nemo-gym\n") + (tmp_path / "pyproject.toml").write_text("[project]\nname = 'nemo-gym'\n") + (tmp_path / "README.md").write_text("# Gym\n") + (tmp_path / "nemo_gym").mkdir() + (tmp_path / "nemo_gym" / "runtime.py").write_text("VERSION = 1\n") + monkeypatch.setattr(app, "PACKAGE_DIR", package_dir) + monkeypatch.setattr(app, "PARENT_DIR", tmp_path) + monkeypatch.setattr(app, "PORTABLE_PYTHON_SH", portable) + monkeypatch.setattr(app, "resolve_agent_setup_script", lambda _module: script) + return package_dir, script + + +def _successful_response(text: str = "Done") -> app.NeMoGymResponse: + return app.NeMoGymResponse.model_validate( + { + "id": "resp-success", + "created_at": 1, + "model": "policy", + "object": "response", + "output": [ + { + "id": "msg-success", + "content": [{"annotations": [], "text": text, "type": "output_text"}], + "role": "assistant", + "status": "completed", + "type": "message", + } + ], + "parallel_tool_calls": False, + "tool_choice": "auto", + "tools": [], + "usage": { + "input_tokens": 1, + "input_tokens_details": {"cached_tokens": 0}, + "output_tokens": 1, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 2, + }, + } + ) + + +@pytest.mark.parametrize( + ("module", "expected"), + [ + ("responses_api_agents.legal_agent_bench_native_agent.app", "legal_agent_bench_native_agent"), + ("responses_api_agents.hermes_agent.app", "hermes_agent"), + ("responses_api_agents.claude_code_agent.app", "claude_code_agent"), + ("responses_api_agents.codex_agent.app", "codex_agent"), + ], +) +def test_agent_key(module, expected) -> None: + assert app.agent_key(module) == expected + + +def test_repository_root_is_derived_from_agent_source() -> None: + assert app.PARENT_DIR == Path(app.__file__).resolve().parents[2] + assert (app.PARENT_DIR / "resources_servers" / "legal_agent_bench" / "vendor" / "harvey_labs").is_dir() + + +@pytest.mark.parametrize("module", ["hermes_agent", "responses_api_agents.bad-name.app", "x.y.z"]) +def test_agent_key_rejects_invalid_modules(module) -> None: + with pytest.raises(ValueError): + app.agent_key(module) + + +def test_all_supported_agents_have_dependency_scripts() -> None: + for module in ( + "responses_api_agents.legal_agent_bench_native_agent.app", + "responses_api_agents.hermes_agent.app", + "responses_api_agents.claude_code_agent.app", + "responses_api_agents.codex_agent.app", + ): + assert app.resolve_agent_setup_script(module).is_file() + + +@pytest.mark.asyncio +async def test_dependency_runtime_cache_is_harness_and_recipe_specific(monkeypatch, tmp_path) -> None: + package_dir, script = _runtime_sources(monkeypatch, tmp_path, "hermes_agent", "hermes-v1") + (tmp_path / "env.yaml").write_text("judge_key: must-not-be-uploaded\n") # pragma: allowlist secret + _RuntimeBuilderSandbox.instances = [] + monkeypatch.setattr(app, "AsyncSandbox", _RuntimeBuilderSandbox) + monkeypatch.setattr(app.shutil, "which", lambda _name: pytest.fail("runtime provisioning used Docker")) + + provider = {"apptainer": {"exec": {"concurrency": 2}}} + runner = app.LegalAgentBenchAgent.model_construct(config=_config(sandbox_provider=provider)) + runner._runtime_lock = app.asyncio.Lock() + runner._runtime_archives = {} + monkeypatch.setattr(runner, "_provider_config", lambda: provider) + monkeypatch.setattr(runner, "_sandbox_metadata", lambda: {}) + + first = await runner._ensure_runtime("lab:image") + second = await runner._ensure_runtime("lab:image") + script.write_text("hermes-v2") + third = await runner._ensure_runtime("lab:image") + (tmp_path / "pyproject.toml").write_text("[project]\nname = 'nemo-gym-updated'\n") + fourth = await runner._ensure_runtime("lab:image") + (tmp_path / "nemo_gym" / "runtime.py").write_text("VERSION = 2\n") + fifth = await runner._ensure_runtime("lab:image") + + assert first == second + assert len({first, third, fourth, fifth}) == 4 + assert all(path.parent == package_dir / ".deps" / "hermes_agent" for path in (first, third, fourth, fifth)) + assert all(path.is_file() for path in (first, third, fourth, fifth)) + assert len(_RuntimeBuilderSandbox.instances) == 4 + first_builder = _RuntimeBuilderSandbox.instances[0] + assert first_builder.provider == provider + assert first_builder.spec.workdir == "/tmp" + assert first_builder.spec.env == app.LAB_SANDBOX_ENV + assert first_builder.exec_calls[0][0] == f"mkdir -p {app.SANDBOX_ROOT}" + build_command, build_kwargs = first_builder.exec_calls[1] + assert f"export HOME={app.SANDBOX_ROOT}/runtime-builder/home" in build_command + assert "HOME" not in build_kwargs["env"] + assert build_kwargs["env"]["TMPDIR"] == f"{app.SANDBOX_ROOT}/runtime-builder/tmp" + assert not any("env.yaml" in name for name in first_builder.upload_members) + assert first_builder.stopped is True + + +@pytest.mark.parametrize( + ("module", "kwargs", "expected"), + [ + ( + "responses_api_agents.claude_code_agent.app", + {"claude_code_version": "2.1.211"}, + {"CLAUDE_SPEC": "@anthropic-ai/claude-code@2.1.211"}, + ), + ( + "responses_api_agents.codex_agent.app", + {"codex_version": "0.144.4"}, + {"CODEX_SPEC": "@openai/codex@0.144.4"}, + ), + ("responses_api_agents.hermes_agent.app", {}, {}), + ], +) +def test_agent_runtime_env_uses_configured_harness_pin(module, kwargs, expected) -> None: + assert app.agent_runtime_env(module, kwargs) == expected + + +@pytest.mark.parametrize( + ("module", "field"), + [ + ("responses_api_agents.claude_code_agent.app", "claude_code_version"), + ("responses_api_agents.codex_agent.app", "codex_version"), + ], +) +def test_agent_runtime_env_rejects_missing_harness_pin(module, field) -> None: + with pytest.raises(app.LegalAgentBenchConfigurationError, match=field): + app.agent_runtime_env(module, {}) + + +@pytest.mark.parametrize("version", ["latest", "next", "^1.2.3", "1.2", "1.2.3 || 2.0.0"]) +def test_agent_runtime_env_rejects_non_exact_harness_pin(version) -> None: + with pytest.raises(app.LegalAgentBenchConfigurationError, match="exact npm version"): + app.agent_runtime_env( + "responses_api_agents.codex_agent.app", + {"codex_version": version}, + ) + + +@pytest.mark.asyncio +async def test_dependency_provisioning_uses_pin_and_invalidates_when_it_changes(monkeypatch, tmp_path) -> None: + _runtime_sources(monkeypatch, tmp_path, "claude_code_agent", "claude") + _RuntimeBuilderSandbox.instances = [] + monkeypatch.setattr(app, "AsyncSandbox", _RuntimeBuilderSandbox) + provider = {"opensandbox": {"connection": {"domain": "sandbox.example"}}} + runner = app.LegalAgentBenchAgent.model_construct( + config=_config( + agent_server_module="responses_api_agents.claude_code_agent.app", + agent_kwargs={"claude_code_version": "2.1.211"}, + sandbox_provider=provider, + ) + ) + runner._runtime_lock = app.asyncio.Lock() + runner._runtime_archives = {} + monkeypatch.setattr(runner, "_provider_config", lambda: provider) + monkeypatch.setattr(runner, "_sandbox_metadata", lambda: {}) + + first = await runner._ensure_runtime("lab:image") + first_env = _RuntimeBuilderSandbox.instances[-1].exec_calls[1][1]["env"] + runner.config.agent_kwargs["claude_code_version"] = "2.1.212" + second = await runner._ensure_runtime("lab:image") + second_env = _RuntimeBuilderSandbox.instances[-1].exec_calls[1][1]["env"] + + assert first != second + assert first_env["CLAUDE_SPEC"] == "@anthropic-ai/claude-code@2.1.211" + assert second_env["CLAUDE_SPEC"] == "@anthropic-ai/claude-code@2.1.212" + + +def test_runtime_archive_accepts_internal_links_and_rejects_escaping_links(tmp_path) -> None: + archive_path = tmp_path / "runtime.tar.gz" + with tarfile.open(archive_path, "w:gz") as archive: + root = tarfile.TarInfo("agent_deps") + root.type = tarfile.DIRTYPE + archive.addfile(root) + target = tarfile.TarInfo("agent_deps/bin/python3") + target.size = 0 + archive.addfile(target, io.BytesIO()) + link = tarfile.TarInfo("agent_deps/bin/python") + link.type = tarfile.SYMTYPE + link.linkname = "python3" + archive.addfile(link) + hardlink = tarfile.TarInfo("agent_deps/bin/python-copy") + hardlink.type = tarfile.LNKTYPE + hardlink.linkname = "agent_deps/bin/python3" + archive.addfile(hardlink) + app._validate_runtime_archive(archive_path) + + with tarfile.open(archive_path, "w:gz") as archive: + link = tarfile.TarInfo("agent_deps/bin/python") + link.type = tarfile.SYMTYPE + link.linkname = "../../../outside" + archive.addfile(link) + with pytest.raises(app.LegalAgentBenchArtifactError, match="escapes"): + app._validate_runtime_archive(archive_path) + + with tarfile.open(archive_path, "w:gz") as archive: + link = tarfile.TarInfo("agent_deps/bin/python") + link.type = tarfile.LNKTYPE + link.linkname = "outside" + archive.addfile(link) + with pytest.raises(app.LegalAgentBenchArtifactError, match="escapes"): + app._validate_runtime_archive(archive_path) + + +def test_archive_helpers_reject_missing_unsafe_and_nonregular_inputs(tmp_path) -> None: + with pytest.raises(FileNotFoundError): + app._create_archive(tmp_path / "missing.tar.gz", [(tmp_path / "absent", "input")]) + + traversal = tarfile.TarInfo("../outside") + with pytest.raises(app.LegalAgentBenchArtifactError, match="Unsafe"): + app._validate_archive_member(traversal) + + device = tarfile.TarInfo("device") + device.type = tarfile.CHRTYPE + with pytest.raises(app.LegalAgentBenchArtifactError, match="regular files"): + app._validate_archive_member(device) + + with pytest.raises(app.LegalAgentBenchArtifactError, match="regular downloaded file"): + app._copy_downloaded_file(tmp_path, tmp_path / "copy") + + +def test_provider_and_installer_validation_rejects_ambiguous_or_missing_configuration(monkeypatch, tmp_path) -> None: + with pytest.raises(app.LegalAgentBenchConfigurationError, match="exactly one provider"): + app._provider_name({"docker": {}, "ecs_fargate": {}}) + + monkeypatch.setattr(app, "PARENT_DIR", tmp_path) + with pytest.raises(app.LegalAgentBenchConfigurationError, match="requires dependency setup script"): + app.resolve_agent_setup_script("responses_api_agents.hermes_agent.app") + + runner = app.LegalAgentBenchAgent.model_construct( + config=_config(sandbox_provider="sandbox"), + server_client=SimpleNamespace( + global_config_dict={"sandbox": {"docker": {}, "default_metadata": "not-a-mapping"}} + ), + ) + with pytest.raises(app.LegalAgentBenchConfigurationError, match="provider metadata"): + runner._sandbox_metadata() + + +@pytest.mark.parametrize( + "provider_name", + ["docker", "ecs_fargate", "enroot", "apptainer", "opensandbox", "daytona", "openshell"], +) +def test_every_builtin_provider_has_a_named_config(provider_name) -> None: + config_path = ( + app.PARENT_DIR / "nemo_gym" / "sandbox" / "providers" / provider_name / "configs" / f"{provider_name}.yaml" + ) + config = OmegaConf.load(config_path) + + assert provider_name in config.sandbox + assert "default_metadata" in config.sandbox + + +@pytest.mark.asyncio +async def test_provider_smoke_helper_uses_only_the_public_lifecycle_contract(monkeypatch) -> None: + instances = [] + + class LifecycleSandbox: + def __init__(self, provider, spec): + self.provider = provider + self.spec = spec + self.payload = None + self.stopped = False + instances.append(self) + + async def start(self): + return self + + async def upload(self, source, destination): + self.payload = Path(source).read_text() + assert destination.endswith("/input.txt") + + async def exec(self, command, **kwargs): + assert self.payload in command + assert "user" not in kwargs + return SimpleNamespace(return_code=0, error_type=None, stdout="", stderr="") + + async def download(self, source, destination): + assert source.endswith("/output.txt") + Path(destination).write_text(self.payload) + + async def stop(self): + self.stopped = True + + monkeypatch.setattr(smoke_provider, "AsyncSandbox", LifecycleSandbox) + result = await smoke_provider._smoke( + SimpleNamespace( + config=None, + provider="apptainer", + sandbox_name="sandbox", + image="provider-native-image", + workdir="/tmp", + timeout=30, + ready_timeout=60, + ttl=120, + ) + ) + + assert result["provider"] == "apptainer" + assert result["cleanup"] == "passed" + assert instances[0].provider == {"apptainer": {}} + assert instances[0].spec.image == "provider-native-image" + assert instances[0].stopped is True + + +def test_recipe_hash_ignores_transient_bytecode(tmp_path) -> None: + package = tmp_path / "package" + (package / "__pycache__").mkdir(parents=True) + (package / "app.py").write_text("VALUE = 1\n") + bytecode = package / "__pycache__" / "app.pyc" + bytecode.write_bytes(b"first") + first = app._recipe_hash([package]) + + bytecode.write_bytes(b"second") + + assert app._recipe_hash([package]) == first + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "provider_name", + ["docker", "ecs_fargate", "enroot", "apptainer", "opensandbox", "daytona", "openshell"], +) +async def test_runtime_provisioning_uses_selected_sandbox_provider(monkeypatch, tmp_path, provider_name) -> None: + _runtime_sources(monkeypatch, tmp_path, "hermes_agent") + _RuntimeBuilderSandbox.instances = [] + monkeypatch.setattr(app, "AsyncSandbox", _RuntimeBuilderSandbox) + monkeypatch.setattr(app.shutil, "which", lambda _name: pytest.fail("runtime provisioning used Docker CLI")) + provider = {provider_name: {}} + runner = app.LegalAgentBenchAgent.model_construct(config=_config(sandbox_provider=provider)) + runner._runtime_lock = app.asyncio.Lock() + runner._runtime_archives = {} + monkeypatch.setattr(runner, "_provider_config", lambda: provider) + monkeypatch.setattr(runner, "_sandbox_metadata", lambda: {}) + + archive = await runner._ensure_runtime("provider-native-image") + + assert archive.is_file() + assert _RuntimeBuilderSandbox.instances[0].provider == provider + expected_options = ( + {"resource_requests": {"cpu": 0.25, "memory_mib": 1024, "disk_gib": 10}} + if provider_name == "opensandbox" + else {} + ) + assert _RuntimeBuilderSandbox.instances[0].spec.provider_options == expected_options + + +@pytest.mark.asyncio +async def test_runtime_builder_merges_phase_specific_provider_options(monkeypatch, tmp_path) -> None: + _runtime_sources(monkeypatch, tmp_path, "hermes_agent") + _RuntimeBuilderSandbox.instances = [] + monkeypatch.setattr(app, "AsyncSandbox", _RuntimeBuilderSandbox) + provider = {"openshell": {}} + runner = app.LegalAgentBenchAgent.model_construct( + config=_config( + sandbox_provider=provider, + runtime_builder_provider_options={"policy": {"version": 1}}, + ) + ) + runner._runtime_lock = app.asyncio.Lock() + runner._runtime_archives = {} + monkeypatch.setattr(runner, "_provider_config", lambda: provider) + monkeypatch.setattr(runner, "_sandbox_metadata", lambda: {}) + + await runner._ensure_runtime("provider-native-image") + + assert _RuntimeBuilderSandbox.instances[0].spec.provider_options == {"policy": {"version": 1}} + + +def test_config_defaults_are_docker_and_single_concurrency() -> None: + config = _config() + assert config.concurrency == 1 + assert config.sandbox_provider == {"docker": {}} + assert config.sandbox_image is None + assert config.opensandbox_request_fraction == 0.25 + assert config.docker_network == "host" + assert config.results_dir == "results/legal_agent_bench" + assert config.agent_timeout_seconds == 10800 + assert config.model_connect_timeout_seconds == 10 + assert config.verifier_timeout_seconds == 3600 + assert config.runtime_build_timeout_seconds == 3600 + assert config.sandbox_staging_timeout_seconds == 900 + + +@pytest.mark.parametrize( + ("module", "job_dir"), + [ + ("responses_api_agents.legal_agent_bench_native_agent.app", "native_jobs"), + ("responses_api_agents.hermes_agent.app", "hermes_jobs"), + ("responses_api_agents.claude_code_agent.app", "claude_code_jobs"), + ("responses_api_agents.codex_agent.app", "codex_jobs"), + ], +) +def test_results_session_dir_is_harness_date_and_model_browsable(tmp_path, module, job_dir) -> None: + session = app._results_session_dir( + tmp_path, + agent_server_module=module, + model_name="nvidia/model name", + timestamp=1785260796, + session_id="0b6511a6", + ) + + timestamp = app.time.strftime("%Y%m%d-%H%M%S", app.time.localtime(1785260796)) + assert session == tmp_path / job_dir / "nvidia-model-name" / f"{timestamp}_0b6511a6" + + +def test_results_segment_rejects_path_syntax() -> None: + assert app._results_segment("../../model/name", fallback="unknown") == "model-name" + assert app._results_segment("...", fallback="unknown") == "unknown" + + +def test_agent_construction_does_not_create_empty_results_session(tmp_path) -> None: + runner = app.LegalAgentBenchAgent.model_construct( + config=_config(results_dir=str(tmp_path)), + server_client=SimpleNamespace(global_config_dict=OmegaConf.create({"policy_model_name": "org/model"})), + ) + + assert runner._session_results_dir.parent == tmp_path / "hermes_jobs" / "org-model" + assert not runner._session_results_dir.exists() + + paths = runner._run_dirs("area__task") + + assert paths["root"].is_dir() + assert runner._session_results_dir.is_dir() + + +def test_model_name_reads_omegaconf_global_config() -> None: + runner = app.LegalAgentBenchAgent.model_construct(config=_config()) + runner.server_client = SimpleNamespace( + global_config_dict=OmegaConf.create({"policy_model_name": "nvidia/nemotron-3-ultra"}) + ) + + assert runner._model_name() == "nvidia/nemotron-3-ultra" + + +def test_resolve_task_dir_accepts_known_task(tmp_path) -> None: + root, task = _task_tree(tmp_path) + assert app.resolve_task_dir(root, "legal_agent_bench::area__task") == task.resolve() + + +@pytest.mark.parametrize( + "instance_id", + [ + "wrong::area__task", + "legal_agent_bench::../task", + "legal_agent_bench::nested/task", + "legal_agent_bench::", + "area__task", + ], +) +def test_resolve_task_dir_rejects_unsafe_or_invalid_ids(tmp_path, instance_id) -> None: + root, _task = _task_tree(tmp_path) + with pytest.raises((ValueError, FileNotFoundError)): + app.resolve_task_dir(root, instance_id) + + +def test_resolve_task_dir_rejects_incomplete_task(tmp_path) -> None: + root = tmp_path / "tasks" + (root / "broken").mkdir(parents=True) + with pytest.raises(app.LegalAgentBenchTaskError, match="incomplete"): + app.resolve_task_dir(root, "legal_agent_bench::broken") + + +def test_compose_agent_input_uses_task_and_skills_without_rubric(monkeypatch, tmp_path) -> None: + _root, task = _task_tree(tmp_path) + skills = _skills(tmp_path) + monkeypatch.setattr(app, "validate_harness_skills", lambda path: Path(path)) + params = NeMoGymResponseCreateParamsNonStreaming(input=[], temperature=0.5) + + result = app.compose_agent_input(task, skills, params) + + assert result.temperature == 0.5 + assert len(result.input) == 2 + serialized = result.model_dump(mode="json", warnings="error") + system = serialized["input"][0]["content"] + assert "Write a memo." in system + assert "Skill: docx" in system + assert "$VDR_DIR" in system + assert "criteria" not in system + assert serialized["input"][1]["content"] == app.INITIAL_USER_PROMPT + + +def test_native_agent_input_uses_upstream_prompt_and_canonical_tools(monkeypatch, tmp_path) -> None: + _root, task = _task_tree(tmp_path) + skills = _skills(tmp_path) + monkeypatch.setattr(app, "validate_harness_skills", lambda path: Path(path)) + + result = app.compose_agent_input( + task, + skills, + NeMoGymResponseCreateParamsNonStreaming(input=[]), + native=True, + ) + + serialized = result.model_dump(mode="json", warnings="error") + system = serialized["input"][0]["content"] + assert system.startswith(app.LAB_SYSTEM_PROMPT) + assert "Write a memo." in system + assert '"criteria"' not in system + assert '"id": "1"' not in system + assert [tool["name"] for tool in result.tools] == ["bash", "read", "write", "write_docx", "edit", "glob", "grep"] + assert all(tool["type"] == "function" and tool["strict"] is False for tool in result.tools) + assert result.parallel_tool_calls is False + + +def test_verifier_credentials_are_read_separately_from_sandbox_resources(tmp_path) -> None: + _root, task = _task_tree(tmp_path) + assert app._verifier_env(task) == { + "LAB_JUDGE_API_KEY": "secret", # pragma: allowlist secret + "LAB_JUDGE_MODEL": "judge", + } + resources = app._sandbox_resources(task) + assert resources.cpu == 2 + assert resources.memory_mib == 4096 + assert resources.disk_gib == 10 + assert app._fractional_resource_requests(resources, 0.25) == { + "cpu": 0.5, + "memory_mib": 1024, + "disk_gib": 10, + } + + gpu_resources = app.SandboxResources(cpu=4, memory_mib=8192, disk_gib=30, gpu=2, gpu_type="H100") + assert app._fractional_resource_requests(gpu_resources, 0.25) == { + "cpu": 1.0, + "memory_mib": 2048, + "disk_gib": 30, + "gpu": 2, + "gpu_type": "H100", + } + + +def test_environment_hash_is_deterministic_and_content_sensitive(tmp_path) -> None: + _root, task = _task_tree(tmp_path) + environment = task / "environment" + first = app._environment_hash(environment) + assert first == app._environment_hash(environment) + (environment / "Dockerfile").write_text("FROM python:3.12-slim\nRUN true\n") + assert app._environment_hash(environment) != first + + +def test_empty_response_is_schema_valid() -> None: + response = app._empty_response("policy") + assert response.model == "policy" + assert response.output == [] + + +def test_runner_config_preserves_dynamic_agent_configuration(tmp_path) -> None: + runner = app.LegalAgentBenchAgent.model_construct( + config=_config(agent_kwargs={"max_turns": 17, "terminal_backend": "local"}) + ) + paths = {"runtime": tmp_path} + params = NeMoGymResponseCreateParamsNonStreaming(input="Do the work") + + runner._write_runner_config(paths, params, "http://model.internal:8000") + + payload = json.loads((tmp_path / "runner.json").read_text()) + assert payload["agent_server_module"] == "responses_api_agents.hermes_agent.app" + assert payload["agent_server_class"] == "HermesAgent" + assert payload["agent_config_class"] == "HermesAgentConfig" + assert payload["agent_kwargs"]["max_turns"] == 17 + assert payload["model_name"] == "policy_model" + assert payload["model_url"] == "http://model.internal:8000" + assert payload["model_connect_timeout_seconds"] == 10 + assert payload["http_proxy_from_environment"] is False + assert payload["disable_endpoint_metadata_probe"] is True + assert "LAB_POLICY_API_KEY" not in json.dumps(payload) + assert "inspect.signature(agent.responses)" in (tmp_path / "agent_runner.py").read_text() + assert "SimpleNamespace(path_params={})" in (tmp_path / "agent_runner.py").read_text() + runner_source = (tmp_path / "agent_runner.py").read_text() + assert "urlopen(" in runner_source + assert "socket.create_connection" not in runner_source + assert "if exc.code in {401, 403}" in runner_source + assert 'os.environ.get("LAB_POLICY_MODEL_URL", runner["model_url"])' in (tmp_path / "agent_runner.py").read_text() + assert 'model_url_root = model_url.removesuffix("/v1").rstrip("/")' in runner_source + assert "client._build_server_base_url = lambda _cfg: model_url_root" in runner_source + assert 'object.__setattr__(agent, "_resolve_base_url", lambda *args, **kwargs: model_url_root)' in runner_source + assert "runner_status.json" in (tmp_path / "agent_runner.py").read_text() + assert '"responses_api_models"' in (tmp_path / "agent_runner.py").read_text() + assert '"global_aiohttp_trust_env": runner.get("http_proxy_from_environment", False)' in runner_source + assert "if not is_global_aiohttp_client_setup():" in runner_source + assert "response = asyncio.run(invoke_agent())" in runner_source + assert 'os.environ.setdefault("NEMO_GYM_CONFIG_DICT", "{}")' in (tmp_path / "agent_runner.py").read_text() + assert "hermes_model_metadata.fetch_endpoint_model_metadata" in (tmp_path / "agent_runner.py").read_text() + assert "hermes_model_metadata._query_local_context_length" in (tmp_path / "agent_runner.py").read_text() + assert "hermes_usage_pricing.fetch_endpoint_model_metadata" in (tmp_path / "agent_runner.py").read_text() + assert 'os.environ.get("LAB_POLICY_API_KEY")' in runner_source + assert 'headers.setdefault("Authorization", f"Bearer {model_api_key}")' in runner_source + assert 'object.__setattr__(config, "anthropic_api_key", model_api_key)' in runner_source + assert 'object.__setattr__(config, "openai_api_key", model_api_key)' in runner_source + assert 'kwargs["model"] = runner["model_name"]' in runner_source + assert 'body = body.model_copy(update={"model": runner["model_name"]})' in runner_source + assert "class AuthenticatedAIAgent(original_ai_agent):" in runner_source + assert "hermes_run_agent.AIAgent = AuthenticatedAIAgent" in runner_source + + +def test_runner_enables_environment_proxy_only_for_openshell(monkeypatch, tmp_path) -> None: + runner = app.LegalAgentBenchAgent.model_construct(config=_config()) + monkeypatch.setattr(runner, "_provider_config", lambda: {"openshell": {}}) + + runner._write_runner_config( + {"runtime": tmp_path}, + NeMoGymResponseCreateParamsNonStreaming(input="Do the work"), + "http://model.internal:8000", + ) + + payload = json.loads((tmp_path / "runner.json").read_text()) + assert payload["http_proxy_from_environment"] is True + + +def test_runner_config_keeps_endpoint_metadata_for_non_hermes_harnesses(tmp_path) -> None: + runner = app.LegalAgentBenchAgent.model_construct( + config=_config( + agent_server_module="responses_api_agents.codex_agent.app", + agent_server_class="CodexAgent", + agent_config_class="CodexAgentConfig", + ) + ) + paths = {"runtime": tmp_path} + params = NeMoGymResponseCreateParamsNonStreaming(input="Do the work") + + runner._write_runner_config(paths, params, "http://model.internal:8000") + + payload = json.loads((tmp_path / "runner.json").read_text()) + assert payload["disable_endpoint_metadata_probe"] is False + + +def test_runner_config_stages_default_codex_model_catalog(tmp_path) -> None: + runner = app.LegalAgentBenchAgent.model_construct( + config=_config( + agent_server_module="responses_api_agents.codex_agent.app", + agent_server_class="CodexAgent", + agent_config_class="CodexAgentConfig", + ) + ) + + runner._write_runner_config( + {"runtime": tmp_path}, + NeMoGymResponseCreateParamsNonStreaming(input="Do the work"), + "http://model.internal:8000", + ) + + payload = json.loads((tmp_path / "runner.json").read_text()) + assert payload["agent_kwargs"]["extra_config"]["model_catalog_json"] == app.CODEX_MODEL_CATALOG_PATH + catalog = json.loads((tmp_path / "codex_model_catalog.json").read_text()) + assert [model["slug"] for model in catalog["models"]] == ["gym-policy-model", "policy_model"] + assert catalog["models"][1]["display_name"] == "policy_model" + assert catalog["models"][0]["base_instructions"] == "" + + +def test_runner_codex_model_catalog_uses_configured_direct_model_name(tmp_path) -> None: + runner = app.LegalAgentBenchAgent.model_construct( + config=_config( + agent_server_module="responses_api_agents.codex_agent.app", + agent_server_class="CodexAgent", + agent_config_class="CodexAgentConfig", + ), + server_client=SimpleNamespace(global_config_dict={"policy_model_name": "vendor/custom-model"}), + ) + + runner._write_runner_config( + {"runtime": tmp_path}, + NeMoGymResponseCreateParamsNonStreaming(input="Do the work"), + "http://model.internal:8000", + ) + + payload = json.loads((tmp_path / "runner.json").read_text()) + catalog = json.loads((tmp_path / "codex_model_catalog.json").read_text()) + assert payload["model_name"] == "vendor/custom-model" + assert [model["slug"] for model in catalog["models"]] == ["gym-policy-model", "vendor/custom-model"] + + +def test_runner_config_preserves_explicit_codex_model_catalog(tmp_path) -> None: + runner = app.LegalAgentBenchAgent.model_construct( + config=_config( + agent_server_module="responses_api_agents.codex_agent.app", + agent_server_class="CodexAgent", + agent_config_class="CodexAgentConfig", + agent_kwargs={"extra_config": {"model_catalog_json": "/custom/catalog.json"}}, + ) + ) + + runner._write_runner_config( + {"runtime": tmp_path}, + NeMoGymResponseCreateParamsNonStreaming(input="Do the work"), + "http://model.internal:8000", + ) + + payload = json.loads((tmp_path / "runner.json").read_text()) + assert payload["agent_kwargs"]["extra_config"]["model_catalog_json"] == "/custom/catalog.json" + assert not (tmp_path / "codex_model_catalog.json").exists() + + +def test_stage_agent_source_copies_only_selected_runtime_package(monkeypatch, tmp_path) -> None: + repository = tmp_path / "repository" + package = repository / "responses_api_agents" / "hermes_agent" + (package / "tests").mkdir(parents=True) + (package / "data").mkdir() + (package / "runtime_helpers").mkdir() + (package / "app.py").write_text("VALUE = 1\n") + (package / "runtime_helpers" / "tool.py").write_text("VALUE = 2\n") + (package / "tests" / "rubric.py").write_text("SECRET = True\n") # pragma: allowlist secret + (package / "data" / "example.jsonl").write_text("{}\n") + (repository / "resources_servers" / "legal_agent_bench" / "data" / "runtime").mkdir(parents=True) + (repository / "resources_servers" / "legal_agent_bench" / "data" / "runtime" / "rubric.json").write_text("{}") + paths = {"agent_source": tmp_path / "staged"} + paths["agent_source"].mkdir() + runner = app.LegalAgentBenchAgent.model_construct(config=_config()) + monkeypatch.setattr(app, "PARENT_DIR", repository) + + runner._stage_agent_source(paths) + + staged = paths["agent_source"] / "responses_api_agents" / "hermes_agent" + assert (staged / "app.py").is_file() + assert (staged / "runtime_helpers" / "tool.py").is_file() + assert not (staged / "tests").exists() + assert not (staged / "data").exists() + assert not (paths["agent_source"] / "resources_servers").exists() + + +def test_model_url_uses_override_without_server_lookup() -> None: + runner = app.LegalAgentBenchAgent.model_construct( + config=_config(sandbox_model_base_url="http://host.docker.internal:9000") + ) + runner.server_client = SimpleNamespace(global_config_dict={}) + body = app.LegalAgentBenchRunRequest( + instance_id="legal_agent_bench::area__task", + responses_create_params=NeMoGymResponseCreateParamsNonStreaming(input=[]), + ) + + assert runner._model_url(body) == "http://host.docker.internal:9000" + + +def test_ecs_model_url_uses_host_policy_proxy_for_reverse_tunnel(monkeypatch) -> None: + runner = app.LegalAgentBenchAgent.model_construct( + config=_config( + sandbox_provider={ + "ecs_fargate": { + "region": "us-east-1", + "cluster": "test", + } + } + ), + server_client=SimpleNamespace( + global_config_dict={}, + _build_server_base_url=lambda _config: "http://0.0.0.0:16300", + ), + ) + monkeypatch.setattr(app, "get_first_server_config_dict", lambda *_args: {}) + body = app.LegalAgentBenchRunRequest( + instance_id="legal_agent_bench::area__task", + responses_create_params=NeMoGymResponseCreateParamsNonStreaming(input=[]), + ) + + assert runner._model_url(body) == "http://127.0.0.1:16300" + + +@pytest.mark.parametrize("provider_name", ["enroot", "apptainer"]) +def test_local_non_docker_providers_use_the_host_policy_proxy(monkeypatch, provider_name) -> None: + provider = {provider_name: {}} + runner = app.LegalAgentBenchAgent.model_construct( + config=_config(sandbox_provider=provider), + server_client=SimpleNamespace( + global_config_dict={}, + _build_server_base_url=lambda _config: "http://0.0.0.0:16300", + ), + ) + monkeypatch.setattr(app, "get_first_server_config_dict", lambda *_args: {}) + monkeypatch.setattr(app.LegalAgentBenchAgent, "rollout_id_from_run", lambda _self, _body: None) + body = app.LegalAgentBenchRunRequest( + instance_id="legal_agent_bench::area__task", + responses_create_params=NeMoGymResponseCreateParamsNonStreaming(input=[]), + ) + + assert runner._model_url(body) == "http://127.0.0.1:16300" + + +@pytest.mark.parametrize("provider_name", ["opensandbox", "daytona", "openshell"]) +def test_remote_providers_require_a_reachable_policy_proxy_for_host_local_models(monkeypatch, provider_name) -> None: + provider = {provider_name: {}} + runner = app.LegalAgentBenchAgent.model_construct( + config=_config(sandbox_provider=provider), + server_client=SimpleNamespace( + global_config_dict={}, + _build_server_base_url=lambda _config: "http://127.0.0.1:16300", + ), + ) + monkeypatch.setattr(app, "get_first_server_config_dict", lambda *_args: {}) + monkeypatch.setattr(app.LegalAgentBenchAgent, "rollout_id_from_run", lambda _self, _body: None) + body = app.LegalAgentBenchRunRequest( + instance_id="legal_agent_bench::area__task", + responses_create_params=NeMoGymResponseCreateParamsNonStreaming(input=[]), + ) + + with pytest.raises(app.LegalAgentBenchConfigurationError, match="sandbox_model_base_url"): + runner._model_url(body) + runner.config.sandbox_model_base_url = "https://policy-proxy.example/v1" + assert runner._model_url(body) == "https://policy-proxy.example/v1" + + +@pytest.mark.parametrize( + ("url", "network", "platform_name", "expected"), + [ + ( + "http://127.0.0.1:16300/rollout/run-1", + "host", + "darwin", + "http://host.docker.internal:16300/rollout/run-1", + ), + ("http://localhost:16300", "host", "win32", "http://host.docker.internal:16300"), + ("http://0.0.0.0:16300", None, "linux", "http://host.docker.internal:16300"), + ("http://127.0.0.1:16300", "host", "linux", "http://127.0.0.1:16300"), + ("http://model.internal:16300", "host", "darwin", "http://model.internal:16300"), + ], +) +def test_sandbox_model_url_routes_loopback_for_docker(url, network, platform_name, expected) -> None: + assert app.sandbox_model_url(url, docker_network=network, platform_name=platform_name) == expected + + +def test_linux_bridge_provider_registers_host_gateway(monkeypatch) -> None: + runner = app.LegalAgentBenchAgent.model_construct(config=_config(docker_network=None)) + monkeypatch.setattr(app.sys, "platform", "linux") + + provider = runner._provider_config() + + assert provider["docker"]["create"]["extra_run_args"] == [ + "--add-host", + "host.docker.internal:host-gateway", + ] + + +def test_hermes_synthetic_connection_error_is_an_agent_failure() -> None: + response = app.NeMoGymResponse.model_validate( + { + **_successful_response().model_dump(mode="json"), + "usage": { + "input_tokens": 0, + "input_tokens_details": {"cached_tokens": 0}, + "output_tokens": 0, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 0, + }, + "output": [ + { + "id": "msg-error", + "content": [{"annotations": [], "text": "Connection error.", "type": "output_text"}], + "role": "assistant", + "status": "completed", + "type": "message", + "prompt_token_ids": [0], + "generation_token_ids": [0], + "generation_log_probs": [0.0], + } + ], + } + ) + + failure = app.agent_response_failure(response, "responses_api_agents.hermes_agent.app") + + assert failure == "Hermes produced no model trajectory: Connection error." + + +def test_real_agent_activity_is_not_treated_as_an_infrastructure_failure() -> None: + assert ( + app.agent_response_failure( + _successful_response(), + "responses_api_agents.hermes_agent.app", + ) + is None + ) + + +def test_partial_response_with_harness_error_is_an_agent_failure() -> None: + response = app.NeMoGymResponse.model_validate( + { + **_successful_response().model_dump(mode="json"), + "status": "failed", + "error": {"code": "server_error", "message": "adapter failed after partial output"}, + } + ) + + failure = app.agent_response_failure(response, "responses_api_agents.codex_agent.app") + + assert failure is not None + assert "adapter failed after partial output" in failure + + +def test_native_timeout_failure_metadata_propagates_timeout_flag() -> None: + response = app.NeMoGymResponse.model_validate( + { + **_successful_response().model_dump(mode="json"), + "status": "failed", + "error": {"code": "server_error", "message": "LAB model call timed out after 1800s"}, + "metadata": {app.AGENT_FAILURE_CLASS_METADATA_KEY: "agent_timed_out"}, + } + ) + + assert app.agent_response_failure_flags(response, app.NATIVE_AGENT_MODULE) == (False, True) + assert app.agent_response_failure_flags(response, "responses_api_agents.codex_agent.app") == (False, False) + + +def test_native_model_connection_failure_metadata_propagates_connection_flag() -> None: + response = app.NeMoGymResponse.model_validate( + { + **_successful_response().model_dump(mode="json"), + "status": "failed", + "error": {"code": "server_error", "message": "LAB model call failed: HTTP 500"}, + "metadata": {app.AGENT_FAILURE_CLASS_METADATA_KEY: "model_connection_failed"}, + } + ) + + assert app.agent_response_failure_flags(response, app.NATIVE_AGENT_MODULE) == (True, False) + assert app.agent_response_failure_flags(response, "responses_api_agents.codex_agent.app") == (False, False) + + +def test_response_masks_harness_and_verifier_failures() -> None: + runner = app.LegalAgentBenchAgent.model_construct(config=_config()) + params = NeMoGymResponseCreateParamsNonStreaming(input=[]) + body = app.LegalAgentBenchRunRequest( + instance_id="legal_agent_bench::area__task", + responses_create_params=params, + ) + + response = runner._response( + body=body, + params=params, + response=app._empty_response("policy"), + reward_data={}, + paths=None, + agent_failed=True, + model_connection_failed=True, + verifier_failed=True, + ) + + assert response.agent_failed is True + assert response.model_connection_failed is True + assert response.verifier_failed is True + assert response.mask_sample is True + assert response.model_dump()["_ng_failure_class"] == "model_connection_failed" + assert "_ng_failure_terminal" not in response.model_dump() + + +@pytest.mark.parametrize( + ("reward_data", "expected_reason"), + [ + ({"judge_error_count": 1}, "Verifier reported 1 judge error"), + ({"judge_error_count": 3}, "Verifier reported 3 judge errors"), + ({"verifier_error": 1}, "Verifier reported an internal error"), + ], +) +def test_response_synthesizes_failure_reason_for_unreliable_verifier_metrics(reward_data, expected_reason) -> None: + runner = app.LegalAgentBenchAgent.model_construct(config=_config()) + params = NeMoGymResponseCreateParamsNonStreaming(input=[]) + body = app.LegalAgentBenchRunRequest( + instance_id="legal_agent_bench::area__task", + responses_create_params=params, + ) + + response = runner._response( + body=body, + params=params, + response=app._empty_response("policy"), + reward_data=reward_data, + paths=None, + ) + + assert response.failure_reason == expected_reason + assert response.verifier_failed is True + assert response.mask_sample is True + assert response.model_dump()["_ng_failure_class"] == "verifier_failed" + + +@pytest.mark.parametrize( + ("failure_flag", "expected_class", "terminal"), + [ + ("agent_failed", "agent_failed", False), + ("model_connection_failed", "model_connection_failed", False), + ("agent_timed_out", "agent_timed_out", False), + ("verifier_failed", "verifier_failed", False), + ("verifier_timed_out", "verifier_failed", False), + ("sandbox_failed", "sandbox_failed", False), + ("task_failed", "task_failed", True), + ("configuration_failed", "configuration_failed", True), + ], +) +def test_response_routes_every_failure_class(failure_flag, expected_class, terminal) -> None: + runner = app.LegalAgentBenchAgent.model_construct(config=_config()) + params = NeMoGymResponseCreateParamsNonStreaming(input=[]) + body = app.LegalAgentBenchRunRequest( + instance_id="legal_agent_bench::area__task", + responses_create_params=params, + ) + + response = runner._response( + body=body, + params=params, + response=app._empty_response("policy"), + reward_data={}, + paths=None, + **{failure_flag: True}, + ) + response_data = response.model_dump() + + assert response.mask_sample is True + assert response_data["_ng_failure_class"] == expected_class + assert response_data.get("_ng_failure_terminal", False) is terminal + + +@pytest.mark.parametrize( + "reward_data", + [ + {"reward": float("nan")}, + {"criteria_pass_rate": 1.1}, + {"judge_error_count": "not-a-number"}, + {"verifier_error": -1}, + ], +) +def test_response_masks_malformed_verifier_metrics_instead_of_raising(reward_data) -> None: + runner = app.LegalAgentBenchAgent.model_construct(config=_config()) + params = NeMoGymResponseCreateParamsNonStreaming(input=[]) + body = app.LegalAgentBenchRunRequest( + instance_id="legal_agent_bench::area__task", + responses_create_params=params, + ) + + response = runner._response( + body=body, + params=params, + response=_successful_response(), + reward_data=reward_data, + paths=None, + ) + + assert response.reward == 0.0 + assert response.verifier_failed is True + assert response.mask_sample is True + assert response.failure_reason.startswith("Invalid verifier") + assert response.model_dump()["_ng_failure_class"] == "verifier_failed" + assert "_ng_failure_terminal" not in response.model_dump() + + +@pytest.mark.asyncio +async def test_run_classifies_invalid_task_without_sandbox_failure(monkeypatch) -> None: + runner = app.LegalAgentBenchAgent.model_construct(config=_config()) + runner._sem = app.asyncio.Semaphore(1) + monkeypatch.setattr(runner, "_model_name", lambda: "policy") + body = app.LegalAgentBenchRunRequest( + instance_id="legal_agent_bench::../unsafe", + responses_create_params=NeMoGymResponseCreateParamsNonStreaming(input=[]), + ) + + response = await runner.run(None, body) + + assert response.task_failed is True + assert response.configuration_failed is False + assert response.model_dump()["_ng_failure_class"] == "task_failed" + assert response.model_dump()["_ng_failure_terminal"] is True + assert response.sandbox_failed is False + assert response.mask_sample is True + assert "Unsafe Legal Agent Bench task name" in response.failure_reason + + +@pytest.mark.asyncio +async def test_run_classifies_bad_agent_configuration_without_sandbox_failure(monkeypatch, tmp_path) -> None: + _root, task = _task_tree(tmp_path) + skills = _skills(tmp_path) + runner = app.LegalAgentBenchAgent.model_construct(config=_config()) + runner._sem = app.asyncio.Semaphore(1) + runner._session_results_dir = tmp_path / "results" + + async def ensure_image(_task): + return "lab:image" + + async def reject_runtime(_image): + raise app.LegalAgentBenchConfigurationError("missing configured CLI pin") + + monkeypatch.setattr(app, "resolve_task_dir", lambda runtime, instance: task) + monkeypatch.setattr(app, "resolve_repo_path", lambda path: skills) + monkeypatch.setattr(app, "compose_agent_input", lambda task_dir, skills_dir, params: params) + monkeypatch.setattr(runner, "_model_name", lambda: "policy") + monkeypatch.setattr(runner, "_ensure_image", ensure_image) + monkeypatch.setattr(runner, "_ensure_runtime", reject_runtime) + params = NeMoGymResponseCreateParamsNonStreaming(input=[]) + body = app.LegalAgentBenchRunRequest( + instance_id="legal_agent_bench::area__task", + responses_create_params=params, + ) + + response = await runner.run(None, body) + + assert response.configuration_failed is True + assert response.task_failed is False + assert response.sandbox_failed is False + assert response.mask_sample is True + assert response.model_dump()["_ng_failure_class"] == "configuration_failed" + assert response.model_dump()["_ng_failure_terminal"] is True + assert "missing configured CLI pin" in response.failure_reason + + +@pytest.mark.asyncio +async def test_run_classifies_bad_sandbox_reference_as_terminal_configuration_failure(monkeypatch, tmp_path) -> None: + _root, task = _task_tree(tmp_path) + skills = _skills(tmp_path) + runner = app.LegalAgentBenchAgent.model_construct( + config=_config(sandbox_provider="missing_sandbox"), + server_client=SimpleNamespace(global_config_dict={}), + ) + runner._sem = app.asyncio.Semaphore(1) + runner._session_results_dir = tmp_path / "results" + + monkeypatch.setattr(app, "resolve_task_dir", lambda runtime, instance: task) + monkeypatch.setattr(app, "resolve_repo_path", lambda path: skills) + monkeypatch.setattr(app, "compose_agent_input", lambda task_dir, skills_dir, params: params) + monkeypatch.setattr(runner, "_model_name", lambda: "policy") + params = NeMoGymResponseCreateParamsNonStreaming(input=[]) + body = app.LegalAgentBenchRunRequest( + instance_id="legal_agent_bench::area__task", + responses_create_params=params, + ) + + response = await runner.run(None, body) + + assert response.configuration_failed is True + assert response.sandbox_failed is False + assert response.mask_sample is True + assert response.model_dump()["_ng_failure_class"] == "configuration_failed" + assert response.model_dump()["_ng_failure_terminal"] is True + assert "missing_sandbox" in response.failure_reason + + +@pytest.mark.asyncio +async def test_concurrent_image_requests_build_once(monkeypatch, tmp_path) -> None: + _root, task = _task_tree(tmp_path) + runner = app.LegalAgentBenchAgent.model_construct(config=_config()) + runner._image_lock = app.asyncio.Lock() + calls = [] + + async def fake_run(args, *, cwd, timeout): + calls.append(args) + if args[1:3] == ["image", "inspect"]: + inspect_count = sum(call[1:3] == ["image", "inspect"] for call in calls) + return (1 if inspect_count == 1 else 0), "", "" + return 0, "", "" + + monkeypatch.setattr(app.shutil, "which", lambda name: "/usr/bin/docker") + monkeypatch.setattr(app, "_run_process", fake_run) + + first, second = await app.asyncio.gather(runner._ensure_image(task), runner._ensure_image(task)) + + assert first == second + assert sum(call[1] == "build" for call in calls) == 1 + + +@pytest.mark.asyncio +async def test_two_run_requests_overlap_when_server_concurrency_is_two(monkeypatch, tmp_path) -> None: + _root, task = _task_tree(tmp_path) + skills = _skills(tmp_path) + runner = app.LegalAgentBenchAgent.model_construct(config=_config(concurrency=2)) + runner._sem = app.asyncio.Semaphore(2) + runner._session_results_dir = tmp_path / "results" + active = 0 + maximum_active = 0 + both_started = app.asyncio.Event() + + async def ensure_image(_task): + nonlocal active, maximum_active + active += 1 + maximum_active = max(maximum_active, active) + if active == 2: + both_started.set() + await app.asyncio.wait_for(both_started.wait(), timeout=1) + active -= 1 + return "lab:image" + + async def stop_after_overlap(_image): + raise app.LegalAgentBenchConfigurationError("stop after overlap assertion") + + monkeypatch.setattr(app, "resolve_task_dir", lambda runtime, instance: task) + monkeypatch.setattr(app, "resolve_repo_path", lambda path: skills) + monkeypatch.setattr(app, "compose_agent_input", lambda task_dir, skills_dir, params: params) + monkeypatch.setattr(runner, "_model_name", lambda: "policy") + monkeypatch.setattr(runner, "_ensure_image", ensure_image) + monkeypatch.setattr(runner, "_ensure_runtime", stop_after_overlap) + body = app.LegalAgentBenchRunRequest( + instance_id="legal_agent_bench::area__task", + responses_create_params=NeMoGymResponseCreateParamsNonStreaming(input=[]), + ) + + first, second = await app.asyncio.gather(runner.run(None, body), runner.run(None, body)) + + assert maximum_active == 2 + assert first.configuration_failed is True + assert second.configuration_failed is True + + +def test_agent_sandbox_has_no_host_mounts(monkeypatch, tmp_path) -> None: + _root, task = _task_tree(tmp_path) + captured = {} + + def fake_sandbox(provider, spec): + captured["provider"] = provider + captured["spec"] = spec + return SimpleNamespace() + + runner = app.LegalAgentBenchAgent.model_construct(config=_config()) + monkeypatch.setattr(runner, "_provider_config", lambda: {"docker": {}}) + monkeypatch.setattr(runner, "_sandbox_metadata", lambda: {}) + monkeypatch.setattr(app, "AsyncSandbox", fake_sandbox) + + runner._agent_sandbox( + image="lab:image", + task_dir=task, + model_url="http://model", + ) + + assert captured["provider"] == {"docker": {}} + assert captured["spec"].workdir == "/tmp" + assert captured["spec"].env == app.LAB_SANDBOX_ENV + assert captured["spec"].provider_options == {} + + +def test_direct_policy_key_is_injected_only_into_agent_sandbox(monkeypatch, tmp_path) -> None: + _root, task = _task_tree(tmp_path) + captured = [] + + def fake_sandbox(_provider, spec): + captured.append(spec) + return SimpleNamespace() + + monkeypatch.setenv("LAB_DIRECT_POLICY_KEY", "test-policy-key") # pragma: allowlist secret + runner = app.LegalAgentBenchAgent.model_construct( + config=_config( + sandbox_provider={"opensandbox": {}}, + sandbox_model_base_url="https://model.example/v1", + sandbox_model_api_key_env="LAB_DIRECT_POLICY_KEY", # pragma: allowlist secret + ) + ) + monkeypatch.setattr(runner, "_sandbox_metadata", lambda: {}) + monkeypatch.setattr(app, "AsyncSandbox", fake_sandbox) + + runner._agent_sandbox( + image="registry.example/lab@sha256:" + "a" * 64, + task_dir=task, + model_url="https://model.example/v1", + ) + runner._verifier_sandbox( + image="registry.example/lab@sha256:" + "a" * 64, + task_dir=task, + ) + + assert captured[0].env == { + **app.LAB_SANDBOX_ENV, + "LAB_POLICY_API_KEY": "test-policy-key", # pragma: allowlist secret + } + assert captured[1].env == app.LAB_SANDBOX_ENV + assert "LAB_POLICY_API_KEY" not in captured[1].env + + +def test_direct_policy_key_must_be_present(monkeypatch, tmp_path) -> None: + _root, task = _task_tree(tmp_path) + monkeypatch.delenv("LAB_DIRECT_POLICY_KEY", raising=False) + runner = app.LegalAgentBenchAgent.model_construct( + config=_config(sandbox_model_api_key_env="LAB_DIRECT_POLICY_KEY") + ) + monkeypatch.setattr(runner, "_provider_config", lambda: {"opensandbox": {}}) + + with pytest.raises(app.LegalAgentBenchConfigurationError, match="unset or empty"): + runner._agent_sandbox( + image="registry.example/lab@sha256:" + "a" * 64, task_dir=task, model_url="https://model" + ) + + +def test_ecs_agent_sandbox_tunnels_derived_policy_model_url(monkeypatch, tmp_path) -> None: + _root, task = _task_tree(tmp_path) + captured = {} + + def fake_sandbox(provider, spec): + captured["provider"] = provider + captured["spec"] = spec + return SimpleNamespace() + + provider = {"ecs_fargate": {"region": "us-east-1", "cluster": "test"}} + runner = app.LegalAgentBenchAgent.model_construct(config=_config(sandbox_provider=provider)) + monkeypatch.setattr(runner, "_sandbox_metadata", lambda: {}) + monkeypatch.setattr(app, "AsyncSandbox", fake_sandbox) + + runner._agent_sandbox( + image="registry.example/lab@sha256:" + "a" * 64, + task_dir=task, + model_url="http://127.0.0.1:16300/rollout/run-1", + ) + + assert captured["provider"] == provider + assert captured["spec"].provider_options == { + "outside_endpoints": [ + { + "url": "http://127.0.0.1:16300/rollout/run-1", + "env_var": "LAB_POLICY_MODEL_URL", + } + ] + } + + +def test_ecs_agent_sandbox_uses_explicit_reachable_model_url_without_tunnel(monkeypatch, tmp_path) -> None: + _root, task = _task_tree(tmp_path) + captured = {} + + def fake_sandbox(provider, spec): + captured["spec"] = spec + return SimpleNamespace() + + provider = {"ecs_fargate": {"region": "us-east-1", "cluster": "test"}} + runner = app.LegalAgentBenchAgent.model_construct( + config=_config( + sandbox_provider=provider, + sandbox_model_base_url="https://model.example/v1", + ) + ) + monkeypatch.setattr(runner, "_sandbox_metadata", lambda: {}) + monkeypatch.setattr(app, "AsyncSandbox", fake_sandbox) + + runner._agent_sandbox( + image="registry.example/lab@sha256:" + "a" * 64, + task_dir=task, + model_url="https://model.example/v1", + ) + + assert captured["spec"].provider_options == {} + + +def test_verifier_sandbox_has_no_host_mounts(monkeypatch, tmp_path) -> None: + _root, task = _task_tree(tmp_path) + captured = {} + + def fake_sandbox(provider, spec): + captured["provider"] = provider + captured["spec"] = spec + return SimpleNamespace() + + runner = app.LegalAgentBenchAgent.model_construct(config=_config()) + monkeypatch.setattr(runner, "_provider_config", lambda: {"docker": {}}) + monkeypatch.setattr(runner, "_sandbox_metadata", lambda: {}) + monkeypatch.setattr(app, "AsyncSandbox", fake_sandbox) + + runner._verifier_sandbox(image="lab:image", task_dir=task) + + assert captured["spec"].workdir == "/tmp" + assert captured["spec"].env == app.LAB_SANDBOX_ENV + assert captured["spec"].provider_options == {} + + +def test_agent_and_verifier_merge_phase_specific_provider_options(monkeypatch, tmp_path) -> None: + _root, task = _task_tree(tmp_path) + captured = [] + + def fake_sandbox(_provider, spec): + captured.append(spec) + return SimpleNamespace() + + provider = {"openshell": {}} + runner = app.LegalAgentBenchAgent.model_construct( + config=_config( + sandbox_provider=provider, + agent_sandbox_provider_options={"policy": {"version": 1, "network_policies": {"agent": {}}}}, + verifier_sandbox_provider_options={"policy": {"version": 1, "network_policies": {"judge": {}}}}, + ) + ) + monkeypatch.setattr(runner, "_sandbox_metadata", lambda: {}) + monkeypatch.setattr(app, "AsyncSandbox", fake_sandbox) + + runner._agent_sandbox(image="lab:image", task_dir=task, model_url="https://model.example/v1") + runner._verifier_sandbox(image="lab:image", task_dir=task) + + assert captured[0].provider_options == {"policy": {"version": 1, "network_policies": {"agent": {}}}} + assert captured[1].provider_options == {"policy": {"version": 1, "network_policies": {"judge": {}}}} + + +@pytest.mark.parametrize( + "provider_name", + ["docker", "ecs_fargate", "enroot", "apptainer", "opensandbox", "daytona", "openshell"], +) +def test_agent_and_verifier_sandboxes_use_the_selected_provider(monkeypatch, tmp_path, provider_name) -> None: + _root, task = _task_tree(tmp_path) + provider = {provider_name: {}} + captured = [] + + def fake_sandbox(received_provider, spec): + captured.append((received_provider, spec)) + return SimpleNamespace() + + runner = app.LegalAgentBenchAgent.model_construct( + config=_config( + sandbox_provider=provider, + sandbox_model_base_url="https://policy-proxy.example/v1", + ) + ) + monkeypatch.setattr(runner, "_sandbox_metadata", lambda: {"benchmark": "legal-agent-bench"}) + monkeypatch.setattr(app, "AsyncSandbox", fake_sandbox) + + runner._agent_sandbox(image="provider-native-image", task_dir=task, model_url="https://policy-proxy.example/v1") + runner._verifier_sandbox( + image="provider-native-image", + task_dir=task, + ) + + assert [next(iter(received)) for received, _spec in captured] == [provider_name, provider_name] + if provider_name != "docker": + assert [received for received, _spec in captured] == [provider, provider] + assert all(spec.image == "provider-native-image" for _provider, spec in captured) + assert all(spec.workdir == "/tmp" for _provider, spec in captured) + assert all(spec.env == app.LAB_SANDBOX_ENV for _provider, spec in captured) + assert all(spec.metadata == {"benchmark": "legal-agent-bench"} for _provider, spec in captured) + expected_options = ( + {"resource_requests": {"cpu": 0.5, "memory_mib": 1024, "disk_gib": 10}} + if provider_name == "opensandbox" + else {} + ) + assert all(spec.provider_options == expected_options for _provider, spec in captured) + + +def test_opensandbox_request_fraction_can_be_disabled(monkeypatch, tmp_path) -> None: + _root, task = _task_tree(tmp_path) + captured = [] + + def fake_sandbox(_provider, spec): + captured.append(spec) + return SimpleNamespace() + + runner = app.LegalAgentBenchAgent.model_construct( + config=_config( + sandbox_provider={"opensandbox": {}}, + opensandbox_request_fraction=None, + sandbox_model_base_url="https://policy-proxy.example/v1", + ) + ) + monkeypatch.setattr(runner, "_sandbox_metadata", lambda: {}) + monkeypatch.setattr(app, "AsyncSandbox", fake_sandbox) + + runner._agent_sandbox(image="provider-native-image", task_dir=task, model_url="https://policy-proxy.example/v1") + runner._verifier_sandbox(image="provider-native-image", task_dir=task) + + assert len(captured) == 2 + assert all(spec.provider_options == {} for spec in captured) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("reward_blob", "expected_reward", "expected_verifier_failed"), + [ + (b'{"reward": 1.0, "criteria_pass_rate": 1.0}', 1.0, False), + (b'{"reward": "invalid", "criteria_pass_rate": 1.0}', 0.0, True), + ], +) +@pytest.mark.parametrize( + "agent_module", + [ + "responses_api_agents.legal_agent_bench_native_agent.app", + "responses_api_agents.hermes_agent.app", + "responses_api_agents.claude_code_agent.app", + "responses_api_agents.codex_agent.app", + ], +) +async def test_incomplete_limit_outcomes_are_verified_for_every_harness( + monkeypatch, + tmp_path, + capsys, + reward_blob, + expected_reward, + expected_verifier_failed, + agent_module, +) -> None: + _root, task = _task_tree(tmp_path) + skills = _skills(tmp_path) + deps = tmp_path / "deps" + deps.mkdir() + events = [] + + class PhaseSandbox: + def __init__(self, phase): + self.phase = phase + + async def start(self): + events.append(f"{self.phase}:start") + + async def exec(self, *args, **kwargs): + assert "user" not in kwargs + events.append(f"{self.phase}:exec") + return SimpleNamespace(return_code=0, error_type=None, stdout="", stderr="") + + async def stop(self): + if self.phase == "agent": + assert not runner._session_results_dir.exists() + events.append(f"{self.phase}:stop") + + agent_sandbox = PhaseSandbox("agent") + verifier_sandbox = PhaseSandbox("verifier") + runner = app.LegalAgentBenchAgent.model_construct(config=_config(agent_server_module=agent_module)) + runner._sem = app.asyncio.Semaphore(1) + runner._session_results_dir = tmp_path / "results" + + async def ensure_image(_task): + return "lab:image" + + async def ensure_runtime(_image): + return deps + + async def stage_agent(*args, **kwargs): + return None + + async def collect_agent(*args, **kwargs): + return {} + + def write_runner(paths, params, model_url): + incomplete = app.NeMoGymResponse.model_validate( + _successful_response().model_dump(mode="json") + | { + "status": "incomplete", + "incomplete_details": {"reason": "max_output_tokens"}, + "metadata": {"nemo_gym_stop_reason": "max_turns"}, + } + ) + (paths["runtime"] / "response.json").write_text(incomplete.model_dump_json()) + + async def run_verifier(sandbox, task_dir, paths): + assert sandbox is verifier_sandbox + assert events == ["agent:start", "agent:exec", "agent:stop", "verifier:start"] + assert runner._session_results_dir.is_dir() + return {"reward.json": reward_blob}, False, None + + monkeypatch.setattr(app, "resolve_task_dir", lambda runtime, instance: task) + monkeypatch.setattr(app, "resolve_repo_path", lambda path: skills) + monkeypatch.setattr(app, "compose_agent_input", lambda task_dir, skills_dir, params, **kwargs: params) + monkeypatch.setattr(runner, "_ensure_image", ensure_image) + monkeypatch.setattr(runner, "_ensure_runtime", ensure_runtime) + monkeypatch.setattr(runner, "_stage_agent_source", lambda paths: None) + monkeypatch.setattr(runner, "_write_runner_config", write_runner) + monkeypatch.setattr(runner, "_model_url", lambda body: "http://model") + monkeypatch.setattr(runner, "_model_name", lambda: "policy") + monkeypatch.setattr(runner, "_agent_sandbox", lambda **kwargs: agent_sandbox) + monkeypatch.setattr(runner, "_stage_agent_sandbox", stage_agent) + monkeypatch.setattr(runner, "_collect_agent_sandbox", collect_agent) + monkeypatch.setattr(runner, "_materialize_agent_downloads", lambda *args, **kwargs: None) + monkeypatch.setattr(runner, "_verifier_sandbox", lambda **kwargs: verifier_sandbox) + monkeypatch.setattr(runner, "_artifacts", lambda *args, **kwargs: None) + monkeypatch.setattr(runner, "_stage_and_run_verifier", run_verifier) + body = app.LegalAgentBenchRunRequest( + instance_id="legal_agent_bench::area__task", + responses_create_params=NeMoGymResponseCreateParamsNonStreaming(input=[]), + ) + + response = await runner.run(None, body) + + assert events == ["agent:start", "agent:exec", "agent:stop", "verifier:start", "verifier:stop"] + assert response.reward == expected_reward + assert response.verifier_failed is expected_verifier_failed + assert response.sandbox_failed is False + response_data = response.model_dump() + if expected_verifier_failed: + assert response_data["_ng_failure_class"] == "verifier_failed" + assert "_ng_failure_terminal" not in response_data + else: + assert "_ng_failure_class" not in response_data + assert "_ng_failure_terminal" not in response_data + assert response.mask_sample is False + assert Path(response.artifact_dir).is_dir() + terminal_output = capsys.readouterr().out + assert "LAB rollout artifacts:" in terminal_output + expected_terminal_status = "failed" if expected_verifier_failed else "complete" + assert f"LAB rollout {expected_terminal_status}:" in terminal_output + + +@pytest.mark.asyncio +async def test_model_connectivity_failure_is_masked_and_skips_verifier(monkeypatch, tmp_path, capsys) -> None: + _root, task = _task_tree(tmp_path) + skills = _skills(tmp_path) + deps = tmp_path / "deps" + deps.mkdir() + events = [] + + class AgentSandbox: + async def start(self): + events.append("agent:start") + + async def exec(self, *args, **kwargs): + events.append("agent:exec") + return SimpleNamespace(return_code=1, error_type=None, stdout="", stderr="connection failed") + + async def stop(self): + events.append("agent:stop") + + runner = app.LegalAgentBenchAgent.model_construct(config=_config()) + runner._sem = app.asyncio.Semaphore(1) + runner._session_results_dir = tmp_path / "results" + runner._session_results_dir.mkdir() + + async def ensure_image(_task): + return "lab:image" + + async def ensure_runtime(_image): + return deps + + def write_runner(paths, params, model_url): + (paths["runtime"] / "runner_status.json").write_text( + json.dumps( + { + "ok": False, + "phase": "model_connectivity", + "error": "Policy model is unreachable from the LAB sandbox: http://model", + } + ) + ) + + async def stage_agent(*args, **kwargs): + return None + + async def collect_agent(*args, **kwargs): + return {} + + monkeypatch.setattr(app, "resolve_task_dir", lambda runtime, instance: task) + monkeypatch.setattr(app, "resolve_repo_path", lambda path: skills) + monkeypatch.setattr(app, "compose_agent_input", lambda task_dir, skills_dir, params: params) + monkeypatch.setattr(runner, "_ensure_image", ensure_image) + monkeypatch.setattr(runner, "_ensure_runtime", ensure_runtime) + monkeypatch.setattr(runner, "_stage_agent_source", lambda paths: None) + monkeypatch.setattr(runner, "_write_runner_config", write_runner) + monkeypatch.setattr(runner, "_model_url", lambda body: "http://model") + monkeypatch.setattr(runner, "_model_name", lambda: "policy") + monkeypatch.setattr(runner, "_agent_sandbox", lambda **kwargs: AgentSandbox()) + monkeypatch.setattr(runner, "_stage_agent_sandbox", stage_agent) + monkeypatch.setattr(runner, "_collect_agent_sandbox", collect_agent) + monkeypatch.setattr(runner, "_materialize_agent_downloads", lambda *args, **kwargs: None) + monkeypatch.setattr( + runner, + "_verifier_sandbox", + lambda **kwargs: pytest.fail("verifier must not run after model connectivity failure"), + ) + body = app.LegalAgentBenchRunRequest( + instance_id="legal_agent_bench::area__task", + responses_create_params=NeMoGymResponseCreateParamsNonStreaming(input=[]), + ) + + response = await runner.run(None, body) + + assert events == ["agent:start", "agent:exec", "agent:stop"] + assert response.agent_failed is True + assert response.model_connection_failed is True + assert response.mask_sample is True + assert response.verifier_failed is False + assert response.model_dump()["_ng_failure_class"] == "model_connection_failed" + assert "_ng_failure_terminal" not in response.model_dump() + assert "unreachable" in response.failure_reason + summary = json.loads(Path(response.run_summary_path).read_text()) + assert summary["flags"]["model_connection_failed"] is True + assert summary["paths"]["agent_trace"] == response.agent_trace_path + terminal_output = capsys.readouterr().out + assert "LAB rollout artifacts:" in terminal_output + assert "LAB rollout failed:" in terminal_output + + +@pytest.mark.asyncio +async def test_native_model_timeout_is_masked_routed_and_skips_verifier(monkeypatch, tmp_path) -> None: + _root, task = _task_tree(tmp_path) + skills = _skills(tmp_path) + deps = tmp_path / "deps" + deps.mkdir() + events = [] + + class AgentSandbox: + async def start(self): + events.append("agent:start") + + async def exec(self, *args, **kwargs): + events.append("agent:exec") + return SimpleNamespace(return_code=0, error_type=None, stdout="", stderr="") + + async def stop(self): + events.append("agent:stop") + + runner = app.LegalAgentBenchAgent.model_construct( + config=_config( + agent_server_module=app.NATIVE_AGENT_MODULE, + agent_server_class="LegalAgentBenchNativeAgent", + agent_config_class="LegalAgentBenchNativeAgentConfig", + ) + ) + runner._sem = app.asyncio.Semaphore(1) + runner._session_results_dir = tmp_path / "results" + + async def ensure_image(_task): + return "lab:image" + + async def ensure_runtime(_image): + return deps + + async def stage_agent(*args, **kwargs): + return None + + async def collect_agent(*args, **kwargs): + return {} + + def write_runner(paths, params, model_url): + failed = app.NeMoGymResponse.model_validate( + { + **_successful_response("Partial work").model_dump(mode="json"), + "status": "failed", + "error": {"code": "server_error", "message": "LAB model call timed out after 1800s"}, + "metadata": {app.AGENT_FAILURE_CLASS_METADATA_KEY: "agent_timed_out"}, + } + ) + (paths["runtime"] / "response.json").write_text(failed.model_dump_json()) + + monkeypatch.setattr(app, "resolve_task_dir", lambda runtime, instance: task) + monkeypatch.setattr(app, "resolve_repo_path", lambda path: skills) + monkeypatch.setattr(app, "compose_agent_input", lambda task_dir, skills_dir, params, **kwargs: params) + monkeypatch.setattr(runner, "_ensure_image", ensure_image) + monkeypatch.setattr(runner, "_ensure_runtime", ensure_runtime) + monkeypatch.setattr(runner, "_stage_agent_source", lambda paths: None) + monkeypatch.setattr(runner, "_write_runner_config", write_runner) + monkeypatch.setattr(runner, "_model_url", lambda body: "http://model") + monkeypatch.setattr(runner, "_model_name", lambda: "policy") + monkeypatch.setattr(runner, "_agent_sandbox", lambda **kwargs: AgentSandbox()) + monkeypatch.setattr(runner, "_stage_agent_sandbox", stage_agent) + monkeypatch.setattr(runner, "_collect_agent_sandbox", collect_agent) + monkeypatch.setattr(runner, "_materialize_agent_downloads", lambda *args, **kwargs: None) + monkeypatch.setattr( + runner, + "_verifier_sandbox", + lambda **kwargs: pytest.fail("verifier must not run after a native model timeout"), + ) + monkeypatch.setattr(runner, "_artifacts", lambda *args, **kwargs: None) + body = app.LegalAgentBenchRunRequest( + instance_id="legal_agent_bench::area__task", + responses_create_params=NeMoGymResponseCreateParamsNonStreaming(input=[]), + ) + + response = await runner.run(None, body) + + assert events == ["agent:start", "agent:exec", "agent:stop"] + assert response.agent_failed is True + assert response.agent_timed_out is True + assert response.model_connection_failed is False + assert response.mask_sample is True + assert response.model_dump()["_ng_failure_class"] == "agent_timed_out" + assert "timed out after 1800s" in response.failure_reason + assert response.response.output + + +class _FakeSandbox: + def __init__(self, reward: dict | None, *, timed_out: bool = False, fail_staging: bool = False): + self.reward = reward + self.timed_out = timed_out + self.fail_staging = fail_staging + self.uploaded = [] + self.upload_modes = [] + self.exec_calls = [] + + async def upload(self, source, destination): + self.uploaded.append(destination) + self.upload_modes.append(Path(source).stat().st_mode & 0o777) + + async def exec(self, command, **kwargs): + assert "user" not in kwargs + self.exec_calls.append((command, kwargs)) + if self.fail_staging and "verifier-input.tar.gz" in command: + return type( + "Result", + (), + { + "return_code": 1, + "error_type": None, + "stdout": "", + "stderr": "staging failed", + }, + )() + if command.startswith("for filename in"): + stdout = "reward.json\nscores.json\n" if self.reward is not None else "" + return type("Result", (), {"return_code": 0, "error_type": None, "stdout": stdout, "stderr": ""})() + timed_out = self.timed_out and command.startswith("bash ") + return type( + "Result", + (), + { + "return_code": 124 if timed_out else 0, + "error_type": "timeout" if timed_out else None, + "stdout": "", + "stderr": "", + }, + )() + + async def download(self, source, destination): + if self.reward is None: + raise FileNotFoundError(source) + payload = self.reward if source.endswith("reward.json") else {"summary": "ok"} + Path(destination).write_text(json.dumps(payload)) + + +@pytest.mark.asyncio +async def test_verifier_is_staged_after_agent_and_receives_secrets_only_on_exec(tmp_path) -> None: + _root, task = _task_tree(tmp_path) + paths = {"verifier": tmp_path / "verifier", "lab_run": tmp_path / "lab-run"} + paths["verifier"].mkdir() + (paths["lab_run"] / "output").mkdir(parents=True) + sandbox = _FakeSandbox({"reward": 1.0, "criteria_pass_rate": 1.0, "judge_error_count": 0}) + runner = app.LegalAgentBenchAgent.model_construct(config=_config()) + + downloaded, timed_out, failure = await runner._stage_and_run_verifier(sandbox, task, paths) + reward = runner._materialize_verifier_downloads(paths, downloaded) + + assert sandbox.uploaded == [f"{app.SANDBOX_ROOT}/verifier-input.tar.gz"] + assert sandbox.upload_modes == [0o644] + assert len(sandbox.exec_calls) == 4 + assert sandbox.exec_calls[0][0] == f"mkdir -p {app.SANDBOX_ROOT}" + staging_call = sandbox.exec_calls[1] + assert f"tar -xzf {app.SANDBOX_ROOT}/verifier-input.tar.gz" in staging_call[0] + assert f"mkdir -p {app.SANDBOX_LOGS}/verifier" in staging_call[0] + assert staging_call[1]["cwd"] == "/tmp" + verifier_call = next(call for call in sandbox.exec_calls if f"bash {app.SANDBOX_TESTS}/test.sh" in call[0]) + assert "user" not in verifier_call[1] + assert verifier_call[1]["cwd"] == f"{app.SANDBOX_LOGS}/agent/artifacts/lab-run/output" + assert verifier_call[1]["env"]["LAB_JUDGE_API_KEY"] == "secret" # pragma: allowlist secret + assert verifier_call[1]["env"]["LAB_TESTS_DIR"] == app.SANDBOX_TESTS + assert verifier_call[1]["env"]["LAB_LOGS_DIR"] == app.SANDBOX_LOGS + manifest_call = next(call for call in sandbox.exec_calls if call[0].startswith("for filename in")) + assert "error.json" in manifest_call[0] + assert manifest_call[0].endswith("done; true") + assert reward["reward"] == 1.0 + assert timed_out is False + assert failure is None + + +@pytest.mark.asyncio +async def test_verifier_staging_failure_skips_execution_and_is_reported(tmp_path) -> None: + _root, task = _task_tree(tmp_path) + paths = {"verifier": tmp_path / "verifier", "lab_run": tmp_path / "lab-run"} + paths["verifier"].mkdir() + (paths["lab_run"] / "output").mkdir(parents=True) + sandbox = _FakeSandbox(None, fail_staging=True) + runner = app.LegalAgentBenchAgent.model_construct(config=_config()) + + reward, timed_out, failure = await runner._stage_and_run_verifier(sandbox, task, paths) + + assert reward == {} + assert timed_out is False + assert failure == "staging failed" + assert len(sandbox.exec_calls) == 2 + + +@pytest.mark.asyncio +async def test_missing_verifier_reward_returns_failure(tmp_path) -> None: + _root, task = _task_tree(tmp_path) + paths = {"verifier": tmp_path / "verifier", "lab_run": tmp_path / "lab-run"} + paths["verifier"].mkdir() + (paths["lab_run"] / "output").mkdir(parents=True) + sandbox = _FakeSandbox(None, timed_out=True) + runner = app.LegalAgentBenchAgent.model_construct(config=_config(verifier_timeout_seconds=1)) + + reward, timed_out, failure = await runner._stage_and_run_verifier(sandbox, task, paths) + + assert reward == {} + assert timed_out is True + assert "reward.json" in failure + + +@pytest.mark.parametrize("link_type", [tarfile.SYMTYPE, tarfile.LNKTYPE]) +def test_untrusted_output_archive_rejects_links_without_touching_host(tmp_path, link_type) -> None: + victim = tmp_path / "victim.txt" + victim.write_text("safe") + archive_path = tmp_path / "malicious.tar.gz" + with tarfile.open(archive_path, "w:gz") as archive: + link = tarfile.TarInfo("stdout.log") + link.type = link_type + link.linkname = str(victim) + archive.addfile(link) + + with pytest.raises(app.LegalAgentBenchArtifactError, match="links"): + app._extract_untrusted_archive(archive_path, tmp_path / "output") + + assert victim.read_text() == "safe" + + +def test_materialized_output_is_owned_by_invoking_user(tmp_path) -> None: + source = tmp_path / "source" + source.mkdir() + (source / "memo.txt").write_text("complete") + archive_path = tmp_path / "output.tar.gz" + app._create_archive(archive_path, [(source, ".")]) + output = tmp_path / "materialized" + + app._extract_untrusted_archive(archive_path, output) + + assert (output / "memo.txt").stat().st_uid == os.getuid() + assert (output / "memo.txt").stat().st_gid == os.getgid() + + +def test_named_remote_provider_accepts_provider_native_image_and_requires_reachable_model_url( + monkeypatch, tmp_path +) -> None: + global_config = { + "sandbox": { + "default_metadata": {"sandbox-api": "remote"}, + "remote": {"endpoint": "sandbox.internal"}, + } + } + runner = app.LegalAgentBenchAgent.model_construct( + config=_config(sandbox_provider="sandbox", sandbox_image="registry.example/lab:latest"), + server_client=SimpleNamespace( + global_config_dict=global_config, + _build_server_base_url=lambda _config: "http://127.0.0.1:8000", + ), + ) + monkeypatch.setattr(app, "get_first_server_config_dict", lambda *_args: {}) + monkeypatch.setattr(app.LegalAgentBenchAgent, "rollout_id_from_run", lambda _self, _body: None) + + assert "remote" in runner._provider_config() + assert runner._sandbox_metadata() == {"sandbox-api": "remote"} + body = app.LegalAgentBenchRunRequest( + instance_id="legal_agent_bench::area__task", + responses_create_params=NeMoGymResponseCreateParamsNonStreaming(input=[]), + ) + with pytest.raises(app.LegalAgentBenchConfigurationError, match="sandbox_model_base_url"): + runner._model_url(body) + assert app.asyncio.run(runner._ensure_image(tmp_path)) == runner.config.sandbox_image + + runner.config.sandbox_model_base_url = "https://policy-proxy.example/v1" + assert runner._model_url(body) == "https://policy-proxy.example/v1" + + +@pytest.mark.parametrize("image", ["docker://registry.example/lab:latest", "/images/lab.sif", "/images/lab.sqsh"]) +def test_non_docker_provider_preserves_native_image_reference(image, tmp_path) -> None: + runner = app.LegalAgentBenchAgent.model_construct( + config=_config( + sandbox_provider={"apptainer": {}}, + sandbox_image=image, + ) + ) + + assert app.asyncio.run(runner._ensure_image(tmp_path)) == image + + +@pytest.mark.asyncio +async def test_agent_staging_excludes_tests_and_judge_credentials(tmp_path) -> None: + _root, task = _task_tree(tmp_path) + skills = _skills(tmp_path) + runtime_archive = tmp_path / "runtime.tar.gz" + with tarfile.open(runtime_archive, "w:gz") as archive: + root = tarfile.TarInfo("agent_deps") + root.type = tarfile.DIRTYPE + archive.addfile(root) + paths = {"agent_source": tmp_path / "agent-source", "runtime": tmp_path / "runtime"} + for path in paths.values(): + path.mkdir() + (paths["agent_source"] / "responses_api_agents" / "hermes_agent").mkdir(parents=True) + (paths["agent_source"] / "responses_api_agents" / "hermes_agent" / "app.py").write_text("VALUE = 1\n") + (paths["runtime"] / "runner.json").write_text("{}") + + class CaptureSandbox: + def __init__(self): + self.members = {} + self.upload_modes = {} + self.exec_commands = [] + + async def upload(self, source, destination): + self.upload_modes[destination] = Path(source).stat().st_mode & 0o777 + with tarfile.open(source, "r:gz") as archive: + self.members[destination] = {member.name for member in archive.getmembers()} + + async def exec(self, command, **kwargs): + assert "user" not in kwargs + self.exec_commands.append(command) + return SimpleNamespace(return_code=0, error_type=None, stdout="", stderr="") + + sandbox = CaptureSandbox() + runner = app.LegalAgentBenchAgent.model_construct(config=_config()) + + await runner._stage_agent_sandbox( + sandbox, + task_dir=task, + skills_dir=skills, + runtime_archive=runtime_archive, + paths=paths, + ) + + staged = sandbox.members[f"{app.SANDBOX_ROOT}/agent-input.tar.gz"] + assert any(name.startswith("workspace/vdr") for name in staged) + assert any(name.startswith("workspace/skills") for name in staged) + assert not any("task.toml" in name or "/tests" in name or "LAB_JUDGE" in name for name in staged) + assert set(sandbox.upload_modes.values()) == {0o644} + assert sandbox.exec_commands[0] == f"mkdir -p {app.SANDBOX_ROOT}" + assert f"chmod -R a+rX,a-w {app.SANDBOX_AGENT_SOURCE} {app.SANDBOX_AGENT_DEPS}" in sandbox.exec_commands[1] + assert "chown" not in sandbox.exec_commands[1] + + +@pytest.mark.asyncio +async def test_run_process_returns_output_and_terminates_on_timeout(tmp_path) -> None: + code, stdout, stderr = await app._run_process( + [sys.executable, "-c", "import sys; print('out'); print('err', file=sys.stderr)"], + cwd=tmp_path, + timeout=5, + ) + assert (code, stdout.strip(), stderr.strip()) == (0, "out", "err") + + with pytest.raises(TimeoutError, match="Command timed out"): + await app._run_process( + [sys.executable, "-c", "import time; time.sleep(10)"], + cwd=tmp_path, + timeout=0.01, + ) + + +@pytest.mark.asyncio +async def test_runtime_is_provisioned_once_per_agent_instance(monkeypatch, tmp_path) -> None: + _runtime_sources(monkeypatch, tmp_path, "hermes_agent") + _RuntimeBuilderSandbox.instances = [] + monkeypatch.setattr(app, "AsyncSandbox", _RuntimeBuilderSandbox) + provider = {"enroot": {}} + runner = app.LegalAgentBenchAgent.model_construct(config=_config()) + runner._runtime_lock = app.asyncio.Lock() + runner._runtime_archives = {} + monkeypatch.setattr(runner, "_provider_config", lambda: provider) + monkeypatch.setattr(runner, "_sandbox_metadata", lambda: {}) + + first, second = await app.asyncio.gather(runner._ensure_runtime("lab:image"), runner._ensure_runtime("lab:image")) + + assert first == second + assert len(_RuntimeBuilderSandbox.instances) == 1 + + +@pytest.mark.asyncio +async def test_docker_image_resolution_handles_missing_cached_and_failed_builds(monkeypatch, tmp_path) -> None: + _root, task = _task_tree(tmp_path) + runner = app.LegalAgentBenchAgent.model_construct(config=_config()) + runner._image_lock = app.asyncio.Lock() + + monkeypatch.setattr(app.shutil, "which", lambda _name: None) + with pytest.raises(FileNotFoundError, match="Docker CLI"): + await runner._ensure_image(task) + + monkeypatch.setattr(app.shutil, "which", lambda _name: "/usr/bin/docker") + + async def cached(*args, **kwargs): + return 0, "", "" + + monkeypatch.setattr(app, "_run_process", cached) + assert await runner._ensure_image(task) == ( + f"{runner.config.image_repository}:" + app._environment_hash(task / "environment")[:16] + ) + + async def failed_build(args, **kwargs): + return (1, "", "missing") if args[1] == "image" else (1, "", "build exploded") + + monkeypatch.setattr(app, "_run_process", failed_build) + with pytest.raises(RuntimeError, match="build exploded"): + await runner._ensure_image(task) + + +def test_model_url_reports_server_lookup_failures_and_routes_docker(monkeypatch) -> None: + body = app.LegalAgentBenchRunRequest( + instance_id="legal_agent_bench::area__task", + responses_create_params=NeMoGymResponseCreateParamsNonStreaming(input=[]), + ) + runner = app.LegalAgentBenchAgent.model_construct( + # Bridge networking makes this assertion platform-independent: loopback + # must be rewritten for a Docker bridge on Linux and Docker Desktop. + config=_config(docker_network="bridge"), + server_client=SimpleNamespace(global_config_dict={}), + ) + with pytest.raises(app.LegalAgentBenchConfigurationError, match="Unable to resolve policy model"): + runner._model_url(body) + + runner.server_client = SimpleNamespace( + global_config_dict={"policy_model": {}}, + _build_server_base_url=lambda _config: "http://127.0.0.1:8000", + ) + monkeypatch.setattr(app, "get_first_server_config_dict", lambda *_args: {}) + monkeypatch.setattr(app.LegalAgentBenchAgent, "rollout_id_from_run", lambda _self, _body: None) + monkeypatch.setattr(app.sys, "platform", "linux") + assert runner._model_url(body) == "http://host.docker.internal:8000" + + +@pytest.mark.asyncio +async def test_remote_provider_requires_an_image_and_missing_agent_source_is_configuration_error(tmp_path) -> None: + runner = app.LegalAgentBenchAgent.model_construct( + config=_config(sandbox_provider={"ecs_fargate": {"region": "us-east-1"}}) + ) + with pytest.raises(app.LegalAgentBenchConfigurationError, match="require.*sandbox_image"): + await runner._ensure_image(tmp_path) + + paths = runner._paths_for_root(tmp_path / "staging", create=True) + runner.config.agent_server_module = "responses_api_agents.nonexistent_agent.app" + with pytest.raises(app.LegalAgentBenchConfigurationError, match="source not found"): + runner._stage_agent_source(paths) + + +def test_failure_detection_covers_empty_and_zero_activity_responses() -> None: + empty = app._empty_response("policy") + assert app.agent_response_failure(empty, "responses_api_agents.codex_agent.app") == ( + "Agent produced an empty trajectory" + ) + + silent = _successful_response("") + silent.usage.total_tokens = 0 + assert app.agent_response_failure(silent, "responses_api_agents.codex_agent.app") == ( + "codex_agent produced no model activity" + ) + + assert app.host_tunnel_model_url("https://model.example/v1") == "https://model.example/v1" + + +def test_task_prompt_and_skills_report_invalid_source_files(tmp_path) -> None: + _root, task = _task_tree(tmp_path) + skills = _skills(tmp_path) + params = NeMoGymResponseCreateParamsNonStreaming(input=[]) + + (task / "task.json").write_text("not json") + with pytest.raises(app.LegalAgentBenchTaskError, match="Invalid LAB task configuration"): + app.compose_agent_input(task, skills, params) + + (task / "task.toml").write_text("not = [valid") + with pytest.raises(app.LegalAgentBenchTaskError, match="Invalid LAB task.toml"): + app._task_toml(task) + + (skills / app.REQUIRED_SKILLS[0] / "SKILL.md").unlink() + with pytest.raises(app.LegalAgentBenchConfigurationError, match="Invalid LAB skills configuration"): + app._load_skill_prompt(skills) + + +@pytest.mark.asyncio +async def test_agent_staging_and_collection_surface_sandbox_failures(tmp_path) -> None: + _root, task = _task_tree(tmp_path) + skills = _skills(tmp_path) + runtime_archive = tmp_path / "runtime.tar.gz" + with tarfile.open(runtime_archive, "w:gz") as archive: + root = tarfile.TarInfo("agent_deps") + root.type = tarfile.DIRTYPE + archive.addfile(root) + paths = app.LegalAgentBenchAgent._paths_for_root(tmp_path / "staged", create=True) + (paths["agent_source"] / "app.py").write_text("pass\n") + + class Sandbox: + async def upload(self, source, destination): + return None + + async def exec(self, command, **kwargs): + return SimpleNamespace(return_code=1, stdout="", stderr="sandbox command failed") + + runner = app.LegalAgentBenchAgent.model_construct(config=_config()) + with pytest.raises(RuntimeError, match="sandbox command failed"): + await runner._stage_agent_sandbox( + Sandbox(), task_dir=task, skills_dir=skills, runtime_archive=runtime_archive, paths=paths + ) + + with pytest.raises(RuntimeError, match="sandbox command failed"): + await runner._collect_agent_sandbox(Sandbox(), tmp_path / "downloads") + + +@pytest.mark.asyncio +async def test_agent_collection_and_materialization_preserve_optional_files(tmp_path) -> None: + download_dir = tmp_path / "downloads" + download_dir.mkdir() + output_source = tmp_path / "output-source" + output_source.mkdir() + (output_source / "memo.txt").write_text("complete") + + class Sandbox: + async def exec(self, command, **kwargs): + return SimpleNamespace(return_code=0, stdout="", stderr="") + + async def download(self, source, destination): + destination = Path(destination) + if source.endswith("response.json"): + destination.write_text(_successful_response().model_dump_json()) + elif source.endswith("runner_status.json"): + raise FileNotFoundError(source) + else: + app._create_archive(destination, [(output_source, ".")]) + + runner = app.LegalAgentBenchAgent.model_construct(config=_config()) + downloads = await runner._collect_agent_sandbox(Sandbox(), download_dir) + paths = runner._paths_for_root(tmp_path / "result", create=True) + runner._materialize_agent_downloads(paths, downloads, stdout="stdout", stderr="stderr") + + assert set(downloads) == {"response.json", "output.tar.gz"} + assert (paths["runtime"] / "response.json").is_file() + assert (paths["output"] / "memo.txt").read_text() == "complete" + assert (paths["agent"] / "stdout.log").read_text() == "stdout" + + with pytest.raises(app.LegalAgentBenchArtifactError, match="output archive"): + runner._materialize_agent_downloads(paths, {}, stdout="", stderr="") + + +def test_verifier_materialization_rejects_bad_reward_before_writing_files(tmp_path) -> None: + paths = {"verifier": tmp_path / "verifier"} + paths["verifier"].mkdir() + + with pytest.raises(app.LegalAgentBenchArtifactError, match="expected an object"): + app.LegalAgentBenchAgent._materialize_verifier_downloads( + paths, + {"reward.json": b"[]", "report.html": b"untrusted"}, + ) + assert list(paths["verifier"].iterdir()) == [] + + with pytest.raises(app.LegalAgentBenchArtifactError, match="Invalid verifier reward.json"): + app.LegalAgentBenchAgent._materialize_verifier_downloads(paths, {"reward.json": b"not-json"}) + + +def test_runner_status_reports_invalid_json_and_nonobject(tmp_path) -> None: + paths = {"runtime": tmp_path} + assert app.LegalAgentBenchAgent._runner_status(paths) == {} + + status = tmp_path / "runner_status.json" + status.write_text("not-json") + assert app.LegalAgentBenchAgent._runner_status(paths)["ok"] is False + + status.write_text("[]") + assert app.LegalAgentBenchAgent._runner_status(paths)["error"].endswith("expected an object") diff --git a/responses_api_agents/legal_agent_bench_native_agent/README.md b/responses_api_agents/legal_agent_bench_native_agent/README.md new file mode 100644 index 0000000000..c4c97b5a6f --- /dev/null +++ b/responses_api_agents/legal_agent_bench_native_agent/README.md @@ -0,0 +1,23 @@ +# Legal Agent Bench Native Agent + +This agent is the direct Gym-native implementation of Harvey LAB's model/tool +loop. It runs only as the inner agent of the hardened +[`legal_agent_bench_agent`](../legal_agent_bench_agent/README.md) sandbox +runner. + +For each turn it sends the accumulated Responses API trajectory and LAB's +canonical function-tool definitions to the configured Gym policy model. It +executes returned `bash`, `read`, `write`, `write_docx`, `edit`, `glob`, and +`grep` calls inside the task sandbox, appends their results, and continues until +the model returns a final assistant message or reaches the configured turn +limit. + +The outer runner owns task resolution, prompt and skill composition, sandbox +staging, result collection, verifier isolation, and artifact publication. A +model error, timeout, repeated empty response, failed runtime preflight, or +turn-limit exhaustion returns an explicit failed response while preserving any +partial trajectory. The outer runner masks that sample and skips verification. + +Use `--benchmark legal_agent_bench` for this default. See the +[benchmark README](../../benchmarks/legal_agent_bench/README.md) for setup, +smoke-test, and artifact-inspection commands. diff --git a/responses_api_agents/legal_agent_bench_native_agent/__init__.py b/responses_api_agents/legal_agent_bench_native_agent/__init__.py new file mode 100644 index 0000000000..42458d2190 --- /dev/null +++ b/responses_api_agents/legal_agent_bench_native_agent/__init__.py @@ -0,0 +1 @@ +"""Gym-native agent loop for Legal Agent Bench.""" diff --git a/responses_api_agents/legal_agent_bench_native_agent/app.py b/responses_api_agents/legal_agent_bench_native_agent/app.py new file mode 100644 index 0000000000..01f60ac260 --- /dev/null +++ b/responses_api_agents/legal_agent_bench_native_agent/app.py @@ -0,0 +1,409 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Gym-native implementation of the Legal Agent Bench tool loop.""" + +from __future__ import annotations + +import asyncio +import json +import os +import signal +from pathlib import Path +from typing import Any, Optional +from uuid import uuid4 + +import aiohttp +from fastapi import Body, Request +from pydantic import ConfigDict, Field + +from nemo_gym.base_resources_server import BaseRunRequest +from nemo_gym.base_responses_api_agent import BaseResponsesAPIAgentConfig, SimpleResponsesAPIAgent +from nemo_gym.config_types import ModelServerRef, ResourcesServerRef +from nemo_gym.openai_utils import ( + NeMoGymEasyInputMessage, + NeMoGymFunctionCallOutput, + NeMoGymResponse, + NeMoGymResponseCreateParamsNonStreaming, + NeMoGymResponseFunctionToolCall, + NeMoGymResponseOutputMessage, +) +from nemo_gym.server_utils import get_response_json, raise_for_status + + +INITIAL_EMPTY_RESPONSE_NUDGE = ( + "Your last response was empty and did not call any tools. Continue the task. " + "Use the available tools to inspect the documents and write the required deliverables." +) +CONTAINER_TOOL_RUNNER = Path("/opt/legal-agent-bench/container_tool_runner.py") +SUPPORTED_TOOLS = frozenset({"bash", "read", "write", "write_docx", "edit", "glob", "grep"}) +AGENT_FAILURE_CLASS_METADATA_KEY = "nemo_gym_failure_class" +MODEL_CONNECTION_FAILURE_CLASS = "model_connection_failed" +RETRYABLE_MODEL_ERROR_CODES = frozenset({"connection_error", "rate_limit_error", "server_error"}) + + +def _is_model_connection_exception(exc: BaseException) -> bool: + """Return whether a model-call exception is an operational transport failure.""" + if isinstance(exc, aiohttp.ClientResponseError): + return exc.status in {408, 429} or 500 <= exc.status < 600 + return isinstance(exc, (aiohttp.ClientConnectionError, aiohttp.ClientPayloadError)) + + +def _with_model_failure_class(response: NeMoGymResponse) -> NeMoGymResponse: + """Annotate retryable Responses API errors without masking incomplete outcomes.""" + if response.error is None or response.error.code not in RETRYABLE_MODEL_ERROR_CODES: + return response + metadata = dict(response.metadata or {}) + metadata[AGENT_FAILURE_CLASS_METADATA_KEY] = MODEL_CONNECTION_FAILURE_CLASS + return response.model_copy(update={"metadata": metadata}) + + +class LegalAgentBenchNativeAgentConfig(BaseResponsesAPIAgentConfig): + resources_server: ResourcesServerRef + model_server: ModelServerRef + max_turns: int = Field(default=60, ge=1) + shell_timeout: int = Field(default=60, ge=1) + preflight_timeout_seconds: int = Field(default=120, ge=1) + model_timeout_seconds: int = Field(default=1800, ge=1) + max_output_chars: int = Field(default=16_384, ge=1) + + +class LegalAgentBenchNativeRunRequest(BaseRunRequest): + model_config = ConfigDict(extra="allow") + + +class LabToolExecutor: + """Execute the upstream LAB tools in the current task sandbox.""" + + def __init__( + self, + *, + timeout_seconds: int, + max_output_chars: int, + preflight_timeout_seconds: int | None = None, + ) -> None: + self.timeout_seconds = timeout_seconds + self.preflight_timeout_seconds = preflight_timeout_seconds or timeout_seconds + self.max_output_chars = max_output_chars + + async def preflight(self) -> None: + result = await self.execute("preflight", {}) + if result.startswith("Error:"): + raise RuntimeError(result) + + async def execute(self, name: str, arguments: str | dict[str, Any]) -> str: + if name != "preflight" and name not in SUPPORTED_TOOLS: + return f"Error: unknown tool: {name}" + if isinstance(arguments, str): + try: + parsed = json.loads(arguments) + except json.JSONDecodeError as exc: + return f"Error: invalid JSON arguments for {name}: {exc}" + else: + parsed = arguments + if not isinstance(parsed, dict): + return f"Error: arguments for {name} must be a JSON object" + + if name == "bash": + command = parsed.get("command") + if not isinstance(command, str) or not command: + return "Error: command is required" + # Commands may contain generated documents or scripts large enough + # to exceed the host's argv limit. Preserve login-shell behavior, + # but stream the command over stdin so argv remains fixed-size. + result = await self._run(["/bin/bash", "-l", "-s"], stdin=command.encode()) + else: + command = ["/usr/local/bin/python", str(CONTAINER_TOOL_RUNNER), name] + stdin = json.dumps(parsed).encode() + if name == "preflight": + result = await self._run( + command, + stdin=stdin, + timeout_seconds=self.preflight_timeout_seconds, + ) + else: + result = await self._run(command, stdin=stdin) + full_read = name == "read" and "limit" in parsed and parsed.get("limit") in {0, None} + return result if full_read else self._truncate(result) + + async def _run( + self, + command: list[str], + *, + stdin: bytes | None = None, + timeout_seconds: int | None = None, + ) -> str: + effective_timeout = timeout_seconds or self.timeout_seconds + process = await asyncio.create_subprocess_exec( + *command, + cwd=os.environ.get("OUTPUT_DIR", "/workspace/output"), + env=os.environ.copy(), + stdin=asyncio.subprocess.PIPE if stdin is not None else None, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + start_new_session=True, + ) + try: + stdout, stderr = await asyncio.wait_for(process.communicate(input=stdin), timeout=effective_timeout) + except asyncio.TimeoutError: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + await process.communicate() + return f"Error: tool timed out after {effective_timeout}s" + + output = stdout.decode(errors="replace") + error = stderr.decode(errors="replace") + command_name = Path(command[1]).name if len(command) > 1 else "tool" + if command[0] == "/usr/local/bin/python": + try: + payload = json.loads(next(line for line in reversed(output.splitlines()) if line.strip())) + except (StopIteration, json.JSONDecodeError): + payload = None + if isinstance(payload, dict) and "result" in payload: + output = str(payload["result"]) + if process.returncode: + detail = error or output or f"{command_name} exited with code {process.returncode}" + return f"Error: {detail.strip()}" + if error: + output = f"{output}\nSTDERR:\n{error}" if output else error + return output or "(no output)" + + def _truncate(self, value: str) -> str: + if len(value) <= self.max_output_chars: + return value + return value[: self.max_output_chars] + "\n[output truncated]" + + +def _output_text(messages: list[NeMoGymResponseOutputMessage]) -> str: + parts: list[str] = [] + for message in messages: + for content in message.content: + text = getattr(content, "text", None) + if text: + parts.append(str(text)) + return "\n".join(parts).strip() + + +def _merge_usage(total: Any, current: Any) -> Any: + if current is None: + return total + if total is None: + return current.model_copy(deep=True) + total.input_tokens += current.input_tokens + total.output_tokens += current.output_tokens + total.total_tokens += current.total_tokens + total.input_tokens_details.cached_tokens += current.input_tokens_details.cached_tokens + total.output_tokens_details.reasoning_tokens += current.output_tokens_details.reasoning_tokens + return total + + +def _failed_response( + *, + body: NeMoGymResponseCreateParamsNonStreaming, + response: Optional[NeMoGymResponse], + output: list[Any], + usage: Any, + message: str, + failure_class: Optional[str] = None, +) -> NeMoGymResponse: + base = response or NeMoGymResponse( + id=f"resp_{uuid4().hex}", + created_at=0, + model=body.model or "policy_model", + object="response", + output=[], + parallel_tool_calls=body.parallel_tool_calls, + tool_choice=body.tool_choice, + tools=body.tools, + ) + metadata = dict(base.metadata or {}) + if failure_class is not None: + metadata[AGENT_FAILURE_CLASS_METADATA_KEY] = failure_class + return NeMoGymResponse.model_validate( + base.model_dump(mode="json") + | { + "status": "failed", + "error": {"code": "server_error", "message": message[-2000:]}, + "metadata": metadata, + "output": output, + "usage": usage, + } + ) + + +def _limit_response( + *, + body: NeMoGymResponseCreateParamsNonStreaming, + response: Optional[NeMoGymResponse], + output: list[Any], + usage: Any, + stop_reason: str, +) -> NeMoGymResponse: + """Return a scoreable incomplete response for a normal agent-loop limit.""" + base = response or NeMoGymResponse( + id=f"resp_{uuid4().hex}", + created_at=0, + model=body.model or "policy_model", + object="response", + output=[], + parallel_tool_calls=body.parallel_tool_calls, + tool_choice=body.tool_choice, + tools=body.tools, + ) + metadata = dict(base.metadata or {}) + metadata["nemo_gym_stop_reason"] = stop_reason + return NeMoGymResponse.model_validate( + base.model_dump(mode="json") + | { + "status": "incomplete", + "error": None, + "incomplete_details": {"reason": "max_output_tokens"}, + "metadata": metadata, + "output": output, + "usage": usage, + } + ) + + +class LegalAgentBenchNativeAgent(SimpleResponsesAPIAgent): + """Run LAB's canonical tool loop through Gym's Responses API model server.""" + + config: LegalAgentBenchNativeAgentConfig + + async def responses( + self, + request: Request, + body: NeMoGymResponseCreateParamsNonStreaming = Body(), + ) -> NeMoGymResponse: + body = body.model_copy(deep=True) + if isinstance(body.input, str): + body.input = [NeMoGymEasyInputMessage(role="user", content=body.input)] + + executor = LabToolExecutor( + timeout_seconds=self.config.shell_timeout, + preflight_timeout_seconds=self.config.preflight_timeout_seconds, + max_output_chars=self.config.max_output_chars, + ) + try: + await executor.preflight() + except Exception as exc: + return _failed_response( + body=body, + response=None, + output=[], + usage=None, + message=f"LAB tool preflight failed: {type(exc).__name__}: {exc}", + ) + + trajectory: list[Any] = [] + usage = None + last_response: Optional[NeMoGymResponse] = None + model_cookies = None + empty_responses = 0 + + for _turn in range(self.config.max_turns): + model_input = body.model_copy(update={"input": list(body.input) + trajectory}) + try: + raw_response = await asyncio.wait_for( + self.server_client.post( + server_name=self.config.model_server.name, + url_path=self.url_path_for_request("/v1/responses", request), + json=model_input, + cookies=model_cookies, + ), + timeout=self.config.model_timeout_seconds, + ) + await raise_for_status(raw_response) + model_response = NeMoGymResponse.model_validate(await get_response_json(raw_response)) + model_cookies = raw_response.cookies + except TimeoutError: + return _failed_response( + body=body, + response=last_response, + output=trajectory, + usage=usage, + message=f"LAB model call timed out after {self.config.model_timeout_seconds}s", + failure_class="agent_timed_out", + ) + except Exception as exc: + return _failed_response( + body=body, + response=last_response, + output=trajectory, + usage=usage, + message=f"LAB model call failed: {type(exc).__name__}: {exc}", + failure_class=(MODEL_CONNECTION_FAILURE_CLASS if _is_model_connection_exception(exc) else None), + ) + + last_response = model_response + usage = _merge_usage(usage, model_response.usage) + trajectory.extend(model_response.output) + if model_response.error is not None: + return _with_model_failure_class( + model_response.model_copy(update={"output": trajectory, "usage": usage}) + ) + if model_response.incomplete_details is not None: + return model_response.model_copy(update={"output": trajectory, "usage": usage}) + + function_calls = [ + item for item in model_response.output if isinstance(item, NeMoGymResponseFunctionToolCall) + ] + assistant_messages = [ + item for item in model_response.output if isinstance(item, NeMoGymResponseOutputMessage) + ] + if not function_calls and assistant_messages: + if _output_text(assistant_messages): + return model_response.model_copy( + update={"status": "completed", "output": trajectory, "usage": usage} + ) + empty_responses += 1 + if empty_responses > 2: + return _failed_response( + body=body, + response=model_response, + output=trajectory, + usage=usage, + message="LAB model repeatedly returned empty responses", + ) + trajectory.append(NeMoGymEasyInputMessage(role="user", content=INITIAL_EMPTY_RESPONSE_NUDGE)) + continue + + if not function_calls and not model_response.output: + empty_responses += 1 + if empty_responses > 2: + return _failed_response( + body=body, + response=model_response, + output=trajectory, + usage=usage, + message="LAB model repeatedly returned empty responses", + ) + trajectory.append(NeMoGymEasyInputMessage(role="user", content=INITIAL_EMPTY_RESPONSE_NUDGE)) + continue + + empty_responses = 0 + for call in function_calls: + result = await executor.execute(call.name, call.arguments) + trajectory.append( + NeMoGymFunctionCallOutput( + type="function_call_output", + call_id=call.call_id, + output=result, + ) + ) + + return _limit_response( + body=body, + response=last_response, + output=trajectory, + usage=usage, + stop_reason="max_turns", + ) + + async def run(self, request: Request, body: LegalAgentBenchNativeRunRequest): + raise NotImplementedError("The LAB-native agent runs inside the task-driven LAB sandbox") + + +if __name__ == "__main__": + LegalAgentBenchNativeAgent.run_webserver() diff --git a/responses_api_agents/legal_agent_bench_native_agent/requirements.txt b/responses_api_agents/legal_agent_bench_native_agent/requirements.txt new file mode 100644 index 0000000000..e16136d43f --- /dev/null +++ b/responses_api_agents/legal_agent_bench_native_agent/requirements.txt @@ -0,0 +1 @@ +# The LAB-native loop depends only on nemo_gym, installed by the setup script. diff --git a/responses_api_agents/legal_agent_bench_native_agent/scripts/legal_agent_bench_native_agent_deps.sh b/responses_api_agents/legal_agent_bench_native_agent/scripts/legal_agent_bench_native_agent_deps.sh new file mode 100755 index 0000000000..4476b84f9f --- /dev/null +++ b/responses_api_agents/legal_agent_bench_native_agent/scripts/legal_agent_bench_native_agent_deps.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -euo pipefail + +source "${PORTABLE_PYTHON_SH:?PORTABLE_PYTHON_SH is required}" + +mkdir -p "${DEPS_DIR:?DEPS_DIR is required}" +install_portable_python +install_nemo_gym_deps diff --git a/responses_api_agents/legal_agent_bench_native_agent/tests/test_app.py b/responses_api_agents/legal_agent_bench_native_agent/tests/test_app.py new file mode 100644 index 0000000000..a6b8a9d09e --- /dev/null +++ b/responses_api_agents/legal_agent_bench_native_agent/tests/test_app.py @@ -0,0 +1,447 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Tests for the Gym-native Legal Agent Bench loop.""" + +from __future__ import annotations + +import asyncio +import json +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from nemo_gym.config_types import ModelServerRef, ResourcesServerRef +from nemo_gym.openai_utils import ( + NeMoGymFunctionCallOutput, + NeMoGymResponseCreateParamsNonStreaming, + NeMoGymResponseFunctionToolCall, + NeMoGymResponseOutputMessage, +) +from nemo_gym.server_utils import ServerClient +from responses_api_agents.legal_agent_bench_native_agent import app + + +_REAL_PREFLIGHT = app.LabToolExecutor.preflight + + +def _config(**overrides) -> app.LegalAgentBenchNativeAgentConfig: + values = { + "host": "0.0.0.0", + "port": 10000, + "name": "legal_agent_bench_native_agent", + "entrypoint": "app.py", + "resources_server": ResourcesServerRef(name="lab", type="resources_servers"), + "model_server": ModelServerRef(name="policy_model", type="responses_api_models"), + } + values.update(overrides) + return app.LegalAgentBenchNativeAgentConfig(**values) + + +def _model_response(output: list[dict], *, response_id: str = "response", usage: bool = True) -> dict: + payload = { + "id": response_id, + "created_at": 1, + "model": "policy", + "object": "response", + "output": output, + "parallel_tool_calls": False, + "tool_choice": "auto", + "tools": [], + } + if usage: + payload["usage"] = { + "input_tokens": 3, + "input_tokens_details": {"cached_tokens": 1}, + "output_tokens": 2, + "output_tokens_details": {"reasoning_tokens": 1}, + "total_tokens": 5, + } + return payload + + +def _function_call(name: str = "glob", arguments: str = '{"pattern":"**/*"}') -> dict: + return { + "id": "fc-1", + "call_id": "call-1", + "name": name, + "arguments": arguments, + "type": "function_call", + "status": "completed", + } + + +def _assistant_message(text: str = "Done") -> dict: + return { + "id": "message-1", + "content": [{"annotations": [], "text": text, "type": "output_text"}], + "role": "assistant", + "status": "completed", + "type": "message", + } + + +def _raw_response(payload: dict) -> SimpleNamespace: + return SimpleNamespace(ok=True, read=AsyncMock(return_value=json.dumps(payload).encode()), cookies={}) + + +def _process(*, stdout: bytes = b"", stderr: bytes = b"", returncode: int = 0) -> SimpleNamespace: + return SimpleNamespace( + pid=123, + returncode=returncode, + communicate=AsyncMock(return_value=(stdout, stderr)), + ) + + +def _agent(*, max_turns: int = 60) -> app.LegalAgentBenchNativeAgent: + client = MagicMock(spec=ServerClient) + return app.LegalAgentBenchNativeAgent(config=_config(max_turns=max_turns), server_client=client) + + +@pytest.fixture(autouse=True) +def _successful_preflight(monkeypatch) -> None: + monkeypatch.setattr(app.LabToolExecutor, "preflight", AsyncMock()) + + +async def test_tool_loop_returns_full_responses_trajectory_and_usage(monkeypatch) -> None: + agent = _agent() + agent.server_client.post = AsyncMock( + side_effect=[ + _raw_response(_model_response([_function_call()], response_id="tool-turn")), + _raw_response(_model_response([_assistant_message()], response_id="final-turn")), + ] + ) + execute = AsyncMock(return_value="contract.docx") + monkeypatch.setattr(app.LabToolExecutor, "execute", execute) + body = NeMoGymResponseCreateParamsNonStreaming(input=[{"role": "user", "content": "Do the task"}]) + + result = await agent.responses(SimpleNamespace(path_params={}), body) + + assert result.status == "completed" + assert result.error is None + assert result.usage.total_tokens == 10 + assert [type(item) for item in result.output] == [ + NeMoGymResponseFunctionToolCall, + NeMoGymFunctionCallOutput, + NeMoGymResponseOutputMessage, + ] + execute.assert_awaited_once_with("glob", '{"pattern":"**/*"}') + second_input = agent.server_client.post.await_args_list[1].kwargs["json"].input + assert isinstance(second_input[-1], NeMoGymFunctionCallOutput) + assert second_input[-1].output == "contract.docx" + + +async def test_model_failure_preserves_partial_trajectory(monkeypatch) -> None: + agent = _agent() + agent.server_client.post = AsyncMock( + side_effect=[ + _raw_response(_model_response([_function_call()])), + RuntimeError("model disconnected"), + ] + ) + monkeypatch.setattr(app.LabToolExecutor, "execute", AsyncMock(return_value="file.txt")) + + result = await agent.responses( + SimpleNamespace(path_params={}), + NeMoGymResponseCreateParamsNonStreaming(input="Do the task"), + ) + + assert result.status == "failed" + assert result.error is not None + assert "model disconnected" in result.error.message + assert any(isinstance(item, NeMoGymResponseFunctionToolCall) for item in result.output) + assert any(isinstance(item, NeMoGymFunctionCallOutput) for item in result.output) + + +async def test_model_timeout_sets_structured_failure_metadata(monkeypatch) -> None: + agent = _agent() + agent.server_client.post = AsyncMock( + side_effect=[ + _raw_response(_model_response([_function_call()])), + TimeoutError(), + ] + ) + monkeypatch.setattr(app.LabToolExecutor, "execute", AsyncMock(return_value="file.txt")) + + result = await agent.responses( + SimpleNamespace(path_params={}), + NeMoGymResponseCreateParamsNonStreaming(input="Do the task"), + ) + + assert result.status == "failed" + assert result.error is not None + assert result.error.message == "LAB model call timed out after 1800s" + assert result.metadata == {app.AGENT_FAILURE_CLASS_METADATA_KEY: "agent_timed_out"} + assert any(isinstance(item, NeMoGymResponseFunctionToolCall) for item in result.output) + assert any(isinstance(item, NeMoGymFunctionCallOutput) for item in result.output) + + +@pytest.mark.parametrize( + ("status", "expected_failure_class"), + [ + (429, app.MODEL_CONNECTION_FAILURE_CLASS), + (500, app.MODEL_CONNECTION_FAILURE_CLASS), + (400, None), + ], +) +async def test_model_http_failures_classify_only_retryable_statuses(status, expected_failure_class) -> None: + agent = _agent() + agent.server_client.post = AsyncMock( + side_effect=app.aiohttp.ClientResponseError( + request_info=MagicMock(real_url="http://policy/v1/responses"), + history=(), + status=status, + message="model request failed", + ) + ) + + result = await agent.responses( + SimpleNamespace(path_params={}), + NeMoGymResponseCreateParamsNonStreaming(input="Do the task"), + ) + + assert result.status == "failed" + assert result.error is not None + assert f"ClientResponseError: {status}" in result.error.message + assert (result.metadata or {}).get(app.AGENT_FAILURE_CLASS_METADATA_KEY) == expected_failure_class + + +async def test_turn_limit_is_a_scoreable_incomplete_outcome(monkeypatch) -> None: + agent = _agent(max_turns=1) + agent.server_client.post = AsyncMock(return_value=_raw_response(_model_response([_function_call()]))) + monkeypatch.setattr(app.LabToolExecutor, "execute", AsyncMock(return_value="file.txt")) + + result = await agent.responses( + SimpleNamespace(path_params={}), + NeMoGymResponseCreateParamsNonStreaming(input="Do the task"), + ) + + assert result.status == "incomplete" + assert result.error is None + assert result.incomplete_details.reason == "max_output_tokens" + assert result.metadata == {"nemo_gym_stop_reason": "max_turns"} + assert len(result.output) == 2 + + +async def test_malformed_tool_arguments_are_returned_to_the_model(monkeypatch) -> None: + executor = app.LabToolExecutor(timeout_seconds=1, max_output_chars=100) + run = AsyncMock() + monkeypatch.setattr(executor, "_run", run) + + result = await executor.execute("read", "not-json") + + assert result.startswith("Error: invalid JSON arguments") + run.assert_not_awaited() + + +async def test_tool_executor_validates_calls_and_truncates_output(monkeypatch) -> None: + executor = app.LabToolExecutor(timeout_seconds=1, max_output_chars=5) + run = AsyncMock(return_value="complete output") + monkeypatch.setattr(executor, "_run", run) + + assert await executor.execute("unknown", {}) == "Error: unknown tool: unknown" + assert await executor.execute("read", []) == "Error: arguments for read must be a JSON object" + assert await executor.execute("bash", {}) == "Error: command is required" + assert await executor.execute("bash", {"command": "pwd"}) == "compl\n[output truncated]" + run.assert_awaited_once_with(["/bin/bash", "-l", "-s"], stdin=b"pwd") + + run.reset_mock(return_value=True) + run.return_value = "short" + assert await executor.execute("read", {"file_path": "memo.txt"}) == "short" + + +async def test_preflight_rejects_container_tool_error(monkeypatch) -> None: + executor = app.LabToolExecutor(timeout_seconds=1, max_output_chars=100) + monkeypatch.setattr(executor, "execute", AsyncMock(return_value="Error: missing document tooling")) + + with pytest.raises(RuntimeError, match="missing document tooling"): + await _REAL_PREFLIGHT(executor) + + +async def test_preflight_uses_its_dedicated_timeout(monkeypatch) -> None: + executor = app.LabToolExecutor( + timeout_seconds=60, + preflight_timeout_seconds=120, + max_output_chars=100, + ) + run = AsyncMock(return_value="OK") + monkeypatch.setattr(executor, "_run", run) + + await _REAL_PREFLIGHT(executor) + + run.assert_awaited_once_with( + ["/usr/local/bin/python", str(app.CONTAINER_TOOL_RUNNER), "preflight"], + stdin=b"{}", + timeout_seconds=120, + ) + + +async def test_explicit_full_read_is_not_truncated(monkeypatch) -> None: + executor = app.LabToolExecutor(timeout_seconds=1, max_output_chars=5) + monkeypatch.setattr(executor, "_run", AsyncMock(return_value="complete document")) + + result = await executor.execute("read", {"file_path": "memo.txt", "limit": 0}) + + assert result == "complete document" + + +async def test_tool_executor_passes_large_non_bash_payload_over_stdin(monkeypatch) -> None: + executor = app.LabToolExecutor(timeout_seconds=1, max_output_chars=2_000_000) + run = AsyncMock(return_value="written") + monkeypatch.setattr(executor, "_run", run) + content = "x" * 1_000_000 + + result = await executor.execute("write", {"file_path": "memo.txt", "content": content}) + + assert result == "written" + command = run.await_args.args[0] + assert command == ["/usr/local/bin/python", str(app.CONTAINER_TOOL_RUNNER), "write"] + assert json.loads(run.await_args.kwargs["stdin"]) == {"file_path": "memo.txt", "content": content} + + +async def test_tool_executor_passes_large_bash_command_over_stdin(monkeypatch) -> None: + executor = app.LabToolExecutor(timeout_seconds=1, max_output_chars=2_000_000) + run = AsyncMock(return_value="written") + monkeypatch.setattr(executor, "_run", run) + command = "printf %s " + "x" * 1_000_000 + + result = await executor.execute("bash", {"command": command}) + + assert result == "written" + assert run.await_args.args[0] == ["/bin/bash", "-l", "-s"] + assert run.await_args.kwargs["stdin"] == command.encode() + + +async def test_process_runner_parses_container_result_and_stderr(monkeypatch) -> None: + executor = app.LabToolExecutor(timeout_seconds=1, max_output_chars=100) + process = _process(stdout=b'log line\n{"result":"document text"}\n', stderr=b"tool warning") + create = AsyncMock(return_value=process) + monkeypatch.setattr(app.asyncio, "create_subprocess_exec", create) + + result = await executor._run( + ["/usr/local/bin/python", str(app.CONTAINER_TOOL_RUNNER), "read"], + stdin=b'{"file_path":"memo.docx"}', + ) + + assert result == "document text\nSTDERR:\ntool warning" + assert create.await_args.kwargs["cwd"] == "/workspace/output" + assert create.await_args.kwargs["start_new_session"] is True + assert create.await_args.kwargs["stdin"] is app.asyncio.subprocess.PIPE + process.communicate.assert_awaited_once_with(input=b'{"file_path":"memo.docx"}') + + +@pytest.mark.parametrize( + ("stdout", "returncode", "expected"), + [ + (b"not json", 0, "not json"), + (b"", 0, "(no output)"), + (b"command failed", 2, "Error: command failed"), + ], +) +async def test_process_runner_handles_unstructured_and_failed_output( + monkeypatch, + stdout: bytes, + returncode: int, + expected: str, +) -> None: + executor = app.LabToolExecutor(timeout_seconds=1, max_output_chars=100) + monkeypatch.setattr( + app.asyncio, + "create_subprocess_exec", + AsyncMock(return_value=_process(stdout=stdout, returncode=returncode)), + ) + + result = await executor._run( + ["/usr/local/bin/python", str(app.CONTAINER_TOOL_RUNNER), "preflight"], + stdin=b"{}", + ) + + assert result == expected + + +async def test_process_runner_kills_timed_out_process_group(monkeypatch) -> None: + executor = app.LabToolExecutor(timeout_seconds=7, max_output_chars=100) + process = _process() + monkeypatch.setattr(app.asyncio, "create_subprocess_exec", AsyncMock(return_value=process)) + + async def time_out(awaitable, *, timeout): + awaitable.close() + assert timeout == 7 + raise asyncio.TimeoutError + + killpg = MagicMock(side_effect=ProcessLookupError) + monkeypatch.setattr(app.asyncio, "wait_for", time_out) + monkeypatch.setattr(app.os, "killpg", killpg) + + result = await executor._run(["/bin/bash", "-lc", "sleep 60"]) + + assert result == "Error: tool timed out after 7s" + killpg.assert_called_once_with(123, app.signal.SIGKILL) + assert process.communicate.call_count == 2 + assert process.communicate.await_count == 1 + process.communicate.assert_any_call(input=None) + + +def test_merge_usage_ignores_missing_current_usage() -> None: + response = app.NeMoGymResponse.model_validate(_model_response([])) + + assert app._merge_usage(response.usage, None) is response.usage + + +async def test_model_error_response_is_returned_with_trajectory() -> None: + agent = _agent() + payload = _model_response([_assistant_message("partial")]) + payload.update( + { + "status": "failed", + "error": {"code": "server_error", "message": "policy backend failed"}, + } + ) + agent.server_client.post = AsyncMock(return_value=_raw_response(payload)) + + result = await agent.responses( + SimpleNamespace(path_params={}), + NeMoGymResponseCreateParamsNonStreaming(input="Do the task"), + ) + + assert result.status == "failed" + assert result.error is not None + assert result.error.message == "policy backend failed" + assert result.metadata == { + app.AGENT_FAILURE_CLASS_METADATA_KEY: app.MODEL_CONNECTION_FAILURE_CLASS, + } + assert any(isinstance(item, NeMoGymResponseOutputMessage) for item in result.output) + + +@pytest.mark.parametrize("empty_output", [[], [_assistant_message("")]]) +async def test_repeated_empty_model_responses_fail_after_two_nudges(empty_output) -> None: + agent = _agent(max_turns=3) + agent.server_client.post = AsyncMock( + side_effect=[_raw_response(_model_response(empty_output, response_id=f"empty-{index}")) for index in range(3)] + ) + + result = await agent.responses( + SimpleNamespace(path_params={}), + NeMoGymResponseCreateParamsNonStreaming(input="Do the task"), + ) + + assert result.status == "failed" + assert result.error is not None + assert "repeatedly returned empty responses" in result.error.message + second_input = agent.server_client.post.await_args_list[1].kwargs["json"].input + assert any(getattr(item, "content", None) == app.INITIAL_EMPTY_RESPONSE_NUDGE for item in second_input) + + +async def test_preflight_failure_returns_failed_response(monkeypatch) -> None: + agent = _agent() + monkeypatch.setattr(app.LabToolExecutor, "preflight", AsyncMock(side_effect=RuntimeError("missing pandoc"))) + + result = await agent.responses( + SimpleNamespace(path_params={}), + NeMoGymResponseCreateParamsNonStreaming(input="Do the task"), + ) + + assert result.status == "failed" + assert result.error is not None + assert "missing pandoc" in result.error.message + agent.server_client.post.assert_not_called() diff --git a/tests/unit_tests/test_apptainer_provider.py b/tests/unit_tests/test_apptainer_provider.py index b65e191e1b..04b4d1261a 100644 --- a/tests/unit_tests/test_apptainer_provider.py +++ b/tests/unit_tests/test_apptainer_provider.py @@ -184,7 +184,7 @@ def test_serialize_env_file_rejects_invalid_input(env: dict[Any, Any], error: ty def test_private_env_file_is_mode_0600_and_always_removed(tmp_path: Path) -> None: - content = b"TOKEN='not-a-real-secret'\n" + content = b"TOKEN='not-a-real-secret'\n" # pragma: allowlist secret with apptainer_provider._private_env_file(tmp_path, content) as path: assert path is not None @@ -387,13 +387,18 @@ def responder(argv: list[str]) -> tuple[int, str, str]: nonlocal env_file env_file = _env_file_path(argv) assert env_file.exists() - assert "not-a-real-secret" not in argv + assert "not-a-real-secret" not in argv # pragma: allowlist secret return (1, "", "boom") provider, _rec = _make_provider(monkeypatch, responder) with pytest.raises(apptainer_provider.ApptainerCreateError, match="failed"): - await provider.create(SandboxSpec(image="docker://img", env={"TOKEN": "not-a-real-secret"})) + await provider.create( + SandboxSpec( + image="docker://img", + env={"TOKEN": "not-a-real-secret"}, # pragma: allowlist secret + ) + ) assert env_file is not None and not env_file.exists() assert not staging.exists() @@ -414,8 +419,13 @@ def responder(argv: list[str]) -> tuple[int, str, str]: provider, _rec = _make_provider(monkeypatch, responder) with pytest.raises(apptainer_provider.ApptainerCreateError, match="timed out") as exc_info: - await provider.create(SandboxSpec(image="docker://img", env={"TOKEN": "not-a-real-secret"})) - assert "not-a-real-secret" not in str(exc_info.value) + await provider.create( + SandboxSpec( + image="docker://img", + env={"TOKEN": "not-a-real-secret"}, # pragma: allowlist secret + ) + ) + assert "not-a-real-secret" not in str(exc_info.value) # pragma: allowlist secret assert env_file is not None and not env_file.exists() assert not staging.exists() @@ -560,7 +570,10 @@ def responder(argv: list[str]) -> tuple[int, str, str]: provider, _rec = _make_provider(monkeypatch, responder) result = await provider.exec( - _make_handle(tmp_path, env={"TOKEN": "not-a-real-secret"}), + _make_handle( + tmp_path, + env={"TOKEN": "not-a-real-secret"}, # pragma: allowlist secret + ), "sleep 99", timeout_s=1, ) @@ -568,7 +581,7 @@ def responder(argv: list[str]) -> tuple[int, str, str]: assert result.return_code == apptainer_provider.SANDBOX_RUNTIME_RETURN_CODE assert result.error_type == "timeout" assert result.stdout is None - assert "not-a-real-secret" not in (result.stderr or "") + assert "not-a-real-secret" not in (result.stderr or "") # pragma: allowlist secret assert env_file is not None and not env_file.exists() @@ -752,6 +765,23 @@ async def test_close_success(fake_binary: str, monkeypatch: pytest.MonkeyPatch, assert _contains_seq(rec.calls[0]["argv"], [FAKE_BINARY, "instance", "stop", "nemo-gym-x"]) +async def test_close_removes_read_only_sandbox_tree( + fake_binary: str, monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + staging = tmp_path / "staging" + locked = staging / "runtime" + locked.mkdir(parents=True) + artifact = locked / "python" + artifact.write_text("runtime") + artifact.chmod(0o444) + locked.chmod(0o555) + provider, _rec = _make_provider(monkeypatch, lambda argv: (0, "", "")) + + await provider.close(_make_handle(staging)) + + assert not staging.exists() + + async def test_close_missing_instance_is_success( fake_binary: str, monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: @@ -807,7 +837,7 @@ async def test_close_staging_removal_failure_is_logged( def boom(path: Any, ignore_errors: bool = False) -> None: raise OSError("locked") - monkeypatch.setattr(apptainer_provider.shutil, "rmtree", boom) + monkeypatch.setattr(apptainer_provider, "_remove_writable_tree", boom) with caplog.at_level("WARNING"): await provider.close(_make_handle(staging)) # does not raise assert "failed to remove staging dir" in caplog.text diff --git a/tests/unit_tests/test_ecs_fargate_provider.py b/tests/unit_tests/test_ecs_fargate_provider.py index 5832c1b62f..cf89666040 100644 --- a/tests/unit_tests/test_ecs_fargate_provider.py +++ b/tests/unit_tests/test_ecs_fargate_provider.py @@ -419,6 +419,14 @@ def test_apply_spec_overrides_maps_resources_and_ttl(): assert _apply_spec_overrides(cfg, SandboxSpec(image="img")) is cfg +def test_apply_spec_overrides_uses_implicit_storage_for_requests_up_to_20_gib(): + cfg = engine.EcsFargateConfig(region="us-east-1") + + for requested in (1, 10, 20): + out = _apply_spec_overrides(cfg, SandboxSpec(image="img", resources={"disk_gib": requested})) + assert out.ephemeral_storage_gib is None + + def test_apply_spec_overrides_rejects_gpu(): from nemo_gym.sandbox.providers import SandboxCreateError diff --git a/tests/unit_tests/test_enroot_provider.py b/tests/unit_tests/test_enroot_provider.py index 8d4e93f674..ed96c73993 100644 --- a/tests/unit_tests/test_enroot_provider.py +++ b/tests/unit_tests/test_enroot_provider.py @@ -332,7 +332,14 @@ async def fake_ensure(image: str) -> Path: assert _contains_seq(start_argv, ["-e", "NVIDIA_VISIBLE_DEVICES=0"]) expected_init = f"{enroot_provider.DEFAULT_INIT_COMMAND} # {handle.sandbox_id}" assert start_argv[-4:] == [handle.sandbox_id, "sh", "-c", expected_init] - assert _contains_seq(start_argv, ["--rc", "/dev/null"]) + rc_index = start_argv.index("--rc") + bypass_rc_path = Path(start_argv[rc_index + 1]) + assert bypass_rc_path.parent == staging + assert bypass_rc_path.name == ".nemo-gym-bypass-entrypoint-rc" + assert bypass_rc_path.is_file() + assert bypass_rc_path.read_text() == enroot_provider.ENTRYPOINT_BYPASS_RC + assert 'exec "$@"' in bypass_rc_path.read_text() + assert bypass_rc_path.stat().st_mode & 0o777 == 0o400 # The generated container name is random; the create test above needs the `list` @@ -840,7 +847,7 @@ async def test_close_staging_removal_failure_is_logged( def boom(path: Any, ignore_errors: bool = False) -> None: raise OSError("locked") - monkeypatch.setattr(enroot_provider.shutil, "rmtree", boom) + monkeypatch.setattr(enroot_provider, "_remove_writable_tree", boom) with caplog.at_level("WARNING"): await provider.close(_make_handle(staging)) assert "failed to remove staging dir" in caplog.text diff --git a/tests/unit_tests/test_opensandbox_provider.py b/tests/unit_tests/test_opensandbox_provider.py index ecb1b22484..5df1eec30e 100644 --- a/tests/unit_tests/test_opensandbox_provider.py +++ b/tests/unit_tests/test_opensandbox_provider.py @@ -518,6 +518,41 @@ def test_connection_config_and_image_policy(fake_opensandbox_sdk: None) -> None: assert "headers" not in direct._connection_config().kwargs +def test_connection_domain_url_normalization_and_proxy_secret_boundary(fake_opensandbox_sdk: None) -> None: + provider = opensandbox_provider.OpenSandboxProvider( + connection={ + "domain": "http://sandbox.example:8080/", + "api_key": "key", # pragma: allowlist secret + "protocol": "http", + "use_server_proxy": True, + } + ) + config = provider._connection_config() + assert config.kwargs["domain"] == "http://sandbox.example:8080" + assert config.kwargs["protocol"] == "http" + assert config.kwargs["headers"] == { + "OPEN-SANDBOX-API-KEY": "key", # pragma: allowlist secret + } + + direct = opensandbox_provider.OpenSandboxProvider( + connection={ + "domain": "sandbox.example", + "api_key": "key", # pragma: allowlist secret + "use_server_proxy": False, + } + )._connection_config() + assert "headers" not in direct.kwargs + + with pytest.raises(ValueError, match="conflicts"): + opensandbox_provider.OpenSandboxProvider( + connection={"domain": "https://sandbox.example", "protocol": "http"} + )._connection_config() + with pytest.raises(ValueError, match="must not contain a path"): + opensandbox_provider.OpenSandboxProvider( + connection={"domain": "http://sandbox.example/api", "protocol": "http"} + )._connection_config() + + def test_connection_transport_backends(fake_opensandbox_sdk: None, monkeypatch: pytest.MonkeyPatch) -> None: # Default backend is httpx, with the configured keepalive expiry on the pool. provider = opensandbox_provider.OpenSandboxProvider() diff --git a/tests/unit_tests/test_openshell_provider.py b/tests/unit_tests/test_openshell_provider.py index e5885230ba..f5af35e084 100644 --- a/tests/unit_tests/test_openshell_provider.py +++ b/tests/unit_tests/test_openshell_provider.py @@ -32,13 +32,26 @@ pytest.importorskip("openshell", reason="openshell optional dependency is not installed") import grpc # noqa: E402 (grpcio ships with the openshell SDK) -from openshell import SandboxError, SandboxRef, SandboxStatusRef # noqa: E402 +from openshell import SandboxClient, SandboxError, SandboxRef # noqa: E402 from openshell._proto import openshell_pb2, sandbox_pb2 # noqa: E402 + +try: # OpenShell 0.0.92+ nests lifecycle state in SandboxStatusRef. + from openshell import SandboxStatusRef # type: ignore[attr-defined] # noqa: E402 +except ImportError: # OpenShell 0.0.36 exposes phase directly on SandboxRef. + SandboxStatusRef = None # type: ignore[assignment,misc] + + +WORKSPACE_SDK = "workspace" in inspect.signature(SandboxClient.create).parameters +RESOURCE_REQUIREMENTS_SDK = "resource_requirements" in openshell_pb2.SandboxSpec.DESCRIPTOR.fields_by_name +DRIVER_CONFIG_SDK = "driver_config" in openshell_pb2.SandboxTemplate.DESCRIPTOR.fields_by_name + from nemo_gym.sandbox.providers.openshell import provider as openshell_provider # noqa: E402 from nemo_gym.sandbox.providers.openshell.provider import ( # noqa: E402 + MAX_SANDBOX_NAME_LENGTH, SANDBOX_LABEL, SANDBOX_NAME_PREFIX, + SANDBOX_NAME_RANDOM_HEX, SANDBOX_RUNTIME_RETURN_CODE, OpenShellConnectionConfig, OpenShellCreateConfig, @@ -49,6 +62,7 @@ OpenShellProbeConfig, OpenShellProvider, OpenShellProviderOptions, + _new_sandbox_name, _OpenShellSandbox, ) @@ -71,7 +85,9 @@ def make_ref( name: str = "nemo-gym-test", workspace: str = "default", ) -> SandboxRef: - """A real SDK SandboxRef, so tests exercise the SDK's actual shape (nested status/phase).""" + """A real SDK SandboxRef, exercising the installed SDK's lifecycle-result shape.""" + if SandboxStatusRef is None: + return SandboxRef(id=sandbox_id, name=name, namespace=workspace, phase=phase) return SandboxRef( id=sandbox_id, name=name, @@ -157,6 +173,22 @@ def close(self) -> None: self.close_calls += 1 +class LegacyFakeClient(FakeClient): + """OpenShell's pre-workspace lifecycle API used by local gateway 0.0.36.""" + + def create(self, *, spec: Any = None) -> Any: + self.create_calls.append({"spec": spec}) + return self._next(self.create_results) + + def get(self, sandbox_name: str) -> Any: + self.get_calls.append({"name": sandbox_name}) + return self._next(self.get_results) + + def delete(self, sandbox_name: str) -> Any: + self.delete_calls.append({"name": sandbox_name}) + return self._next(self.delete_results) + + @pytest.fixture(autouse=True) def clear_shared_clients() -> Any: openshell_provider._SHARED_CLIENTS.clear() @@ -230,23 +262,80 @@ def fake_import( openshell_provider._require_openshell() +def test_generated_name_respects_openshell_limit(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(openshell_provider.uuid, "uuid4", lambda: SimpleNamespace(hex="a" * 32)) + name = _new_sandbox_name() + assert name == SANDBOX_NAME_PREFIX + "a" * SANDBOX_NAME_RANDOM_HEX + assert len(name) == MAX_SANDBOX_NAME_LENGTH + + def test_provider_call_shapes_bind_against_installed_sdk() -> None: """Bind the provider's exact SDK call shapes against the installed openshell signatures. This is the drift guard: an SDK release that changes a lifecycle signature (like 0.0.92 adding required ``workspace``) fails here even though the unit tests run against fakes. """ - from openshell import SandboxClient - - inspect.signature(SandboxClient.create).bind(None, workspace="w", spec=None, name="n", labels={}) - inspect.signature(SandboxClient.get).bind(None, "n", workspace="w") - inspect.signature(SandboxClient.delete).bind(None, "n", workspace="w") + if WORKSPACE_SDK: + inspect.signature(SandboxClient.create).bind(None, workspace="w", spec=None, name="n", labels={}) + inspect.signature(SandboxClient.get).bind(None, "n", workspace="w") + inspect.signature(SandboxClient.delete).bind(None, "n", workspace="w") + else: + inspect.signature(SandboxClient.create).bind(None, spec=None) + inspect.signature(SandboxClient.get).bind(None, "n") + inspect.signature(SandboxClient.delete).bind(None, "n") inspect.signature(SandboxClient.exec).bind( None, "sid", ["/bin/sh", "-c", "true"], workdir=None, env=None, stdin=None, timeout_seconds=None ) inspect.signature(SandboxClient.close).bind(None) +@pytest.mark.asyncio +async def test_legacy_lifecycle_api_omits_unsupported_workspace_name_and_labels( + monkeypatch: pytest.MonkeyPatch, +) -> None: + client = LegacyFakeClient() + client.get_results = [ + make_ref(openshell_pb2.SANDBOX_PHASE_READY), + FakeRpcError(grpc.StatusCode.NOT_FOUND), + ] + monkeypatch.setattr(openshell_provider, "_build_client", lambda _connection: client) + provider = OpenShellProvider( + create={"poll_interval_s": 0.01}, + probe={"command": None}, + operations={"poll_interval_s": 0.01}, + ) + + handle = await provider.create(SandboxSpec(metadata={"purpose": "test"})) + await provider.close(handle) + + assert client.create_calls == [{"spec": client.create_calls[0]["spec"]}] + assert client.get_calls == [{"name": "nemo-gym-test"}, {"name": "nemo-gym-test"}] + assert client.delete_calls == [{"name": "nemo-gym-test"}] + + +@pytest.mark.asyncio +async def test_legacy_lifecycle_api_rejects_workspace_and_does_not_retry_ambiguous_create( + monkeypatch: pytest.MonkeyPatch, +) -> None: + client = LegacyFakeClient() + client.create_results = [FakeRpcError(grpc.StatusCode.UNAVAILABLE)] + monkeypatch.setattr(openshell_provider, "_build_client", lambda _connection: client) + provider = OpenShellProvider( + connection={"workspace": "team-a"}, + create={"retries": 3, "retry_delay_s": 0}, + probe={"command": None}, + ) + + with pytest.raises(OpenShellCreateError, match="non-default workspaces"): + await provider.create(SandboxSpec()) + assert client.create_calls == [] + + provider._connection = openshell_provider.OpenShellConnectionConfig() + with pytest.raises(OpenShellCreateError, match="CreateSandbox failed"): + await provider.create(SandboxSpec()) + assert len(client.create_calls) == 1 + + def test_constructor_builds_real_client_and_config_coercion() -> None: provider = OpenShellProvider(connection={"endpoint": "localhost:1", "request_timeout_s": 5}) try: @@ -319,7 +408,7 @@ async def test_create_success_maps_spec(make_provider, fake_client: FakeClient) env={"FOO": "bar"}, metadata={"task": "demo", SANDBOX_LABEL: "user-override"}, workdir="/workspace", - resources={"gpu": 2}, + resources={"gpu": 2 if RESOURCE_REQUIREMENTS_SDK else 1}, provider_options={"providers": ["nvidia"]}, ) handle = await provider.create(spec) @@ -327,13 +416,17 @@ async def test_create_success_maps_spec(make_provider, fake_client: FakeClient) call = fake_client.create_calls[0] assert call["workspace"] == "default" assert call["name"].startswith(SANDBOX_NAME_PREFIX) + assert len(call["name"]) <= MAX_SANDBOX_NAME_LENGTH # The marker label is applied last, so user metadata cannot clobber it. assert call["labels"] == {"task": "demo", SANDBOX_LABEL: "1"} pb_spec = call["spec"] assert pb_spec.template.image == "python:3.12-slim" assert dict(pb_spec.environment) == {"FOO": "bar"} assert list(pb_spec.providers) == ["nvidia"] - assert pb_spec.resource_requirements.gpu.count == 2 + if RESOURCE_REQUIREMENTS_SDK: + assert pb_spec.resource_requirements.gpu.count == 2 + else: + assert pb_spec.gpu is True assert handle.provider_name == "openshell" assert handle.sandbox_id == "sbx-1" @@ -360,6 +453,11 @@ async def test_create_without_image_uses_gateway_default(make_provider, fake_cli async def test_create_template_resources_and_driver_config(make_provider, fake_client: FakeClient) -> None: provider = make_provider() + if not DRIVER_CONFIG_SDK: + with pytest.raises(OpenShellCreateError, match="driver_config.*workspace-aware"): + await provider.create(SandboxSpec(provider_options={"driver_config": {"runtime": "kata"}})) + assert not fake_client.create_calls + return await provider.create( SandboxSpec( provider_options={ diff --git a/tests/unit_tests/test_responses_api_model_streaming.py b/tests/unit_tests/test_responses_api_model_streaming.py index 78d7703f11..9b3d44f580 100644 --- a/tests/unit_tests/test_responses_api_model_streaming.py +++ b/tests/unit_tests/test_responses_api_model_streaming.py @@ -317,6 +317,34 @@ def test_rewrites_namespaced_calls_in_input_history(self) -> None: assert "namespace" not in call NeMoGymResponseCreateParamsNonStreaming.model_validate(cleaned) + def test_normalizes_codex_replayed_reasoning_and_assistant_output(self) -> None: + body = { + "stream": True, + "input": [ + {"type": "reasoning", "summary": []}, + { + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "I inspected the files."}], + }, + {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Continue."}]}, + ], + } + + cleaned, _ = sanitize_streaming_responses_body(body) + cleaned_again, _ = sanitize_streaming_responses_body(body) + + reasoning, assistant, _user = cleaned["input"] + assert reasoning["id"].startswith("rs_") + assert assistant["id"].startswith("msg_") + assert assistant["content"][0]["annotations"] == [] + assert reasoning["id"] == cleaned_again["input"][0]["id"] + assert assistant["id"] == cleaned_again["input"][1]["id"] + assert "id" not in body["input"][0] + assert "annotations" not in body["input"][1]["content"][0] + NeMoGymResponseCreateParamsNonStreaming.model_validate(cleaned) + def test_drops_unsupported_input_items(self) -> None: # Codex's code_mode interleaves an `additional_tools` carrier item into the input history; # the Gym input union has no representation for it, so it is dropped item-by-item. diff --git a/tests/unit_tests/test_server_utils.py b/tests/unit_tests/test_server_utils.py index 35780418fa..dc6c2e4c79 100644 --- a/tests/unit_tests/test_server_utils.py +++ b/tests/unit_tests/test_server_utils.py @@ -287,10 +287,27 @@ def test_keepalive_socket_factory_skips_missing_platform_sockopts(self, monkeypa def test_GlobalAIOHTTPAsyncClientConfig_keepalive_defaults(self) -> None: cfg = GlobalAIOHTTPAsyncClientConfig() + assert cfg.global_aiohttp_trust_env is False assert cfg.global_aiohttp_tcp_keepalive_idle_seconds == 60 assert cfg.global_aiohttp_tcp_keepalive_interval_seconds == 10 assert cfg.global_aiohttp_tcp_keepalive_probes == 3 + def test_set_global_aiohttp_client_forwards_trust_env(self, monkeypatch: MonkeyPatch) -> None: + session = MagicMock() + session_ctor = MagicMock(return_value=session) + monkeypatch.setattr(nemo_gym.server_utils, "_GLOBAL_AIOHTTP_CLIENT", None) + monkeypatch.setattr(nemo_gym.server_utils, "ClientSession", session_ctor) + monkeypatch.setattr(nemo_gym.server_utils, "TCPConnector", MagicMock()) + monkeypatch.setattr(nemo_gym.server_utils, "DummyCookieJar", MagicMock()) + monkeypatch.setattr(nemo_gym.server_utils, "get_nemo_gym_fastapi_num_workers", lambda: 1) + + result = nemo_gym.server_utils.set_global_aiohttp_client( + GlobalAIOHTTPAsyncClientConfig(global_aiohttp_trust_env=True) + ) + + assert result is session + assert session_ctor.call_args.kwargs["trust_env"] is True + def test_keepalive_socket_factory_uses_configured_values(self, monkeypatch: MonkeyPatch) -> None: mock_sock = MagicMock() socket_ctor_mock = MagicMock(return_value=mock_sock) diff --git a/uv.lock b/uv.lock index c3f6d21401..d7a8f9495b 100644 --- a/uv.lock +++ b/uv.lock @@ -3340,7 +3340,7 @@ wheels = [ [[package]] name = "openshell" -version = "0.0.92" +version = "0.0.103" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cloudpickle" }, @@ -3349,9 +3349,9 @@ dependencies = [ { name = "protobuf" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/cc/e3/be7c389ae1b12312b302da847b82734475d2689188d2a19645ac8c1223c8/openshell-0.0.92-py3-none-macosx_13_0_arm64.whl", hash = "sha256:3d673a98e66520eabd4b06fbff126d55718fb92c868fbfeb994adb34970a4796", size = 8470512, upload-time = "2026-07-27T15:32:55.574Z" }, - { url = "https://files.pythonhosted.org/packages/5f/ca/604e9a4b9701f11c51ca4129699ca3a548aad734254431bdfcda48f17e15/openshell-0.0.92-py3-none-manylinux_2_39_aarch64.whl", hash = "sha256:036fa76cd89dd49375405edde55ee35f72c0a3bd22f990942a336f4ededf15ef", size = 8532736, upload-time = "2026-07-27T15:33:16.548Z" }, - { url = "https://files.pythonhosted.org/packages/4c/fc/24a15a862925d19b1d318be2796aa996233dc30b1b9e3c4862ec1039c6f4/openshell-0.0.92-py3-none-manylinux_2_39_x86_64.whl", hash = "sha256:8a20fea53ffff0c6127ca3f6841fb4b268ced1ccb03917cdbf278e419c1e9fdf", size = 9018946, upload-time = "2026-07-27T15:33:37.985Z" }, + { url = "https://files.pythonhosted.org/packages/6b/67/2ce32a11297ba68cdb2b26b276f5ccd1e97e869e90ce4e51925b18aa2b80/openshell-0.0.103-py3-none-macosx_13_0_arm64.whl", hash = "sha256:d1c467a0c6925385c415fb8856a3c9ec627b7e51b711c6b33cfea51d718d965d", size = 8493933, upload-time = "2026-08-11T15:07:59.193Z" }, + { url = "https://files.pythonhosted.org/packages/31/42/4d25a11e8ff20690b0f63c254d8cc870faf793722f89e9ca8617bda19577/openshell-0.0.103-py3-none-manylinux_2_39_aarch64.whl", hash = "sha256:b623addf09fea656f66306201877c5259fd7a20aaa9a15a704ee97a669535cd6", size = 8562102, upload-time = "2026-08-11T15:08:17.257Z" }, + { url = "https://files.pythonhosted.org/packages/b5/4b/33bce6de7993704f1acc00e5b6d476f292b3ae724fbf65121aadcf61724b/openshell-0.0.103-py3-none-manylinux_2_39_x86_64.whl", hash = "sha256:b0e961425bcdfed930967efd89f806e24142436c34a30234d4dc888f92fec8d0", size = 9055035, upload-time = "2026-08-11T15:08:41.267Z" }, ] [[package]]