feat: make Legal Agent Bench agent harness and sandbox-backends configurable

benchmarks/legal_agent_bench/config.yaml

@@ -2,20 +2,20 @@
 # SPDX-License-Identifier: Apache-2.0
 
 config_paths:
-  - resources_servers/legal_agent_bench/configs/legal_agent_bench.yaml
+  - responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_native.yaml
 
 legal_agent_bench_benchmark_resources_server:
   _inherit_from: legal_agent_bench
 
-legal_agent_bench_benchmark_harbor_agent:
-  _inherit_from: legal_agent_bench_harbor_agent
+legal_agent_bench_benchmark_native_agent:
+  _inherit_from: legal_agent_bench_native_agent
   responses_api_agents:
-    harbor_agent:
-      harbor_datasets:
-        legal_agent_bench:
-          local_dataset_path: ${legal_agent_bench_benchmark_resources_server.resources_servers.legal_agent_bench.harbor_tasks_dir}
-      harbor_agent_kwargs:
-        skills_dir: ${legal_agent_bench_benchmark_resources_server.resources_servers.legal_agent_bench.harness_skills_dir}
+    legal_agent_bench_agent:
+      resources_server:
+        type: resources_servers
+        name: legal_agent_bench_benchmark_resources_server
+      runtime_tasks_dir: ${legal_agent_bench_benchmark_resources_server.resources_servers.legal_agent_bench.harbor_tasks_dir}
+      skills_dir: ${legal_agent_bench_benchmark_resources_server.resources_servers.legal_agent_bench.harness_skills_dir}
       datasets:
         - name: legal_agent_bench
           type: benchmark

benchmarks/legal_agent_bench/config_claude_code.yaml

@@ -0,0 +1,25 @@
+config_paths:
+  - responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_claude_code.yaml
+
+legal_agent_bench_benchmark_claude_code_resources_server:
+  _inherit_from: legal_agent_bench
+  resources_servers:
+    legal_agent_bench:
+      description: Configurable-agent integration of Legal Agent Benchmark (LAB) using Claude Code
+
+legal_agent_bench_benchmark_claude_code_agent:
+  _inherit_from: legal_agent_bench_claude_code_agent
+  responses_api_agents:
+    legal_agent_bench_agent:
+      resources_server:
+        type: resources_servers
+        name: legal_agent_bench_benchmark_claude_code_resources_server
+      runtime_tasks_dir: ${legal_agent_bench_benchmark_claude_code_resources_server.resources_servers.legal_agent_bench.harbor_tasks_dir}
+      skills_dir: ${legal_agent_bench_benchmark_claude_code_resources_server.resources_servers.legal_agent_bench.harness_skills_dir}
+      datasets:
+        - name: legal_agent_bench
+          type: benchmark
+          jsonl_fpath: benchmarks/legal_agent_bench/data/legal_agent_bench_benchmark.jsonl
+          prompt_config: null
+          prepare_script: benchmarks/legal_agent_bench/prepare.py
+          num_repeats: 1

benchmarks/legal_agent_bench/config_codex.yaml

@@ -0,0 +1,25 @@
+config_paths:
+  - responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_codex.yaml
+
+legal_agent_bench_benchmark_codex_resources_server:
+  _inherit_from: legal_agent_bench
+  resources_servers:
+    legal_agent_bench:
+      description: Configurable-agent integration of Legal Agent Benchmark (LAB) using Codex
+
+legal_agent_bench_benchmark_codex_agent:
+  _inherit_from: legal_agent_bench_codex_agent
+  responses_api_agents:
+    legal_agent_bench_agent:
+      resources_server:
+        type: resources_servers
+        name: legal_agent_bench_benchmark_codex_resources_server
+      runtime_tasks_dir: ${legal_agent_bench_benchmark_codex_resources_server.resources_servers.legal_agent_bench.harbor_tasks_dir}
+      skills_dir: ${legal_agent_bench_benchmark_codex_resources_server.resources_servers.legal_agent_bench.harness_skills_dir}
+      datasets:
+        - name: legal_agent_bench
+          type: benchmark
+          jsonl_fpath: benchmarks/legal_agent_bench/data/legal_agent_bench_benchmark.jsonl
+          prompt_config: null
+          prepare_script: benchmarks/legal_agent_bench/prepare.py
+          num_repeats: 1

benchmarks/legal_agent_bench/config_harbor.yaml

@@ -0,0 +1,25 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+config_paths:
+  - resources_servers/legal_agent_bench/configs/legal_agent_bench.yaml
+
+legal_agent_bench_benchmark_harbor_resources_server:
+  _inherit_from: legal_agent_bench
+
+legal_agent_bench_benchmark_harbor_agent:
+  _inherit_from: legal_agent_bench_harbor_agent
+  responses_api_agents:
+    harbor_agent:
+      harbor_datasets:
+        legal_agent_bench:
+          local_dataset_path: ${legal_agent_bench_benchmark_harbor_resources_server.resources_servers.legal_agent_bench.harbor_tasks_dir}
+      harbor_agent_kwargs:
+        skills_dir: ${legal_agent_bench_benchmark_harbor_resources_server.resources_servers.legal_agent_bench.harness_skills_dir}
+      datasets:
+        - name: legal_agent_bench
+          type: benchmark
+          jsonl_fpath: benchmarks/legal_agent_bench/data/legal_agent_bench_benchmark.jsonl
+          prompt_config: null
+          prepare_script: benchmarks/legal_agent_bench/prepare.py
+          num_repeats: 1

benchmarks/legal_agent_bench/config_hermes.yaml

@@ -0,0 +1,25 @@
+config_paths:
+  - responses_api_agents/legal_agent_bench_agent/configs/legal_agent_bench_hermes.yaml
+
+legal_agent_bench_benchmark_hermes_resources_server:
+  _inherit_from: legal_agent_bench
+  resources_servers:
+    legal_agent_bench:
+      description: Configurable-agent integration of Legal Agent Benchmark (LAB) using Hermes
+
+legal_agent_bench_benchmark_hermes_agent:
+  _inherit_from: legal_agent_bench_hermes_agent
+  responses_api_agents:
+    legal_agent_bench_agent:
+      resources_server:
+        type: resources_servers
+        name: legal_agent_bench_benchmark_hermes_resources_server
+      runtime_tasks_dir: ${legal_agent_bench_benchmark_hermes_resources_server.resources_servers.legal_agent_bench.harbor_tasks_dir}
+      skills_dir: ${legal_agent_bench_benchmark_hermes_resources_server.resources_servers.legal_agent_bench.harness_skills_dir}
+      datasets:
+        - name: legal_agent_bench
+          type: benchmark
+          jsonl_fpath: benchmarks/legal_agent_bench/data/legal_agent_bench_benchmark.jsonl
+          prompt_config: null
+          prepare_script: benchmarks/legal_agent_bench/prepare.py
+          num_repeats: 1

benchmarks/legal_agent_bench/prepare.py

@@ -15,7 +15,6 @@
 BENCHMARK_DIR = Path(__file__).resolve().parent
 DATA_DIR = BENCHMARK_DIR / "data"
 OUTPUT_FPATH = DATA_DIR / "legal_agent_bench_benchmark.jsonl"
-BENCHMARK_AGENT_NAME = "legal_agent_bench_benchmark_harbor_agent"
 
 
 def _render_benchmark_index(source_index: Path) -> str:
@@ -31,10 +30,7 @@ def _render_benchmark_index(source_index: Path) -> str:
             if not isinstance(row, dict):
                 raise ValueError(f"LAB task index line {line_number} must contain a JSON object")
 
-            row["agent_ref"] = {
-                "name": BENCHMARK_AGENT_NAME,
-                "type": "responses_api_agents",
-            }
+            row.pop("agent_ref", None)
             rendered_rows.append(json.dumps(row, ensure_ascii=False, sort_keys=True) + "\n")
 
     if len(rendered_rows) != EXPECTED_TASK_COUNT:

benchmarks/legal_agent_bench/tests/test_prepare.py

@@ -8,11 +8,13 @@
 from pathlib import Path
 
 import pytest
-from omegaconf import OmegaConf
+from omegaconf import DictConfig, OmegaConf
 
 from benchmarks.legal_agent_bench import prepare as benchmark_prepare
 from nemo_gym.benchmarks import BenchmarkConfig
+from nemo_gym.config_types import ResponsesAPIAgentServerInstanceConfig
 from nemo_gym.global_config import GlobalConfigDictParser, GlobalConfigDictParserConfig
+from nemo_gym.train_data_utils import TrainDataProcessor
 from resources_servers.legal_agent_bench.prepare import EXPECTED_TASK_COUNT, INDEX_FILENAME
 
 
@@ -28,10 +30,6 @@ def _write_task_index(parent: Path, count: int) -> tuple[Path, list[str]]:
     for task_name in task_names:
         rows.append(
             {
-                "agent_ref": {
-                    "name": "legal_agent_bench_harbor_agent",
-                    "type": "responses_api_agents",
-                },
                 "instance_id": f"legal_agent_bench::{task_name}",
                 "responses_create_params": {
                     "input": [],
@@ -73,14 +71,7 @@ def test_prepare_writes_deterministic_complete_benchmark_index(monkeypatch, tmp_
     rows = [json.loads(line) for line in output_path.read_text(encoding="utf-8").splitlines()]
     assert len(rows) == EXPECTED_TASK_COUNT
     assert [row["instance_id"].split("::", 1)[1] for row in rows] == task_names
-    assert all(
-        row["agent_ref"]
-        == {
-            "name": benchmark_prepare.BENCHMARK_AGENT_NAME,
-            "type": "responses_api_agents",
-        }
-        for row in rows
-    )
+    assert all("agent_ref" not in row for row in rows)
 
 
 def test_wrong_row_count_does_not_replace_existing_output(monkeypatch, tmp_path) -> None:
@@ -129,7 +120,7 @@ def test_benchmark_config_is_isolated_and_resolves_shared_cache_paths() -> None:
     benchmark = BenchmarkConfig.from_config_path(CONFIG_FPATH, strict=False)
     assert benchmark is not None
     assert benchmark.name == "legal_agent_bench"
-    assert benchmark.agent_name == "legal_agent_bench_benchmark_harbor_agent"
+    assert benchmark.agent_name == "legal_agent_bench_benchmark_native_agent"
     assert benchmark.num_repeats == 1
     assert benchmark.dataset.prompt_config is None
     assert benchmark.dataset.jsonl_fpath == Path("benchmarks/legal_agent_bench/data/legal_agent_bench_benchmark.jsonl")
@@ -141,12 +132,170 @@ def test_benchmark_config_is_isolated_and_resolves_shared_cache_paths() -> None:
     )
     resolved = GlobalConfigDictParser().parse_no_environment(initial_global_config_dict=initial_config)
     assert "legal_agent_bench" not in resolved
-    assert "legal_agent_bench_harbor_agent" not in resolved
+    assert "legal_agent_bench_native_agent" not in resolved
 
     resource = resolved.legal_agent_bench_benchmark_resources_server.resources_servers.legal_agent_bench
+    agent = resolved.legal_agent_bench_benchmark_native_agent.responses_api_agents.legal_agent_bench_agent
+    assert agent.agent_server_module == "responses_api_agents.legal_agent_bench_native_agent.app"
+    assert agent.runtime_tasks_dir == resource.harbor_tasks_dir
+    assert agent.skills_dir == resource.harness_skills_dir
+    assert agent.runtime_builder_provider_options == {}
+    assert agent.agent_sandbox_provider_options == {}
+    assert agent.verifier_sandbox_provider_options == {}
+    assert agent.agent_kwargs.max_turns == 60
+    assert len(agent.datasets) == 1
+    assert agent.datasets[0].type == "benchmark"
+
+
+def test_harbor_compatibility_variant_resolves() -> None:
+    config_path = BENCHMARK_DIR / "config_harbor.yaml"
+    benchmark = BenchmarkConfig.from_config_path(config_path, strict=False)
+    assert benchmark is not None
+    assert benchmark.name == "legal_agent_bench"
+    assert benchmark.agent_name == "legal_agent_bench_benchmark_harbor_agent"
+
+    initial_config = OmegaConf.merge(
+        OmegaConf.load(config_path),
+        GlobalConfigDictParserConfig.NO_MODEL_GLOBAL_CONFIG_DICT,
+    )
+    resolved = GlobalConfigDictParser().parse_no_environment(initial_global_config_dict=initial_config)
+    resource = resolved.legal_agent_bench_benchmark_harbor_resources_server.resources_servers.legal_agent_bench
     agent = resolved.legal_agent_bench_benchmark_harbor_agent.responses_api_agents.harbor_agent
     assert agent.harbor_datasets.legal_agent_bench.local_dataset_path == resource.harbor_tasks_dir
     assert agent.harbor_agent_kwargs.skills_dir == resource.harness_skills_dir
     assert agent.harbor_agent_kwargs.max_turns == 60
-    assert len(agent.datasets) == 1
     assert agent.datasets[0].type == "benchmark"
+
+
+@pytest.mark.parametrize(
+    ("filename", "expected_agent", "expected_module"),
+    [
+        ("config_hermes.yaml", "legal_agent_bench_benchmark_hermes_agent", "responses_api_agents.hermes_agent.app"),
+        (
+            "config_claude_code.yaml",
+            "legal_agent_bench_benchmark_claude_code_agent",
+            "responses_api_agents.claude_code_agent.app",
+        ),
+        ("config_codex.yaml", "legal_agent_bench_benchmark_codex_agent", "responses_api_agents.codex_agent.app"),
+    ],
+)
+def test_configurable_benchmark_variants_resolve(filename, expected_agent, expected_module) -> None:
+    config_path = BENCHMARK_DIR / filename
+    benchmark = BenchmarkConfig.from_config_path(config_path, strict=False)
+    assert benchmark is not None
+    assert benchmark.name == "legal_agent_bench"
+    assert benchmark.agent_name == expected_agent
+    assert benchmark.dataset.jsonl_fpath == Path("benchmarks/legal_agent_bench/data/legal_agent_bench_benchmark.jsonl")
+
+    initial_config = OmegaConf.merge(
+        OmegaConf.load(config_path),
+        GlobalConfigDictParserConfig.NO_MODEL_GLOBAL_CONFIG_DICT,
+    )
+    resolved = GlobalConfigDictParser().parse_no_environment(initial_global_config_dict=initial_config)
+    agent_names = [
+        name for name, value in resolved.items() if isinstance(value, DictConfig) and "responses_api_agents" in value
+    ]
+    resource_names = [
+        name for name, value in resolved.items() if isinstance(value, DictConfig) and "resources_servers" in value
+    ]
+    assert agent_names == [expected_agent]
+    assert resource_names == [f"{expected_agent.removesuffix('_agent')}_resources_server"]
+    agent = resolved[expected_agent].responses_api_agents.legal_agent_bench_agent
+    assert agent.agent_server_module == expected_module
+    if expected_module == "responses_api_agents.claude_code_agent.app":
+        assert agent.agent_kwargs.claude_code_version == "2.1.211"
+    elif expected_module == "responses_api_agents.codex_agent.app":
+        assert agent.agent_kwargs.codex_version == "0.144.4"
+        assert agent.agent_kwargs.cwd == "/sandbox/nemo-gym-legal-agent-bench/workspace/output"
+    assert agent.datasets[0].type == "benchmark"
+    assert agent.runtime_tasks_dir.endswith("data/runtime/harbor_tasks/legal_agent_bench")
+    assert agent.runtime_builder_provider_options == {}
+    assert agent.agent_sandbox_provider_options == {}
+    assert agent.verifier_sandbox_provider_options == {}
+
+
+@pytest.mark.parametrize(
+    ("filename", "expected_agent"),
+    [
+        ("config.yaml", "legal_agent_bench_benchmark_native_agent"),
+        ("config_hermes.yaml", "legal_agent_bench_benchmark_hermes_agent"),
+        ("config_claude_code.yaml", "legal_agent_bench_benchmark_claude_code_agent"),
+        ("config_codex.yaml", "legal_agent_bench_benchmark_codex_agent"),
+    ],
+)
+def test_configurable_variants_decode_phase_provider_options_from_environment(
+    monkeypatch, filename, expected_agent
+) -> None:
+    monkeypatch.setenv(
+        "NEMO_GYM_LAB_RUNTIME_BUILDER_PROVIDER_OPTIONS",
+        "{policy: /tmp/lab-builder-policy.yaml}",
+    )
+    monkeypatch.setenv(
+        "NEMO_GYM_LAB_AGENT_SANDBOX_PROVIDER_OPTIONS",
+        "{policy: /tmp/lab-agent-policy.yaml}",
+    )
+    monkeypatch.setenv(
+        "NEMO_GYM_LAB_VERIFIER_SANDBOX_PROVIDER_OPTIONS",
+        "{policy: /tmp/lab-verifier-policy.yaml}",
+    )
+    initial_config = OmegaConf.merge(
+        OmegaConf.load(BENCHMARK_DIR / filename),
+        GlobalConfigDictParserConfig.NO_MODEL_GLOBAL_CONFIG_DICT,
+    )
+
+    resolved = GlobalConfigDictParser().parse_no_environment(initial_global_config_dict=initial_config)
+    agent = resolved[expected_agent].responses_api_agents.legal_agent_bench_agent
+
+    assert agent.runtime_builder_provider_options == {"policy": "/tmp/lab-builder-policy.yaml"}
+    assert agent.agent_sandbox_provider_options == {"policy": "/tmp/lab-agent-policy.yaml"}
+    assert agent.verifier_sandbox_provider_options == {"policy": "/tmp/lab-verifier-policy.yaml"}
+
+
+@pytest.mark.parametrize(
+    "agent_name",
+    [
+        "legal_agent_bench_benchmark_native_agent",
+        "legal_agent_bench_benchmark_harbor_agent",
+        "legal_agent_bench_benchmark_hermes_agent",
+        "legal_agent_bench_benchmark_claude_code_agent",
+        "legal_agent_bench_benchmark_codex_agent",
+    ],
+)
+def test_benchmark_collation_stamps_selected_agent_without_changing_source(tmp_path, agent_name) -> None:
+    source = tmp_path / "legal_agent_bench.jsonl"
+    neutral_row = {
+        "instance_id": "legal_agent_bench::corporate__task",
+        "responses_create_params": {"input": []},
+    }
+    source.write_text(json.dumps(neutral_row) + "\n", encoding="utf-8")
+    agent_config = {
+        "responses_api_agents": {
+            "agent": {
+                "host": "127.0.0.1",
+                "port": 12345,
+                "entrypoint": "app.py",
+                "resources_server": {"type": "resources_servers", "name": "legal_agent_bench"},
+                "model_server": {"type": "responses_api_models", "name": "policy_model"},
+                "datasets": [
+                    {
+                        "name": "legal_agent_bench",
+                        "type": "benchmark",
+                        "jsonl_fpath": str(source),
+                        "prepare_script": "benchmarks/legal_agent_bench/prepare.py",
+                        "num_repeats": 1,
+                    }
+                ],
+            }
+        }
+    }
+    instance = ResponsesAPIAgentServerInstanceConfig(
+        name=agent_name,
+        server_type_config_dict=DictConfig(agent_config),
+        responses_api_agents=agent_config["responses_api_agents"],
+    )
+
+    prepared = TrainDataProcessor()._collate_samples_single_type("benchmark", [instance])[0]
+    collated_row = json.loads(prepared.read_text(encoding="utf-8"))
+
+    assert json.loads(source.read_text(encoding="utf-8")) == neutral_row
+    assert collated_row["agent_ref"] == {"type": "responses_api_agents", "name": agent_name}