From 5f1426661c143bdd8cc6052f36822f35ab7e9bd0 Mon Sep 17 00:00:00 2001 From: Jeff Peng Date: Thu, 30 Jul 2026 22:55:43 +0800 Subject: [PATCH 01/10] feat(osworld): add OpenSandbox pool backend Signed-off-by: Jeff Peng --- .../osworld/configs/osworld_opensandbox.yaml | 35 +++ benchmarks/osworld/prepare.py | 42 ++- benchmarks/osworld/tests/test_prepare.py | 96 +++++++ benchmarks/osworld/tests/test_run_scripts.py | 8 + nemo_gym/sandbox/api.py | 41 ++- .../sandbox/providers/opensandbox/provider.py | 264 +++++++++++++++++- responses_api_agents/osworld_agent/client.py | 9 +- .../osworld_agent/configs/osworld_agent.yaml | 4 + .../osworld_agent/local_forwarder.py | 197 +++++++++++++ .../osworld_agent/pyproject.toml | 4 +- .../osworld_agent/sandbox_provider.py | 134 +++++++-- .../osworld_agent/tests/test_client.py | 33 +++ .../tests/test_sandbox_provider.py | 130 ++++++++- tests/unit_tests/test_opensandbox_provider.py | 212 ++++++++++++++ tests/unit_tests/test_sandbox.py | 22 ++ 15 files changed, 1191 insertions(+), 40 deletions(-) create mode 100644 benchmarks/osworld/configs/osworld_opensandbox.yaml create mode 100644 responses_api_agents/osworld_agent/local_forwarder.py diff --git a/benchmarks/osworld/configs/osworld_opensandbox.yaml b/benchmarks/osworld/configs/osworld_opensandbox.yaml new file mode 100644 index 0000000000..6d2e95590f --- /dev/null +++ b/benchmarks/osworld/configs/osworld_opensandbox.yaml @@ -0,0 +1,35 @@ +# Cell-2 OpenSandbox lifecycle provider for the server-side OSWorld KVM Pool. +# Credentials stay in environment variables; generated env.yaml files never +# contain the API key. + +osworld_opensandbox: + default_metadata: + sandbox-api: opensandbox-sdk + workload: osworld + opensandbox: + connection: + domain: ${oc.env:OPENSANDBOX_BASE_URL} + api_key: ${oc.env:OPENSANDBOX_API_KEY} + protocol: http + request_timeout_s: 300 + # External controllers cannot route Cell-2's 100.100/16 Pod CIDR. + # The OSWorld adapter supplies local host:port forwarders for these + # path-based gateway endpoints, including Chrome CDP WebSockets. + use_server_proxy: true + create: + request_timeout_s: 1200 + timeout_s: 1200 + skip_health_check: true + retries: 3 + retry_delay_s: 5.0 + retry_max_delay_s: 60.0 + probe: + # The KVM Pool's pod-level execd probe is not guest readiness. The + # adapter waits for the OSWorld guest screenshot endpoint instead. + command: null + operations: + retries: 5 + retry_delay_s: 1.0 + retry_max_delay_s: 45.0 + command_retries: 0 + close_timeout_s: 60.0 diff --git a/benchmarks/osworld/prepare.py b/benchmarks/osworld/prepare.py index dcac2a79ec..865b7d2cec 100644 --- a/benchmarks/osworld/prepare.py +++ b/benchmarks/osworld/prepare.py @@ -40,6 +40,8 @@ POINTER_AGENT_CONFIG = BENCHMARK_DIR / "configs" / "osworld_agent_pointer.yaml" NANO_OMNI_AGENT_CONFIG = BENCHMARK_DIR / "configs" / "osworld_agent_nano_omni.yaml" OSWORLD_PROVIDER_CONFIG = BENCHMARK_DIR / "configs" / "osworld_docker_pinned.yaml" +OPENSANDBOX_CONFIG = BENCHMARK_DIR / "configs" / "osworld_opensandbox.yaml" +OPENSANDBOX_VM_SENTINEL = "/opensandbox/Ubuntu.qcow2" PROFILE_CONFIGS: dict[str, tuple[Path, ...]] = { "default": (DEFAULT_CONFIG,), @@ -55,6 +57,7 @@ # The reusable OSWorld agent config defines the Docker Sandbox provider; # env.yaml activates it for this backend. "gym_sandbox": None, + "gym_opensandbox": OPENSANDBOX_CONFIG, "osworld_provider": OSWORLD_PROVIDER_CONFIG, } @@ -290,6 +293,17 @@ def write_env( resolved_vm_path = vm_path.expanduser().resolve() if vm_path else None if execution_backend == "gym_sandbox" and resolved_vm_path is None: raise ValueError("gym_sandbox execution requires an explicit vm_path") + if execution_backend == "gym_opensandbox" and resolved_vm_path is not None: + raise ValueError("gym_opensandbox uses the server-side Pool image and does not accept vm_path") + sandbox_provider_name = { + "gym_sandbox": "osworld_sandbox", + "gym_opensandbox": "osworld_opensandbox", + }.get(execution_backend) + emitted_vm_path: str | Path | None = ( + OPENSANDBOX_VM_SENTINEL + if execution_backend == "gym_opensandbox" + else resolved_vm_path + ) contents = "\n".join( [ "# Generated by benchmarks/osworld/prepare.py. This file is gitignored.", @@ -325,8 +339,23 @@ def write_env( f" concurrency: {num_samples_in_parallel}", f" setup_cache_dir: {_yaml_string(setup_cache_dir.resolve())}", f" asset_input_jsonl: {_yaml_string((asset_input_jsonl or input_jsonl).resolve())}", - f" sandbox_provider: {'osworld_sandbox' if execution_backend == 'gym_sandbox' else 'null'}", - *([] if resolved_vm_path is None else [f" vm_path: {_yaml_string(resolved_vm_path)}"]), + f" sandbox_provider: {sandbox_provider_name or 'null'}", + *([] if emitted_vm_path is None else [f" vm_path: {_yaml_string(emitted_vm_path)}"]), + *( + [] + if execution_backend != "gym_opensandbox" + else [ + " sandbox_require_kvm: false", + " sandbox_ready_timeout_s: 600.0", + " sandbox_spec:", + " image: null", + " ttl_s: 14400", + " ready_timeout_s: 1200", + " provider_options:", + " extensions:", + " poolRef: ${oc.env:OPENSANDBOX_POOL_REF,osworld-kvm}", + ] + ), *([] if max_steps is None else [f" max_steps: {max_steps}"]), "", ] @@ -380,13 +409,16 @@ def main() -> None: "--execution-backend", choices=tuple(BACKEND_CONFIGS), default="osworld_provider", - help="VM lifecycle owner; both choices still execute through Gym env", + help="VM lifecycle owner; every choice still executes through Gym env", ) parser.add_argument( "--vm-path", type=Path, default=None, - help="Explicit qcow2 base; required for gym_sandbox and recommended for reproducible native runs", + help=( + "Explicit qcow2 base; required for gym_sandbox, unsupported for " + "gym_opensandbox, and recommended for reproducible native runs" + ), ) parser.add_argument( "--expected-vm-sha256", @@ -483,6 +515,8 @@ def main() -> None: vm_path = args.vm_path.expanduser().resolve() if args.vm_path else None if args.execution_backend == "gym_sandbox" and vm_path is None: parser.error("--execution-backend gym_sandbox requires --vm-path") + if args.execution_backend == "gym_opensandbox" and vm_path is not None: + parser.error("--execution-backend gym_opensandbox uses the Pool image and rejects --vm-path") if args.expected_vm_sha256 and vm_path is None: parser.error("--expected-vm-sha256 requires --vm-path") if vm_path is not None: diff --git a/benchmarks/osworld/tests/test_prepare.py b/benchmarks/osworld/tests/test_prepare.py index 084632ef0e..0312bd833f 100644 --- a/benchmarks/osworld/tests/test_prepare.py +++ b/benchmarks/osworld/tests/test_prepare.py @@ -7,12 +7,16 @@ import pytest import yaml +from omegaconf import OmegaConf from benchmarks.osworld import assets from benchmarks.osworld.assets import asset_specs_from_task, ensure_osworld_assets from benchmarks.osworld.prepare import ( + BASE_AGENT_CONFIG, DEFAULT_INPUT, NANO_OMNI_AGENT_CONFIG, + OPENSANDBOX_CONFIG, + OPENSANDBOX_VM_SENTINEL, POINTER_AGENT_CONFIG, main, prepare, @@ -21,6 +25,7 @@ write_task_shard, write_vm_snapshot_manifest, ) +from nemo_gym.global_config import GlobalConfigDictParser def test_prepare_validates_committed_example() -> None: @@ -120,6 +125,18 @@ def test_nano_omni_profile_is_one_complete_benchmark_config() -> None: assert paths == (NANO_OMNI_AGENT_CONFIG.resolve(),) +def test_opensandbox_backend_adds_cell2_provider_config() -> None: + paths = select_config_paths( + profile="nano_omni", + execution_backend="gym_opensandbox", + ) + + assert paths == ( + NANO_OMNI_AGENT_CONFIG.resolve(), + OPENSANDBOX_CONFIG.resolve(), + ) + + def test_main_writes_complete_nano_omni_profile(monkeypatch, tmp_path: Path) -> None: vm_path = tmp_path / "Ubuntu.qcow2" vm_path.write_bytes(b"qcow2-base") @@ -245,6 +262,85 @@ def test_write_env_rejects_sandbox_without_explicit_vm(tmp_path: Path) -> None: ) +def test_write_env_configures_image_less_opensandbox_pool(tmp_path: Path) -> None: + env_path = tmp_path / "run" / "env.yaml" + + assert write_env( + env_path, + config_paths=( + NANO_OMNI_AGENT_CONFIG, + OPENSANDBOX_CONFIG, + ), + input_jsonl=DEFAULT_INPUT, + output_jsonl=tmp_path / "rollouts.jsonl", + policy_base_url="http://model.test/v1", + policy_api_key="local", # pragma: allowlist secret + policy_model_name="model", + agent_name="osworld_nano_omni_agent", + execution_backend="gym_opensandbox", + ) + + config = yaml.safe_load(env_path.read_text(encoding="utf-8")) + agent = config["osworld_nano_omni_agent"]["responses_api_agents"]["osworld_agent"] + assert agent["sandbox_provider"] == "osworld_opensandbox" + assert agent["vm_path"] == OPENSANDBOX_VM_SENTINEL + assert agent["sandbox_require_kvm"] is False + assert agent["sandbox_spec"]["image"] is None + assert agent["sandbox_spec"]["ttl_s"] == 14400 + assert agent["sandbox_spec"]["provider_options"]["extensions"]["poolRef"] == ( + "${oc.env:OPENSANDBOX_POOL_REF,osworld-kvm}" + ) + assert "OPENSANDBOX_API_KEY" not in env_path.read_text(encoding="utf-8") + + +def test_opensandbox_env_composes_with_strict_inherited_sandbox_spec(tmp_path: Path) -> None: + env_path = tmp_path / "env.yaml" + assert write_env( + env_path, + config_paths=( + NANO_OMNI_AGENT_CONFIG, + OPENSANDBOX_CONFIG, + ), + input_jsonl=DEFAULT_INPUT, + output_jsonl=tmp_path / "rollouts.jsonl", + policy_base_url="http://model.test/v1", + policy_api_key="local", # pragma: allowlist secret + policy_model_name="model", + agent_name="osworld_nano_omni_agent", + execution_backend="gym_opensandbox", + ) + + config = OmegaConf.merge( + OmegaConf.load(BASE_AGENT_CONFIG), + OmegaConf.load(NANO_OMNI_AGENT_CONFIG), + OmegaConf.load(OPENSANDBOX_CONFIG), + OmegaConf.load(env_path), + ) + OmegaConf.set_struct(config, True) + GlobalConfigDictParser()._recursively_swap_keys(config) + + agent = config["osworld_nano_omni_agent"]["responses_api_agents"]["osworld_agent"] + assert agent["sandbox_spec"]["ttl_s"] == 14400 + assert agent["sandbox_spec"]["provider_options"]["extensions"]["poolRef"] == "osworld-kvm" + + +def test_write_env_rejects_local_vm_for_opensandbox(tmp_path: Path) -> None: + vm_path = tmp_path / "Ubuntu.qcow2" + vm_path.write_bytes(b"unused") + with pytest.raises(ValueError, match="does not accept vm_path"): + write_env( + tmp_path / "env.yaml", + config_path=OPENSANDBOX_CONFIG, + input_jsonl=DEFAULT_INPUT, + output_jsonl=tmp_path / "rollouts.jsonl", + policy_base_url="http://model.test/v1", + policy_api_key="local", # pragma: allowlist secret + policy_model_name="model", + execution_backend="gym_opensandbox", + vm_path=vm_path, + ) + + def test_vm_snapshot_manifest_is_content_addressed(tmp_path: Path) -> None: vm_path = tmp_path / "Ubuntu.qcow2" vm_path.write_bytes(b"fixed-qcow2") diff --git a/benchmarks/osworld/tests/test_run_scripts.py b/benchmarks/osworld/tests/test_run_scripts.py index 5807f3dff2..ef21818ab0 100644 --- a/benchmarks/osworld/tests/test_run_scripts.py +++ b/benchmarks/osworld/tests/test_run_scripts.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 import subprocess +import tomllib from pathlib import Path import pytest @@ -15,6 +16,7 @@ RUN_EVAL_SCRIPT = REPO_ROOT / "benchmarks/osworld/tools/run_eval.sh" CLEANUP_RUN_SCRIPT = REPO_ROOT / "benchmarks/osworld/tools/cleanup_run.sh" OSWORLD_AGENT_CONFIG = REPO_ROOT / "responses_api_agents/osworld_agent/configs/osworld_agent.yaml" +OSWORLD_AGENT_PYPROJECT = REPO_ROOT / "responses_api_agents/osworld_agent/pyproject.toml" @pytest.mark.parametrize( @@ -47,6 +49,12 @@ def test_start_control_preflights_native_build_toolchain() -> None: assert "python3-dev" in text +def test_managed_osworld_agent_installs_opensandbox_sdk() -> None: + project = tomllib.loads(OSWORLD_AGENT_PYPROJECT.read_text(encoding="utf-8")) + + assert "nemo-gym[dev,sandbox]" in project["project"]["dependencies"] + + def test_remote_docker_requires_a_reachable_publish_host() -> None: start_text = START_CONTROL_SCRIPT.read_text(encoding="utf-8") sandbox_text = OSWORLD_AGENT_CONFIG.read_text(encoding="utf-8") diff --git a/nemo_gym/sandbox/api.py b/nemo_gym/sandbox/api.py index cf46db25a2..f54532cc84 100644 --- a/nemo_gym/sandbox/api.py +++ b/nemo_gym/sandbox/api.py @@ -15,6 +15,7 @@ """Provider-neutral public sandbox API.""" import asyncio +import logging import tempfile import threading from collections.abc import Awaitable, Callable, Mapping @@ -37,8 +38,10 @@ T = TypeVar("T") +LOGGER = logging.getLogger(__name__) SYNC_OPERATION_TIMEOUT_S = 3600.0 SYNC_LOOP_CLOSE_TIMEOUT_S = 5.0 +SYNC_CANCELLATION_TIMEOUT_S = 75.0 class AsyncSandbox: @@ -81,7 +84,7 @@ async def start( source_path = tmp_path / f"file-{index}" source_path.write_text(contents, encoding="utf-8") await self._provider.upload_file(handle, source_path, target_path) - except Exception: + except BaseException: await self._provider.close(handle) await self._provider.aclose() self._closed = True @@ -202,9 +205,11 @@ def __init__( *, wait_timeout_s: float = SYNC_OPERATION_TIMEOUT_S, close_timeout_s: float = SYNC_LOOP_CLOSE_TIMEOUT_S, + cancellation_timeout_s: float = SYNC_CANCELLATION_TIMEOUT_S, ) -> None: self._wait_timeout_s = wait_timeout_s self._close_timeout_s = close_timeout_s + self._cancellation_timeout_s = cancellation_timeout_s self._loop = asyncio.new_event_loop() self._ready = threading.Event() self._closed = False @@ -234,6 +239,9 @@ def _wait_for_result(self, operation: str, future: Future[T]) -> T: raise TimeoutError( f"Sandbox.{operation}() timed out waiting for the sync loop after {self._wait_timeout_s:g}s" ) from e + except BaseException: + future.cancel() + raise def call(self, operation: str, func: Callable[[], T]) -> T: self._ensure_can_block(operation) @@ -254,14 +262,41 @@ def invoke() -> None: def run(self, operation: str, awaitable_factory: Callable[[], Awaitable[T]]) -> T: self._ensure_can_block(operation) - future = asyncio.run_coroutine_threadsafe(awaitable_factory(), self._loop) + completion = threading.Event() + awaitable = awaitable_factory() + + async def tracked_awaitable() -> T: + try: + return await awaitable + finally: + # ``concurrent.futures.Future.cancel()`` becomes done before + # the underlying asyncio Task has finished unwinding. Signal + # the caller only after provider-side cancellation cleanup. + completion.set() + + future = asyncio.run_coroutine_threadsafe(tracked_awaitable(), self._loop) + + def cancel_and_drain() -> None: + future.cancel() + if not completion.wait(timeout=self._cancellation_timeout_s): + LOGGER.warning( + "Sandbox.%s() cancellation cleanup did not finish within %ss", + operation, + self._cancellation_timeout_s, + ) + try: return future.result(timeout=self._wait_timeout_s) except FutureTimeoutError as e: - future.cancel() + cancel_and_drain() raise TimeoutError( f"Sandbox.{operation}() timed out waiting for the sync loop after {self._wait_timeout_s:g}s" ) from e + except BaseException: + # Ctrl-C and other caller-side BaseExceptions must not leave the + # provider coroutine running invisibly on the sync loop. + cancel_and_drain() + raise def close(self) -> None: if self._closed: diff --git a/nemo_gym/sandbox/providers/opensandbox/provider.py b/nemo_gym/sandbox/providers/opensandbox/provider.py index 002ad47410..68e48967c6 100644 --- a/nemo_gym/sandbox/providers/opensandbox/provider.py +++ b/nemo_gym/sandbox/providers/opensandbox/provider.py @@ -15,9 +15,11 @@ """OpenSandbox provider implementation.""" import asyncio +import json import logging import re import shlex +import uuid from collections.abc import Mapping from dataclasses import dataclass, field, replace from datetime import timedelta @@ -28,6 +30,7 @@ from nemo_gym.sandbox.providers.base import ( SandboxCreateError, SandboxCreateVerificationError, + SandboxEndpoint, SandboxExecResult, SandboxHandle, SandboxResources, @@ -62,6 +65,7 @@ class SandboxBackendUnreachableError(RuntimeError): RETRYABLE_HTTP_STATUS_CODES = {408, 409, 425, 429, 500, 502, 503, 504} +POOLED_CREATE_MARKER_KEY = "nemo-gym-create-id" RETRYABLE_ERROR_MARKERS = ( "all connection attempts failed", "connection refused", @@ -285,6 +289,29 @@ def _log_operation_retry(retry_state: Any, *, operation: str = "?", sandbox_id: ) +async def _rest_request( + method: str, + url: str, + *, + json_body: Any | None = None, + headers: Mapping[str, str] | None = None, + timeout_s: float = 300.0, +) -> tuple[int, str]: + """Send one lifecycle request without initializing Gym's global config.""" + + import aiohttp # noqa: PLC0415 + + timeout = aiohttp.ClientTimeout(total=timeout_s) + async with aiohttp.ClientSession(timeout=timeout, trust_env=False) as session: + async with session.request( + method, + url, + json=json_body, + headers=dict(headers or {}), + ) as response: + return response.status, await response.text() + + def _string_map(values: Mapping[str, Any]) -> dict[str, str]: return {str(key): str(value) for key, value in values.items()} @@ -948,11 +975,246 @@ async def _connect_after_create(self, handle: SandboxHandle, spec: SandboxSpec) if sleep_s > 0: await asyncio.sleep(sleep_s) + def _rest_base_url(self) -> str: + domain = self._connection.domain + if not domain: + raise ValueError("OpenSandbox connection.domain is required for pool-mode create") + if domain.startswith(("http://", "https://")): + return domain.rstrip("/") + return f"{self._connection.protocol or 'http'}://{domain}".rstrip("/") + + def _rest_headers(self) -> dict[str, str]: + if self._connection.api_key: + return {"OPEN-SANDBOX-API-KEY": self._connection.api_key} + return {} + + async def _rest_create_pooled( + self, + spec: SandboxSpec, + options: OpenSandboxProviderOptions, + ) -> str: + """Allocate from a server-side Pool using an image-less lifecycle POST.""" + + create_id = uuid.uuid4().hex + body: dict[str, Any] = { + "extensions": dict(options.extensions), + "metadata": { + **(spec.metadata or {}), + POOLED_CREATE_MARKER_KEY: create_id, + }, + } + if spec.ttl_s is not None: + body["timeout"] = int(spec.ttl_s) + + try: + status, text = await _rest_request( + "POST", + f"{self._rest_base_url()}/v1/sandboxes", + json_body=body, + headers=self._rest_headers(), + timeout_s=float(self._create.request_timeout_s or self._connection.request_timeout_s or 300), + ) + except BaseException: + await self._reap_pooled_create_marker(create_id) + raise + + if status not in {200, 201, 202}: + raise OpenSandboxCreateError(f"OpenSandbox pool-mode create failed (HTTP {status}): {text[:300]}") + try: + sandbox_id = json.loads(text)["id"] + except (json.JSONDecodeError, KeyError, TypeError) as error: + raise OpenSandboxCreateError( + f"OpenSandbox pool-mode create returned an unexpected body: {text[:300]}" + ) from error + return str(sandbox_id) + + async def _reap_pooled_create_marker(self, create_id: str) -> None: + """Delete only a lost create carrying this client's exact marker.""" + + base = self._rest_base_url() + timeout_s = float(self._operations.close_timeout_s or 30.0) + loop = asyncio.get_running_loop() + deadline = loop.time() + timeout_s + page = 1 + page_size = 200 + + def remaining_timeout_s() -> float: + return max(deadline - loop.time(), 0.0) + + try: + while page <= 100: + remaining_s = remaining_timeout_s() + if remaining_s <= 0: + LOGGER.warning( + "Timed out scanning OpenSandbox sandboxes for abandoned " + "create marker %s after %s pages", + create_id, + page - 1, + ) + return + status, text = await _rest_request( + "GET", + f"{base}/v1/sandboxes?page={page}&pageSize={page_size}", + headers=self._rest_headers(), + timeout_s=remaining_s, + ) + if status != 200: + LOGGER.warning( + "Could not list OpenSandbox sandboxes to reap abandoned " + "create marker %s: page=%s HTTP %s", + create_id, + page, + status, + ) + return + data = json.loads(text) + items = ( + data + if isinstance(data, list) + else data.get("items") or data.get("sandboxes") or [] + ) + for item in items: + if not isinstance(item, Mapping): + continue + metadata = item.get("metadata") or {} + if ( + not isinstance(metadata, Mapping) + or metadata.get(POOLED_CREATE_MARKER_KEY) != create_id + ): + continue + sandbox_id = str(item.get("id") or "") + if not sandbox_id: + continue + remaining_s = remaining_timeout_s() + if remaining_s <= 0: + LOGGER.warning( + "Timed out before deleting abandoned OpenSandbox " + "pooled create %s (marker %s)", + sandbox_id, + create_id, + ) + return + delete_status, _ = await _rest_request( + "DELETE", + f"{base}/v1/sandboxes/{sandbox_id}", + headers=self._rest_headers(), + timeout_s=remaining_s, + ) + LOGGER.warning( + "Reaped abandoned OpenSandbox pooled create %s " + "(marker %s, DELETE HTTP %s)", + sandbox_id, + create_id, + delete_status, + ) + return + + if isinstance(data, list): + has_next_page = len(items) >= page_size + else: + pagination = data.get("pagination") or {} + has_next_page = bool(pagination.get("hasNextPage")) + total_pages = pagination.get("totalPages") + if isinstance(total_pages, int): + has_next_page = has_next_page or page < total_pages + if not has_next_page: + return + page += 1 + LOGGER.warning( + "Stopped scanning OpenSandbox sandboxes for abandoned create " + "marker %s after %s pages", + create_id, + page - 1, + ) + except Exception as error: # noqa: BLE001 + LOGGER.warning( + "Failed to reap abandoned OpenSandbox pooled create (marker %s): %r", + create_id, + error, + ) + + async def _cleanup_failed_pooled_create(self, handle: SandboxHandle) -> None: + try: + await _rest_request( + "DELETE", + f"{self._rest_base_url()}/v1/sandboxes/{handle.sandbox_id}", + headers=self._rest_headers(), + timeout_s=float(self._operations.close_timeout_s or 30.0), + ) + except Exception as error: # noqa: BLE001 + LOGGER.warning( + "Failed to clean up pooled sandbox after create failure; sandbox_id=%s; error=%r", + handle.sandbox_id, + error, + ) + + async def _create_pooled( + self, + spec: SandboxSpec, + options: OpenSandboxProviderOptions, + ) -> SandboxHandle: + if "poolRef" not in options.extensions: + raise ValueError( + "OpenSandbox create without image/snapshot_id requires provider_options.extensions.poolRef" + ) + sandbox_id = await self._rest_create_pooled(spec, options) + rest_handle = _PooledRestSandbox( + sandbox_id=sandbox_id, + base_url=self._rest_base_url(), + headers=self._rest_headers(), + protocol=self._connection.protocol or "http", + timeout_s=float( + self._connection.request_timeout_s + or self._create.request_timeout_s + or 300.0 + ), + use_server_proxy=self._connection.use_server_proxy, + ) + created_handle = SandboxHandle( + sandbox_id=sandbox_id, + provider_name=self.name, + raw=rest_handle, + ) + try: + await self._verify_created_handle(created_handle) + except BaseException: + await self._cleanup_failed_pooled_create(created_handle) + raise + return created_handle + + async def endpoint( + self, + handle: SandboxHandle, + port: int, + ) -> SandboxEndpoint: + """Resolve one client-reachable direct or server-proxied service URL.""" + + if handle.raw is None: + raise RuntimeError(f"OpenSandbox handle for {handle.sandbox_id!r} has no SDK object") + endpoint = await self._await_sdk_operation( + lambda: handle.raw.get_endpoint(port), + operation=f"get_endpoint({port})", + sandbox_id=handle.sandbox_id, + timeout_s=( + float(self._connection.request_timeout_s) if self._connection.request_timeout_s is not None else None + ), + ) + url = str(getattr(endpoint, "endpoint", "") or "") + if not url: + raise RuntimeError(f"OpenSandbox returned an empty endpoint for sandbox {handle.sandbox_id!r} port {port}") + if "://" not in url: + url = f"{self._connection.protocol or 'http'}://{url}" + headers = dict(getattr(endpoint, "headers", None) or {}) + return SandboxEndpoint(endpoint=url, headers=headers) + async def _create_once(self, spec: SandboxSpec) -> SandboxHandle: """Create a sandbox through ``opensandbox.Sandbox.create``.""" Sandbox, _, _, _, _ = _require_opensandbox_sdk() options = OpenSandboxProviderOptions.from_mapping(spec.provider_options) + if spec.image is None and options.snapshot_id is None: + return await self._create_pooled(spec, options) + kwargs: dict[str, Any] = { "env": spec.env, "metadata": spec.metadata, @@ -1017,7 +1279,7 @@ async def _create_once(self, spec: SandboxSpec) -> SandboxHandle: if self._create.skip_health_check: handle = await self._connect_after_create(created_handle, spec) await self._verify_created_handle(handle) - except Exception: + except BaseException: await self._cleanup_failed_create_handle(created_handle) raise return handle diff --git a/responses_api_agents/osworld_agent/client.py b/responses_api_agents/osworld_agent/client.py index 960f8da636..bffd6dce6f 100644 --- a/responses_api_agents/osworld_agent/client.py +++ b/responses_api_agents/osworld_agent/client.py @@ -1716,7 +1716,14 @@ def proxy_precondition_failure(reason: str, message: str) -> RolloutResult: } if use_gym_sandbox: effective_sandbox_spec = dict(sandbox_spec or {}) - effective_sandbox_spec.setdefault("image", container_image) + sandbox_provider_name = str(next(iter(sandbox_provider_config or {}), "")).lower().strip() + if sandbox_provider_name == "docker": + effective_sandbox_spec.setdefault("image", container_image) + elif sandbox_provider_name == "opensandbox": + # An image-less create allocates a prebuilt QEMU guest from + # the server-side Pool. Do not let the reusable Docker + # profile's image select the SDK's container-create path. + effective_sandbox_spec.pop("image", None) env_kwargs.update( { "sandbox_provider": dict(sandbox_provider_config or {}), diff --git a/responses_api_agents/osworld_agent/configs/osworld_agent.yaml b/responses_api_agents/osworld_agent/configs/osworld_agent.yaml index 76e538684e..c51a7c16e4 100644 --- a/responses_api_agents/osworld_agent/configs/osworld_agent.yaml +++ b/responses_api_agents/osworld_agent/configs/osworld_agent.yaml @@ -14,6 +14,7 @@ osworld_simple_agent: sandbox_provider: null # named nemo_gym.sandbox config; when set, Sandbox owns VM container lifecycle sandbox_spec: image: docker://happysixd/osworld-docker@sha256:0e6497a9295647cf05bf2b2af522fdd79bdeba2737595259cab310a3bcf6baa9 + ttl_s: null ready_timeout_s: 120 entrypoint: - /usr/bin/tini @@ -27,6 +28,9 @@ osworld_simple_agent: cpu: 4 memory_mib: 16384 disk_gib: 40 + provider_options: + extensions: + poolRef: null vm_path: ${oc.env:OSWORLD_VM_PATH,${oc.env:OSWORLD_SANDBOX_VM_PATH,""}} # optional absolute qcow2 path sandbox_vm_path: null # deprecated compatibility alias for vm_path sandbox_require_kvm: true diff --git a/responses_api_agents/osworld_agent/local_forwarder.py b/responses_api_agents/osworld_agent/local_forwarder.py new file mode 100644 index 0000000000..3f4bc29c1c --- /dev/null +++ b/responses_api_agents/osworld_agent/local_forwarder.py @@ -0,0 +1,197 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Local reverse proxy for path-based OpenSandbox guest endpoints. + +OSWorld's controllers accept only ``http://host:port/...``. OpenSandbox's +server-proxy endpoints instead look like +``http://gateway/v1/sandboxes//proxy/`` and may require route +headers. This forwarder exposes an ephemeral loopback port, prepends the +upstream path, injects route headers, and carries Chrome CDP WebSocket +upgrades. + +The implementation is adapted from Cell-1's validated +``resources_servers/osworld/local_forwarder.py``. It deliberately disables +ambient HTTP proxy variables: this bridge must reach the OpenSandbox gateway, +not a workstation proxy. +""" + +from __future__ import annotations + +import re +import socket +import ssl +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from urllib.parse import urlsplit + +import requests + + +_HOP_BY_HOP_REQUEST_HEADERS = ( + "host", + "content-length", + "connection", + "accept-encoding", +) +_HOP_BY_HOP_RESPONSE_HEADERS = ( + "transfer-encoding", + "content-length", + "connection", + "content-encoding", +) + +# Chrome's /json/version and /json/list responses embed an absolute CDP +# WebSocket URL. Rewrite it to the local bridge so the subsequent Upgrade +# request follows the same OpenSandbox proxy path. +_WS_URL_RE = re.compile(rb"wss?://[^/\"]+/") + + +def _open_upstream_socket( + *, + scheme: str, + host: str, + port: int, +) -> socket.socket: + upstream = socket.create_connection((host, port), timeout=30) + if scheme == "https": + context = ssl.create_default_context() + upstream = context.wrap_socket(upstream, server_hostname=host) + upstream.settimeout(None) + return upstream + + +def start_forwarder( + base_url: str, + extra_headers: dict[str, str] | None = None, + timeout_s: float = 300.0, +) -> tuple[ThreadingHTTPServer, int]: + """Start ``127.0.0.1:/ -> /``. + + The caller owns the returned server and must call both ``shutdown()`` and + ``server_close()`` during sandbox cleanup. + """ + + base = base_url.rstrip("/") + headers_to_add = dict(extra_headers or {}) + split = urlsplit(base) + if split.scheme not in {"http", "https"} or split.hostname is None: + raise ValueError(f"forwarder requires an absolute HTTP(S) URL, got {base_url!r}") + + proxy_host = split.hostname + proxy_port = split.port or (443 if split.scheme == "https" else 80) + proxy_authority = split.netloc + proxy_path_prefix = split.path.rstrip("/") + + class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, *args: object) -> None: + del args + + def _tunnel_websocket(self) -> None: + upstream = _open_upstream_socket( + scheme=split.scheme, + host=proxy_host, + port=proxy_port, + ) + lines = [ + f"GET {proxy_path_prefix}{self.path} HTTP/1.1", + f"Host: {proxy_authority}", + ] + client_keys = {key.lower() for key in self.headers} + for key, value in self.headers.items(): + if key.lower() != "host": + lines.append(f"{key}: {value}") + for key, value in headers_to_add.items(): + if key.lower() not in client_keys: + lines.append(f"{key}: {value}") + upstream.sendall(("\r\n".join(lines) + "\r\n\r\n").encode()) + client = self.connection + + def pump(source: socket.socket, destination: socket.socket) -> None: + try: + while chunk := source.recv(65536): + destination.sendall(chunk) + except OSError: + pass + finally: + for stream in (source, destination): + try: + stream.shutdown(socket.SHUT_RDWR) + except OSError: + pass + + upstream_to_client = threading.Thread( + target=pump, + args=(upstream, client), + daemon=True, + ) + upstream_to_client.start() + pump(client, upstream) + upstream_to_client.join(timeout=5) + self.close_connection = True + + def _forward(self) -> None: + if (self.headers.get("Upgrade") or "").lower() == "websocket": + self._tunnel_websocket() + return + + length = int(self.headers.get("Content-Length", 0) or 0) + body = self.rfile.read(length) if length else None + headers = { + key: value + for key, value in self.headers.items() + if key.lower() not in _HOP_BY_HOP_REQUEST_HEADERS + } + # Guest authentication headers (for example VLC Basic auth) win + # over route headers with the same name. + lower_client_headers = {key.lower() for key in headers} + for key, value in headers_to_add.items(): + if key.lower() not in lower_client_headers: + headers[key] = value + + try: + with requests.Session() as session: + session.trust_env = False + upstream = session.request( + self.command, + base + self.path, + data=body, + headers=headers, + timeout=timeout_s, + allow_redirects=False, + ) + except Exception as error: # noqa: BLE001 + message = str(error).encode() + self.send_response(502) + self.send_header("Content-Length", str(len(message))) + self.end_headers() + self.wfile.write(message) + return + + content = b"" if self.command == "HEAD" else upstream.content + if b"webSocketDebuggerUrl" in content or b"webSocketUrl" in content: + local = f"ws://127.0.0.1:{self.server.server_address[1]}/".encode() + content = _WS_URL_RE.sub(local, content) + + self.send_response(upstream.status_code) + for key, value in upstream.headers.items(): + if key.lower() not in _HOP_BY_HOP_RESPONSE_HEADERS: + self.send_header(key, value) + self.send_header("Content-Length", str(len(content))) + self.end_headers() + if content: + self.wfile.write(content) + + do_DELETE = _forward + do_GET = _forward + do_HEAD = _forward + do_OPTIONS = _forward + do_PATCH = _forward + do_POST = _forward + do_PUT = _forward + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + port = int(server.server_address[1]) + threading.Thread(target=server.serve_forever, daemon=True).start() + return server, port diff --git a/responses_api_agents/osworld_agent/pyproject.toml b/responses_api_agents/osworld_agent/pyproject.toml index b24a9462cd..fe76257dc1 100644 --- a/responses_api_agents/osworld_agent/pyproject.toml +++ b/responses_api_agents/osworld_agent/pyproject.toml @@ -18,7 +18,9 @@ name = "osworld-agent" version = "0.0.0" requires-python = ">=3.13.14" dependencies = [ - "nemo-gym[dev]", + # The managed agent server gets its own venv. Include the sandbox extra + # here so OpenSandbox-backed runs have the SDK in that isolated runtime. + "nemo-gym[dev,sandbox]", # JeffPengCoder/OSWorld nv-gym at an immutable commit: upstream main # 83e85344 plus the nv-gym provider overlay and explicit proxy-runtime # repair. Use an archive because uv recursively initializes git diff --git a/responses_api_agents/osworld_agent/sandbox_provider.py b/responses_api_agents/osworld_agent/sandbox_provider.py index 2f3aea93c7..1c8c978675 100644 --- a/responses_api_agents/osworld_agent/sandbox_provider.py +++ b/responses_api_agents/osworld_agent/sandbox_provider.py @@ -15,12 +15,14 @@ import os import time from collections.abc import Mapping +from http.server import ThreadingHTTPServer from typing import Any from urllib.parse import urlsplit import requests from nemo_gym.sandbox import Sandbox, SandboxEndpoint, SandboxSpec, SandboxStatus +from responses_api_agents.osworld_agent.local_forwarder import start_forwarder LOG = logging.getLogger("nemo_gym.osworld_agent.sandbox_provider") @@ -77,7 +79,7 @@ def _http_origin(host: str, port: int) -> str: class GymSandboxDesktopProvider: - """Implement OSWorld's provider contract with one Gym Docker Sandbox per VM.""" + """Implement OSWorld's provider contract with one Gym Sandbox per VM.""" def __init__( self, @@ -99,9 +101,10 @@ def __init__( self._sandbox_provider = copy.deepcopy(dict(sandbox_provider)) self._sandbox_provider_name = str(next(iter(self._sandbox_provider))).lower().strip() - if self._sandbox_provider_name != "docker": + if self._sandbox_provider_name not in {"docker", "opensandbox"}: raise ValueError( - "The OSWorld Gym Sandbox deployment requires Gym's Docker provider, " + "The OSWorld Gym Sandbox deployment requires Gym's Docker or " + "OpenSandbox provider, " f"got {self._sandbox_provider_name!r}" ) self._sandbox_spec = copy.deepcopy(dict(sandbox_spec)) @@ -109,6 +112,7 @@ def __init__( self._ready_timeout_s = float(ready_timeout_s) self._ready_poll_s = float(ready_poll_s) self._sandbox: Sandbox | None = None + self._forwarders: list[ThreadingHTTPServer] = [] self._host: str | None = None self.server_port: int | None = None self.chromium_port: int | None = None @@ -118,11 +122,54 @@ def __init__( def _build_spec(self, path_to_vm: str, *, headless: bool, os_type: str) -> SandboxSpec: if os_type.lower() not in {"ubuntu", "linux"}: raise ValueError(f"Gym Sandbox OSWorld adapter currently supports Ubuntu only, got {os_type!r}") + + values = copy.deepcopy(self._sandbox_spec) + values["ports"] = list(dict.fromkeys([*(values.get("ports") or ()), *OSWORLD_SERVICE_PORTS])) + + metadata = dict(values.get("metadata") or {}) + metadata.setdefault("workload", "osworld") + metadata.setdefault( + "osworld-provider", + f"gym-{self._sandbox_provider_name}-sandbox", + ) + run_id = os.environ.get("OSWORLD_RUN_ID", "").strip() + if run_id: + metadata.setdefault("run-id", run_id) + values["metadata"] = metadata + + if self._sandbox_provider_name == "opensandbox": + if values.get("image"): + raise ValueError( + "OpenSandbox OSWorld Pool allocation must be image-less; the Pool owns the QEMU image" + ) + provider_options = dict(values.get("provider_options") or {}) + extensions = dict(provider_options.get("extensions") or {}) + if not extensions.get("poolRef"): + raise ValueError("OpenSandbox OSWorld sandbox_spec requires provider_options.extensions.poolRef") + provider_options["extensions"] = extensions + values["provider_options"] = provider_options + values.setdefault("ttl_s", 7200) + values.setdefault("ready_timeout_s", self._ready_timeout_s) + # The reusable OSWorld profile also carries Docker/QEMU defaults. + # They are intentionally discarded because the server-side Pool + # owns the image, entrypoint, environment, and resources. + pool_fields = { + key: values[key] + for key in ( + "ttl_s", + "ready_timeout_s", + "ports", + "metadata", + "provider_options", + ) + if key in values + } + return SandboxSpec(**pool_fields) + vm_path = os.path.realpath(os.path.abspath(os.path.expanduser(path_to_vm))) if not os.path.isfile(vm_path) or not os.access(vm_path, os.R_OK): raise FileNotFoundError(f"OSWorld base qcow2 is not readable: {vm_path}") - values = copy.deepcopy(self._sandbox_spec) if not values.get("image"): raise ValueError("sandbox_spec.image is required for OSWorld") @@ -134,12 +181,6 @@ def _build_spec(self, path_to_vm: str, *, headless: bool, os_type: str) -> Sandb environment["KVM"] = "Y" if self._require_kvm else "N" values["env"] = environment values.setdefault("entrypoint", list(OSWORLD_IMAGE_ENTRYPOINT)) - values["ports"] = list(dict.fromkeys([*(values.get("ports") or ()), *OSWORLD_SERVICE_PORTS])) - - metadata = dict(values.get("metadata") or {}) - metadata.setdefault("workload", "osworld") - metadata.setdefault("osworld-provider", "gym-docker-sandbox") - values["metadata"] = metadata provider_options = dict(values.get("provider_options") or {}) volumes = _string_list(provider_options.get("volumes"), field="volumes") @@ -151,7 +192,6 @@ def _build_spec(self, path_to_vm: str, *, headless: bool, os_type: str) -> Sandb run_args = _string_list(provider_options.get("run_args"), field="run_args") if not _has_option(run_args, "--label", OSWORLD_WORKLOAD_LABEL): run_args.extend(["--label", OSWORLD_WORKLOAD_LABEL]) - run_id = os.environ.get("OSWORLD_RUN_ID", "").strip() run_id_label = f"{OSWORLD_RUN_ID_LABEL}={run_id}" if run_id and not _has_option(run_args, "--label", run_id_label): run_args.extend(["--label", run_id_label]) @@ -165,29 +205,65 @@ def _build_spec(self, path_to_vm: str, *, headless: bool, os_type: str) -> Sandb return SandboxSpec(**values) def _resolve_service_endpoints(self, sandbox: Sandbox) -> tuple[str, dict[int, int]]: - host: str | None = None - resolved_ports: dict[int, int] = {} - for container_port in OSWORLD_SERVICE_PORTS: - endpoint_host, endpoint_port = _parse_plain_http_endpoint( - sandbox.endpoint(container_port), - container_port, - ) - if host is None: - host = endpoint_host - elif endpoint_host != host: - raise ValueError( - "OSWorld requires all Sandbox service endpoints to share one host; " - f"got {host!r} and {endpoint_host!r}" + endpoints = { + container_port: sandbox.endpoint(container_port) + for container_port in OSWORLD_SERVICE_PORTS + } + + # Preserve the zero-hop path for local Docker or a routed Pod network. + try: + direct = { + container_port: _parse_plain_http_endpoint(endpoint, container_port) + for container_port, endpoint in endpoints.items() + } + except ValueError: + direct = {} + if direct: + hosts = {host for host, _ in direct.values()} + if len(hosts) == 1: + return hosts.pop(), { + container_port: endpoint_port + for container_port, (_, endpoint_port) in direct.items() + } + + # OpenSandbox's externally reachable endpoint is a path-based gateway + # URL. OSWorld only understands a shared host plus four integer ports, + # so give each service a loopback forwarder. The forwarder also carries + # Chrome CDP WebSockets and injects any route headers. + forwarders: list[ThreadingHTTPServer] = [] + forwarded_ports: dict[int, int] = {} + try: + for container_port, endpoint in endpoints.items(): + server, local_port = start_forwarder( + endpoint.endpoint, + endpoint.headers, + timeout_s=max(self._ready_timeout_s, 300.0), ) - resolved_ports[container_port] = endpoint_port - if host is None: - raise RuntimeError("Gym Sandbox returned no OSWorld service endpoints") - return host, resolved_ports + forwarders.append(server) + forwarded_ports[container_port] = local_port + except BaseException: + for server in forwarders: + server.shutdown() + server.server_close() + raise + + self._forwarders.extend(forwarders) + return "127.0.0.1", forwarded_ports + + def _stop_forwarders(self) -> None: + forwarders = self._forwarders + self._forwarders = [] + for server in forwarders: + with contextlib.suppress(Exception): + server.shutdown() + with contextlib.suppress(Exception): + server.server_close() def _wait_for_vm_ready(self, sandbox: Sandbox, host: str, server_port: int) -> None: deadline = time.monotonic() + self._ready_timeout_s last_error = "guest readiness was not attempted" with requests.Session() as session: + session.trust_env = False while time.monotonic() < deadline: try: response = session.get( @@ -218,6 +294,7 @@ def start_emulator(self, path_to_vm: str, headless: bool, os_type: str) -> None: host, ports = self._resolve_service_endpoints(sandbox) self._wait_for_vm_ready(sandbox, host, ports[5000]) except BaseException: + self._stop_forwarders() # Preserve the startup failure if best-effort cleanup also fails. with contextlib.suppress(Exception): sandbox.stop() @@ -263,5 +340,6 @@ def stop_emulator(self, path_to_vm: str, region: str | None = None, *args: Any, self.chromium_port = None self.vnc_port = None self.vlc_port = None + self._stop_forwarders() if sandbox is not None: sandbox.stop() diff --git a/responses_api_agents/osworld_agent/tests/test_client.py b/responses_api_agents/osworld_agent/tests/test_client.py index 81c563cc2f..606e9d955d 100644 --- a/responses_api_agents/osworld_agent/tests/test_client.py +++ b/responses_api_agents/osworld_agent/tests/test_client.py @@ -360,6 +360,39 @@ def test_gym_sandbox_backend_is_passed_as_plain_env_configuration(monkeypatch) - assert kwargs["sandbox_ready_poll_s"] == 0.25 +def test_opensandbox_pool_backend_discards_inherited_docker_image(monkeypatch) -> None: + _patch_client_for_fake_runtime(monkeypatch) + + result = osworld_client.run_osworld_task( + {"id": "task-opensandbox", "instruction": "Finish the task."}, + model_fn=lambda *_args: "```DONE```", + env_class_path="fake.FakeEnv", + sandbox_provider_config={"opensandbox": {"connection": {}}}, + sandbox_spec={ + "image": "docker://inherited-osworld-image", + "provider_options": { + "extensions": {"poolRef": "osworld-kvm"}, + }, + }, + vm_path="/opensandbox/Ubuntu.qcow2", + sandbox_require_kvm=False, + sleep_after_execution=0, + task_timeout=10, + ) + + assert result.finished is True + kwargs = FakeEnv.instances[0].kwargs + assert kwargs["sandbox_provider"] == { + "opensandbox": {"connection": {}}, + } + assert "image" not in kwargs["sandbox_spec"] + assert kwargs["sandbox_spec"]["provider_options"]["extensions"]["poolRef"] == ( + "osworld-kvm" + ) + assert kwargs["path_to_vm"] == "/opensandbox/Ubuntu.qcow2" + assert kwargs["sandbox_require_kvm"] is False + + def test_pointer_gym_sandbox_uses_pointer_environment(monkeypatch, tmp_path: Path) -> None: _patch_client_for_fake_runtime(monkeypatch) monkeypatch.setenv("OSWORLD_POINTER_RESULTS_DIR", str(tmp_path)) diff --git a/responses_api_agents/osworld_agent/tests/test_sandbox_provider.py b/responses_api_agents/osworld_agent/tests/test_sandbox_provider.py index 9bbfadc0ef..d3c7e34d02 100644 --- a/responses_api_agents/osworld_agent/tests/test_sandbox_provider.py +++ b/responses_api_agents/osworld_agent/tests/test_sandbox_provider.py @@ -3,12 +3,16 @@ from __future__ import annotations +import http.server +import threading from typing import Any import pytest +import requests from nemo_gym.sandbox import SandboxEndpoint, SandboxStatus from responses_api_agents.osworld_agent import sandbox_provider as osworld_sandbox +from responses_api_agents.osworld_agent.local_forwarder import start_forwarder class FakeSandbox: @@ -113,8 +117,73 @@ def test_build_spec_docker_tcg_mode_does_not_map_kvm(tmp_path) -> None: assert osworld_sandbox._has_option(spec.provider_options["run_args"], "--cap-add", "NET_ADMIN") +def test_build_spec_uses_image_less_opensandbox_pool(monkeypatch) -> None: + monkeypatch.setenv("OSWORLD_RUN_ID", "opensandbox-run") + provider = osworld_sandbox.GymSandboxDesktopProvider( + { + "opensandbox": { + "connection": { + "domain": "http://sandbox.example", + "use_server_proxy": False, + } + } + }, + { + "ttl_s": 1800, + "image": None, + "entrypoint": ["/run/entry.sh"], + "env": {"KVM": "Y"}, + "resources": {"cpu": 4, "memory_mib": 16384}, + "provider_options": {"extensions": {"poolRef": "osworld-kvm"}}, + }, + ) + + spec = provider._build_spec( + "/opensandbox/Ubuntu.qcow2", + headless=True, + os_type="Ubuntu", + ) + + assert spec.image is None + assert spec.ttl_s == 1800 + assert spec.ports == osworld_sandbox.OSWORLD_SERVICE_PORTS + assert spec.provider_options == {"extensions": {"poolRef": "osworld-kvm"}} + assert spec.metadata["osworld-provider"] == "gym-opensandbox-sandbox" + assert spec.metadata["run-id"] == "opensandbox-run" + assert spec.entrypoint is None + assert spec.env == {} + assert spec.resources.cpu is None + + +def test_build_spec_rejects_invalid_opensandbox_pool_spec() -> None: + provider = osworld_sandbox.GymSandboxDesktopProvider( + {"opensandbox": {}}, + {"provider_options": {"extensions": {}}}, + ) + with pytest.raises(ValueError, match="poolRef"): + provider._build_spec( + "/opensandbox/Ubuntu.qcow2", + headless=True, + os_type="Ubuntu", + ) + + provider = osworld_sandbox.GymSandboxDesktopProvider( + {"opensandbox": {}}, + { + "image": "docker://osworld:latest", + "provider_options": {"extensions": {"poolRef": "osworld-kvm"}}, + }, + ) + with pytest.raises(ValueError, match="image-less"): + provider._build_spec( + "/opensandbox/Ubuntu.qcow2", + headless=True, + os_type="Ubuntu", + ) + + def test_provider_rejects_non_docker_config() -> None: - with pytest.raises(ValueError, match="requires Gym's Docker provider"): + with pytest.raises(ValueError, match="Docker or OpenSandbox provider"): osworld_sandbox.GymSandboxDesktopProvider( {"apptainer": {}}, {"image": "osworld:fixed"}, @@ -150,6 +219,56 @@ def test_endpoint_contract_rejects_proxy_headers_and_paths() -> None: ) +def test_local_forwarder_maps_proxy_path_headers_and_cdp_url(monkeypatch) -> None: + seen: dict[str, str] = {} + + class Upstream(http.server.BaseHTTPRequestHandler): + def log_message(self, *args: object) -> None: + del args + + def do_GET(self) -> None: + seen["path"] = self.path + seen["route"] = self.headers.get("X-Route", "") + content = ( + b'{"webSocketDebuggerUrl":' + b'"ws://100.100.1.2:9222/devtools/browser/test"}' + ) + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(content))) + self.end_headers() + self.wfile.write(content) + + upstream = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Upstream) + threading.Thread(target=upstream.serve_forever, daemon=True).start() + monkeypatch.setenv("HTTP_PROXY", "http://127.0.0.1:1") + monkeypatch.setenv("NO_PROXY", "") + forwarder, port = start_forwarder( + f"http://127.0.0.1:{upstream.server_address[1]}/proxy/9222", + {"X-Route": "cell2"}, + ) + try: + with requests.Session() as session: + session.trust_env = False + response = session.get( + f"http://127.0.0.1:{port}/json/version", + timeout=10, + ) + assert response.status_code == 200 + assert seen == { + "path": "/proxy/9222/json/version", + "route": "cell2", + } + assert response.json()["webSocketDebuggerUrl"] == ( + f"ws://127.0.0.1:{port}/devtools/browser/test" + ) + finally: + forwarder.shutdown() + forwarder.server_close() + upstream.shutdown() + upstream.server_close() + + def test_lifecycle_recreates_from_snapshot_and_close_is_idempotent(tmp_path, monkeypatch) -> None: FakeSandbox.instances.clear() monkeypatch.setattr(osworld_sandbox, "Sandbox", FakeSandbox) @@ -190,7 +309,14 @@ def endpoint(self, port: int) -> SandboxEndpoint: {"docker": {}}, {"image": "osworld:fixed"}, ) + monkeypatch.setattr( + osworld_sandbox, + "start_forwarder", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + RuntimeError("forwarder failed") + ), + ) - with pytest.raises(ValueError, match="requires headers"): + with pytest.raises(RuntimeError, match="forwarder failed"): provider.start_emulator(str(vm_path), headless=True, os_type="Ubuntu") assert BadEndpointSandbox.instances[-1].stopped == 1 diff --git a/tests/unit_tests/test_opensandbox_provider.py b/tests/unit_tests/test_opensandbox_provider.py index f310185d63..889c8d6c40 100644 --- a/tests/unit_tests/test_opensandbox_provider.py +++ b/tests/unit_tests/test_opensandbox_provider.py @@ -239,6 +239,218 @@ async def test_direct_create_passes_image_auth_to_sdk_create( assert image.image == "registry.example/repo:tag" assert image.auth.username == "user" assert image.auth.password == TEST_REGISTRY_PASSWORD +async def test_pool_create_uses_cell2_auth_without_execd_connect( + fake_opensandbox_sdk: None, + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls: list[dict[str, Any]] = [] + + async def fake_rest_request( + method: str, + url: str, + *, + json_body: Any | None = None, + headers: dict[str, str] | None = None, + timeout_s: float = 300.0, + ) -> tuple[int, str]: + calls.append( + { + "method": method, + "url": url, + "json_body": json_body, + "headers": headers, + "timeout_s": timeout_s, + } + ) + return 201, '{"id": "pooled-sandbox-1"}' + + monkeypatch.setattr(opensandbox_provider, "_rest_request", fake_rest_request) + provider = opensandbox_provider.OpenSandboxProvider( + connection={ + "domain": "http://sandbox.example", + "api_key": "cell2-key", # pragma: allowlist secret + "request_timeout_s": 30, + }, + create={"request_timeout_s": 120, "timeout_s": 30}, + probe={"command": None}, + ) + handle = await provider.create( + SandboxSpec( + ttl_s=1800, + metadata={"purpose": "osworld"}, + provider_options={"extensions": {"poolRef": "osworld-kvm"}}, + ) + ) + + assert handle.sandbox_id == "pooled-sandbox-1" + assert isinstance(handle.raw, opensandbox_provider._PooledRestSandbox) + assert FakeSandbox.connected_args == () + assert calls[0]["method"] == "POST" + assert calls[0]["url"] == "http://sandbox.example/v1/sandboxes" + assert calls[0]["headers"] == { + "OPEN-SANDBOX-API-KEY": "cell2-key" # pragma: allowlist secret + } + assert calls[0]["json_body"]["timeout"] == 1800 + assert calls[0]["json_body"]["extensions"] == {"poolRef": "osworld-kvm"} + assert calls[0]["json_body"]["metadata"]["purpose"] == "osworld" + assert calls[0]["json_body"]["metadata"][opensandbox_provider.POOLED_CREATE_MARKER_KEY] + + +async def test_pool_connect_cancellation_deletes_known_sandbox( + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls: list[tuple[str, str]] = [] + + async def fake_rest_request( + method: str, + url: str, + **_kwargs: Any, + ) -> tuple[int, str]: + calls.append((method, url)) + if method == "POST": + return 201, '{"id": "pooled-cancelled-connect"}' + if method == "DELETE": + return 204, "" + raise AssertionError(f"unexpected request: {method} {url}") + + monkeypatch.setattr(opensandbox_provider, "_rest_request", fake_rest_request) + provider = opensandbox_provider.OpenSandboxProvider( + connection={"domain": "http://sandbox.example"}, + probe={"command": None}, + ) + + async def cancelled_verify(_handle: Any) -> None: + raise asyncio.CancelledError + + monkeypatch.setattr(provider, "_verify_created_handle", cancelled_verify) + + with pytest.raises(asyncio.CancelledError): + await provider._create_once( + SandboxSpec( + ready_timeout_s=30, + provider_options={ + "extensions": { + "poolRef": "osworld-kvm", + } + }, + ) + ) + + assert calls == [ + ("POST", "http://sandbox.example/v1/sandboxes"), + ( + "DELETE", + "http://sandbox.example/v1/sandboxes/pooled-cancelled-connect", + ), + ] + + +async def test_pool_post_cancellation_reaps_exact_create_marker( + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls: list[tuple[str, str]] = [] + create_marker = "" + + async def fake_rest_request( + method: str, + url: str, + *, + json_body: Any | None = None, + **_kwargs: Any, + ) -> tuple[int, str]: + nonlocal create_marker + calls.append((method, url)) + if method == "POST": + create_marker = str( + json_body["metadata"][ + opensandbox_provider.POOLED_CREATE_MARKER_KEY + ] + ) + raise asyncio.CancelledError + if method == "GET": + if "page=1&" in url: + return ( + 200, + ( + '{"items":[{"id":"some-other-create",' + '"metadata":{"nemo-gym-create-id":"other"}}],' + '"pagination":{"page":1,"pageSize":200,' + '"totalPages":2,"hasNextPage":true}}' + ), + ) + return ( + 200, + ( + '{"items":[{"id":"lost-create",' + f'"metadata":{{"{opensandbox_provider.POOLED_CREATE_MARKER_KEY}":' + f'"{create_marker}"}}}}],' + '"pagination":{"page":2,"pageSize":200,' + '"totalPages":2,"hasNextPage":false}}' + ), + ) + if method == "DELETE": + return 204, "" + raise AssertionError(f"unexpected request: {method} {url}") + + monkeypatch.setattr(opensandbox_provider, "_rest_request", fake_rest_request) + provider = opensandbox_provider.OpenSandboxProvider( + connection={"domain": "http://sandbox.example"}, + probe={"command": None}, + ) + + with pytest.raises(asyncio.CancelledError): + await provider._create_once( + SandboxSpec( + provider_options={ + "extensions": { + "poolRef": "osworld-kvm", + } + }, + ) + ) + + assert create_marker + assert calls == [ + ("POST", "http://sandbox.example/v1/sandboxes"), + ("GET", "http://sandbox.example/v1/sandboxes?page=1&pageSize=200"), + ("GET", "http://sandbox.example/v1/sandboxes?page=2&pageSize=200"), + ("DELETE", "http://sandbox.example/v1/sandboxes/lost-create"), + ] + + +async def test_pool_create_requires_pool_ref( + fake_opensandbox_sdk: None, +) -> None: + provider = opensandbox_provider.OpenSandboxProvider(probe={"command": None}) + with pytest.raises(ValueError, match="poolRef"): + await provider._create_once(SandboxSpec()) + + +async def test_endpoint_normalizes_cell2_scheme() -> None: + class FakeRaw: + async def get_endpoint(self, port: int) -> Any: + assert port == 5000 + return SimpleNamespace( + endpoint="100.100.232.228:5000", + headers={"X-Route": "sandbox"}, + ) + + provider = opensandbox_provider.OpenSandboxProvider( + connection={"protocol": "http"}, + operations={"retries": 0}, + probe={"command": None}, + ) + resolved = await provider.endpoint( + opensandbox_provider.SandboxHandle( + sandbox_id="sandbox-1", + provider_name="opensandbox", + raw=FakeRaw(), + ), + 5000, + ) + + assert resolved.endpoint == "http://100.100.232.228:5000" + assert resolved.headers == {"X-Route": "sandbox"} def test_provider_validation_and_retry_helpers() -> None: diff --git a/tests/unit_tests/test_sandbox.py b/tests/unit_tests/test_sandbox.py index 02a63ba607..19d20ce056 100644 --- a/tests/unit_tests/test_sandbox.py +++ b/tests/unit_tests/test_sandbox.py @@ -709,6 +709,28 @@ async def never_finishes() -> None: runner.close() +def test_sync_loop_runner_waits_for_async_cancellation_cleanup() -> None: + runner = _AsyncLoopRunner( + wait_timeout_s=0.01, + cancellation_timeout_s=1.0, + ) + cleanup_finished = threading.Event() + + async def needs_async_cleanup() -> None: + try: + await asyncio.get_running_loop().create_future() + finally: + await asyncio.sleep(0.05) + cleanup_finished.set() + + try: + with pytest.raises(TimeoutError, match="timed out waiting for the sync loop"): + runner.run("blocked", needs_async_cleanup) + assert cleanup_finished.is_set() + finally: + runner.close() + + def test_sync_sandbox_file_operations(tmp_path: Path) -> None: provider = FakeSandboxProvider() with Sandbox(provider) as sandbox: From aa648cff93f2b2f44a730227b7f80ae2782e1b2e Mon Sep 17 00:00:00 2001 From: Jeff Peng Date: Tue, 4 Aug 2026 20:58:08 +0800 Subject: [PATCH 02/10] fix(osworld): publish OpenSandbox runtime contract Signed-off-by: Jeff Peng --- benchmarks/osworld/README.md | 117 ++++++++++++-- .../osworld/configs/osworld_opensandbox.yaml | 8 +- .../tests/test_cleanup_opensandbox_run.py | 101 ++++++++++++ benchmarks/osworld/tests/test_prepare.py | 2 +- benchmarks/osworld/tests/test_run_scripts.py | 17 +- benchmarks/osworld/tools/README.md | 15 +- .../osworld/tools/cleanup_opensandbox_run.py | 153 ++++++++++++++++++ benchmarks/osworld/tools/cleanup_run.sh | 16 ++ benchmarks/osworld/tools/start_control.sh | 1 + .../osworld_agent/pyproject.toml | 76 --------- .../osworld_agent/requirements.txt | 42 +++++ .../tests/test_sandbox_provider.py | 4 +- tests/unit_tests/test_opensandbox_provider.py | 14 +- 13 files changed, 456 insertions(+), 110 deletions(-) create mode 100644 benchmarks/osworld/tests/test_cleanup_opensandbox_run.py create mode 100644 benchmarks/osworld/tools/cleanup_opensandbox_run.py delete mode 100644 responses_api_agents/osworld_agent/pyproject.toml create mode 100644 responses_api_agents/osworld_agent/requirements.txt diff --git a/benchmarks/osworld/README.md b/benchmarks/osworld/README.md index 2cf90a2048..8ec148d783 100644 --- a/benchmarks/osworld/README.md +++ b/benchmarks/osworld/README.md @@ -9,23 +9,28 @@ The reusable Responses API runtime lives in [`responses_api_agents/osworld_agent`](../../responses_api_agents/osworld_agent/README.md). That README is the source of truth for request/response semantics, supported runners, agent ownership, parser contracts, and runtime configuration. The -runtime uses an unmodified, pinned OSWorld dependency. In the current deployment -path, Gym Docker Sandbox owns the VM container lifecycle while OSWorld keeps -its setup, action, and evaluator behavior intact. +runtime uses an unmodified, pinned OSWorld dependency. Gym's selected Docker or +OpenSandbox provider owns the VM lifecycle while OSWorld keeps its setup, +action, and evaluator behavior intact. ## Deployment roles The runtime has three explicit roles: ```text -MODEL_HOST ←─ model HTTP ─ AGENT_CONTROL_HOST ─ Docker SSH → ENVIRONMENT_HOST - prepare / control / eval Docker / KVM / VM +MODEL_HOST ←─ model HTTP ─ AGENT_CONTROL_HOST + prepare / control / eval + │ + ├─ Docker / Docker SSH → ENVIRONMENT_HOST → KVM VM + └─ OpenSandbox API → server-managed KVM Pool → VM ``` - The model host serves a compatible vision-language model. It does not run Gym eval or OSWorld VM containers. -- The environment host needs Docker, `/dev/kvm`, and the verified qcow2. It - does not need the benchmark or agent checkout. +- In the Docker path, the environment host needs Docker, `/dev/kvm`, and the + verified qcow2. It does not need the benchmark or agent checkout. +- In the OpenSandbox path, the management service and pre-provisioned Pool own + the environment capacity, VM image, and entrypoint. - The agent/control host needs this Gym checkout, the task input, and the same qcow2 path for preparation-time identity validation. It runs `prepare.py`, `tools/start_control.sh`, and `tools/run_eval.sh`. @@ -43,7 +48,10 @@ interactive SSH session is part of runtime communication. - About 30 GB free disk for the Docker image and `Ubuntu.qcow2` cache. - A reachable vision-language model endpoint. Text-only models cannot act on screenshot observations. -- Read/write access to `/dev/kvm` for the Gym Docker Sandbox path. +- For the Gym Docker path, Docker and read/write access to `/dev/kvm` on the + environment host. +- For the OpenSandbox path, a reachable API and a pre-provisioned OSWorld KVM + Pool; Docker, KVM, the VM image, and its entrypoint remain server-side. - A C compiler and Python development headers on the agent/control host. On Ubuntu, install `build-essential` and `python3-dev`; Gym's first server start builds the pinned `evdev` dependency in its managed environment. @@ -197,6 +205,82 @@ not repeated. This prompt contract is implemented directly by the standard `NemotronV3NanoOmniAgent`, so deployments must not stage a Python subclass or extend `PYTHONPATH` with a reproduction overlay. +### OpenSandbox Pool backend + +The `gym_opensandbox` backend keeps the same OSWorld agent, task setup, action, +and evaluator path while replacing the local or remote Docker lifecycle with +an allocation from a server-managed KVM Pool. The Pool operator owns its VM +image, entrypoint, and capacity. Consequently, clients provide neither a +container image nor `--vm-path`; they only name a pre-provisioned Pool. The +checked-in default is `osworld-kvm`, overridable with +`OPENSANDBOX_POOL_REF`. + +Install Gym with the OpenSandbox SDK and configure the management endpoint. +Keep credentials in the environment; `prepare.py` does not write the API key +to `env.yaml`: + +```bash +uv sync --extra dev --extra sandbox +export OPENSANDBOX_BASE_URL=opensandbox.example.com:8080 +export OPENSANDBOX_API_KEY=YOUR_OPENSANDBOX_API_KEY +export OPENSANDBOX_POOL_REF=osworld-kvm +``` + +Verify the model endpoint with the request shape used by Nano Omni, then +prepare a run. The input, output, and server-environment paths are client-owned +durable paths; choose them outside a small login home when running on a shared +cluster: + +```bash +python3 benchmarks/osworld/tools/probe_model_endpoint.py \ + --base-url http://MODEL_HOST:8000/v1 \ + --api-key local-vllm \ + --model SERVED_NANO_OMNI_MODEL \ + --image-count 3 + +cd benchmarks/osworld +python3 prepare.py \ + --profile nano_omni \ + --execution-backend gym_opensandbox \ + --input /absolute/path/to/tasks.jsonl \ + --output /absolute/run/root/results/rollouts.jsonl \ + --server-venv-root /absolute/run/root/server-venvs \ + --policy-base-url http://MODEL_HOST:8000/v1 \ + --policy-api-key local-vllm \ + --policy-model-name SERVED_NANO_OMNI_MODEL \ + --force-env +``` + +OpenSandbox may return path-based gateway endpoints with required routing +headers rather than directly routable Pod addresses. The adapter creates +client-local forwarders that preserve those paths and headers for OSWorld's +HTTP services and Chrome CDP WebSockets. The pinned OSWorld revision also +falls back to the guest loopback address when VLC status authentication is +reached through such a gateway, so Chrome, direct desktop, and VLC evaluators +use the same backend without a local source overlay. + +Start control and eval with the normal wrappers: + +```bash +export OSWORLD_RUN_ID=my-osworld-opensandbox-run +tools/start_control.sh /absolute/run/root +# After control is ready, in a second terminal: +tools/run_eval.sh /absolute/run/root +``` + +Normal Gym shutdown releases the Sandbox. If an interrupted run leaves an +instance behind, keep the OpenSandbox variables exported and invoke +`tools/cleanup_run.sh /absolute/run/root`. The wrapper stops only processes +whose recorded environment matches `OSWORLD_RUN_ID`, then queries both the +OSWorld and Gym run metadata keys and rechecks exact values before requesting +termination. To audit without changing remote state, run from the repository +root without `--reap`: + +```bash +.venv/bin/python benchmarks/osworld/tools/cleanup_opensandbox_run.py \ + --run-id my-osworld-opensandbox-run +``` + ## Multi-environment runs Set concurrency and data selection during preparation, then use the same two @@ -365,19 +449,20 @@ model-specific and do not change defaults for other runners. ### Proxy-required tasks -Proxy policy belongs to the Gym `feature/osworld` adapter; VM setup belongs to -the pinned OSWorld `nv-gym` runtime. The branch names are independent Git refs -in different repositories. Gym connects them only through the immutable -OSWorld commit in `pyproject.toml`. That OSWorld commit merges upstream main -`83e85344` and retains the `nv-gym` integration overlay. +Proxy policy belongs to the Gym OSWorld adapter; VM setup belongs to the pinned +OSWorld `nv-gym` runtime. The integration lines are independent Git refs in +different repositories. Gym connects them only through the immutable OSWorld +commit in `responses_api_agents/osworld_agent/requirements.txt`. That OSWorld +commit merges upstream main `83e85344` and retains the `nv-gym` integration +overlay. #### OSWorld version selection | Consumer workflow | Required OSWorld version | | --- | --- | -| Gym `feature/osworld` | No manual checkout. The agent package installs the exact SHA from `pyproject.toml`. | +| Gym OSWorld benchmark | No manual checkout. The agent package installs the exact SHA from `responses_api_agents/osworld_agent/requirements.txt`. | | Direct OSWorld, plain Docker/VMware, no proxy-required tasks | Upstream xlang OSWorld main is sufficient; this adapter's pre-fix baseline was `83e8534451ba8b3ab6477448ef3f0a8e563f05be`. | -| Direct OSWorld with `provider_name=remote_docker` | `JeffPengCoder/OSWorld` `nv-gym`, pinned to `31b76bf1c4d4e589238b314caa91470afc52651e` or a documented successor. | +| Direct OSWorld with `provider_name=remote_docker` | `JeffPengCoder/OSWorld` `nv-gym`, pinned to `dc23424e9f6316b181bde149e0dc9bc3c5ff78c9` or a documented successor. | | Direct OSWorld with proxy-required tasks | The same `nv-gym` pinned SHA; set `PROXY_CONFIG_FILE` and construct `DesktopEnv(enable_proxy=True)`. | | Direct OSWorld with both features | The same `nv-gym` pinned SHA provides both independent capabilities. | @@ -386,7 +471,7 @@ For a direct integration of the tested version: ```bash git clone https://github.com/JeffPengCoder/OSWorld.git cd OSWorld -git checkout 31b76bf1c4d4e589238b314caa91470afc52651e +git checkout dc23424e9f6316b181bde149e0dc9bc3c5ff78c9 ``` Use an immutable SHA in a lockfile or deployment manifest. The `nv-gym` diff --git a/benchmarks/osworld/configs/osworld_opensandbox.yaml b/benchmarks/osworld/configs/osworld_opensandbox.yaml index 6d2e95590f..148337c040 100644 --- a/benchmarks/osworld/configs/osworld_opensandbox.yaml +++ b/benchmarks/osworld/configs/osworld_opensandbox.yaml @@ -1,4 +1,4 @@ -# Cell-2 OpenSandbox lifecycle provider for the server-side OSWorld KVM Pool. +# OpenSandbox lifecycle provider for a server-side OSWorld KVM Pool. # Credentials stay in environment variables; generated env.yaml files never # contain the API key. @@ -12,9 +12,9 @@ osworld_opensandbox: api_key: ${oc.env:OPENSANDBOX_API_KEY} protocol: http request_timeout_s: 300 - # External controllers cannot route Cell-2's 100.100/16 Pod CIDR. - # The OSWorld adapter supplies local host:port forwarders for these - # path-based gateway endpoints, including Chrome CDP WebSockets. + # External controllers may not route a Pool's private Pod network. The + # OSWorld adapter supplies local host:port forwarders for path-based + # gateway endpoints, including Chrome CDP WebSockets. use_server_proxy: true create: request_timeout_s: 1200 diff --git a/benchmarks/osworld/tests/test_cleanup_opensandbox_run.py b/benchmarks/osworld/tests/test_cleanup_opensandbox_run.py new file mode 100644 index 0000000000..affd0035e2 --- /dev/null +++ b/benchmarks/osworld/tests/test_cleanup_opensandbox_run.py @@ -0,0 +1,101 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from types import SimpleNamespace +from unittest.mock import Mock, call + +from benchmarks.osworld.tools import cleanup_opensandbox_run + + +class _Filter: + def __init__(self, **kwargs: object) -> None: + self.metadata = kwargs["metadata"] + self.page = kwargs["page"] + self.page_size = kwargs["page_size"] + + +class _Manager: + def __init__(self, pages: list[object]) -> None: + self.pages = iter(pages) + self.filters: list[_Filter] = [] + + def list_sandbox_infos(self, sandbox_filter: _Filter) -> object: + self.filters.append(sandbox_filter) + return next(self.pages) + + +def _info(sandbox_id: str, metadata: dict[str, str]) -> object: + return SimpleNamespace(id=sandbox_id, metadata=metadata) + + +def _page(infos: list[object], *, has_next_page: bool = False) -> object: + return SimpleNamespace( + sandbox_infos=infos, + pagination=SimpleNamespace(has_next_page=has_next_page), + ) + + +def test_list_exact_ids_paginates_deduplicates_and_rechecks_metadata() -> None: + manager = _Manager( + [ + _page( + [ + _info("sandbox-b", {"run-id": "run-7"}), + _info("wrong-run", {"run-id": "run-70"}), + ], + has_next_page=True, + ), + _page([_info("sandbox-a", {"run-id": "run-7"})]), + _page( + [ + _info("sandbox-a", {"nemo-gym.nvidia.com/run": "run-7"}), + _info("sandbox-c", {"nemo-gym.nvidia.com/run": "run-7"}), + ] + ), + ] + ) + + assert cleanup_opensandbox_run._list_exact_ids(manager, _Filter, "run-7") == [ + "sandbox-a", + "sandbox-b", + "sandbox-c", + ] + assert [(item.metadata, item.page, item.page_size) for item in manager.filters] == [ + ({"run-id": "run-7"}, 1, 200), + ({"run-id": "run-7"}, 2, 200), + ({"nemo-gym.nvidia.com/run": "run-7"}, 1, 200), + ] + + +def test_reap_kills_only_initial_exact_ids_and_waits_until_none(monkeypatch) -> None: + manager = Mock() + matches = iter( + [ + ["sandbox-a", "sandbox-b"], + ["sandbox-a"], + [], + ] + ) + monkeypatch.setattr( + cleanup_opensandbox_run, + "_list_exact_ids", + lambda *_args: next(matches), + ) + monkeypatch.setattr(cleanup_opensandbox_run.time, "sleep", lambda _seconds: None) + + report = cleanup_opensandbox_run._reap_exact_ids( + manager, + _Filter, + "run-7", + timeout_s=10, + poll_s=0.01, + ) + + assert manager.kill_sandbox.call_args_list == [call("sandbox-a"), call("sandbox-b")] + assert report == { + "run_id": "run-7", + "matched_ids": ["sandbox-a", "sandbox-b"], + "kill_errors": {}, + "remaining_ids": [], + "all_gone": True, + } diff --git a/benchmarks/osworld/tests/test_prepare.py b/benchmarks/osworld/tests/test_prepare.py index 0312bd833f..e8391f196d 100644 --- a/benchmarks/osworld/tests/test_prepare.py +++ b/benchmarks/osworld/tests/test_prepare.py @@ -125,7 +125,7 @@ def test_nano_omni_profile_is_one_complete_benchmark_config() -> None: assert paths == (NANO_OMNI_AGENT_CONFIG.resolve(),) -def test_opensandbox_backend_adds_cell2_provider_config() -> None: +def test_opensandbox_backend_adds_pool_provider_config() -> None: paths = select_config_paths( profile="nano_omni", execution_backend="gym_opensandbox", diff --git a/benchmarks/osworld/tests/test_run_scripts.py b/benchmarks/osworld/tests/test_run_scripts.py index ef21818ab0..ac56b3cb25 100644 --- a/benchmarks/osworld/tests/test_run_scripts.py +++ b/benchmarks/osworld/tests/test_run_scripts.py @@ -2,7 +2,6 @@ # SPDX-License-Identifier: Apache-2.0 import subprocess -import tomllib from pathlib import Path import pytest @@ -15,8 +14,9 @@ START_CONTROL_SCRIPT = REPO_ROOT / "benchmarks/osworld/tools/start_control.sh" RUN_EVAL_SCRIPT = REPO_ROOT / "benchmarks/osworld/tools/run_eval.sh" CLEANUP_RUN_SCRIPT = REPO_ROOT / "benchmarks/osworld/tools/cleanup_run.sh" +OPENSANDBOX_CLEANUP_SCRIPT = REPO_ROOT / "benchmarks/osworld/tools/cleanup_opensandbox_run.py" OSWORLD_AGENT_CONFIG = REPO_ROOT / "responses_api_agents/osworld_agent/configs/osworld_agent.yaml" -OSWORLD_AGENT_PYPROJECT = REPO_ROOT / "responses_api_agents/osworld_agent/pyproject.toml" +OSWORLD_AGENT_REQUIREMENTS = REPO_ROOT / "responses_api_agents/osworld_agent/requirements.txt" @pytest.mark.parametrize( @@ -38,6 +38,7 @@ def test_runtime_wrappers_delegate_to_current_gym_commands() -> None: start_control = START_CONTROL_SCRIPT.read_text(encoding="utf-8") assert "env start \\" in start_control assert "model-io.jsonl" not in start_control + assert "NEMO_GYM_RUN_ID=${NEMO_GYM_RUN_ID:-${RUN_ID}}" in start_control assert "eval run --no-serve \\" in RUN_EVAL_SCRIPT.read_text(encoding="utf-8") @@ -50,9 +51,11 @@ def test_start_control_preflights_native_build_toolchain() -> None: def test_managed_osworld_agent_installs_opensandbox_sdk() -> None: - project = tomllib.loads(OSWORLD_AGENT_PYPROJECT.read_text(encoding="utf-8")) + requirements = OSWORLD_AGENT_REQUIREMENTS.read_text(encoding="utf-8").splitlines() - assert "nemo-gym[dev,sandbox]" in project["project"]["dependencies"] + assert "-e nemo-gym[dev] @ ../../" in requirements + assert "opensandbox>=0.1.15" in requirements + assert "tenacity>=9.1.4" in requirements def test_remote_docker_requires_a_reachable_publish_host() -> None: @@ -85,4 +88,10 @@ def test_cleanup_is_scoped_to_the_run_id() -> None: assert 'rm -f "${pid_file}"' in text assert "label=nemo-gym.run-id=${RUN_ID}" in text assert "nemo-gym.workload=osworld" in text + assert '"${OPENSANDBOX_CLEANUP}" --run-id "${RUN_ID}" --reap' in text assert "logs and results were preserved" in text + + opensandbox_text = OPENSANDBOX_CLEANUP_SCRIPT.read_text(encoding="utf-8") + compile(opensandbox_text, str(OPENSANDBOX_CLEANUP_SCRIPT), "exec") + assert 'RUN_METADATA_KEYS = ("run-id", "nemo-gym.nvidia.com/run")' in opensandbox_text + assert "SandboxManagerSync" in opensandbox_text diff --git a/benchmarks/osworld/tools/README.md b/benchmarks/osworld/tools/README.md index abbdde1e8c..d45362e957 100644 --- a/benchmarks/osworld/tools/README.md +++ b/benchmarks/osworld/tools/README.md @@ -8,7 +8,7 @@ configuration entry point; host checks and lifecycle wrappers live here: model host -> probe_model_endpoint.py environment host -> check_environment.sh agent/control -> prepare.py -> start_control.sh -> run_eval.sh -abnormal recovery -> cleanup_run.sh +abnormal recovery -> cleanup_run.sh -> cleanup_opensandbox_run.py ``` | Tool | Purpose | @@ -18,6 +18,7 @@ abnormal recovery -> cleanup_run.sh | `start_control.sh` | Preflight the agent/control build toolchain, then run `gym env start` | | `run_eval.sh` | Supervisor-friendly wrapper around `gym eval run --no-serve` | | `cleanup_run.sh` | Recovery-only cleanup for stale processes or labeled Sandbox containers after abnormal termination | +| `cleanup_opensandbox_run.py` | Read-only audit or opt-in reaping of OpenSandbox instances matching one exact run ID | | `prepare_osworld_vm.sh` | Download and verify the pinned OSWorld qcow2 baseline | Model serving itself belongs to the selected model's deployment project; @@ -89,3 +90,15 @@ validates both the PID environment and command before signaling it. Docker cleanup requires the Sandbox, OSWorld workload, and run-ID labels to match; it does not remove unlabeled or other-run containers. Model services are outside this lifecycle and are never stopped by this tool. + +When both `OPENSANDBOX_BASE_URL` and `OPENSANDBOX_API_KEY` are set, +`cleanup_run.sh` also invokes the OpenSandbox reaper through the Gym Python +environment. The reaper queries both OSWorld's `run-id` metadata and Gym's +`nemo-gym.nvidia.com/run` attribution metadata, then rechecks returned metadata +client-side before terminating anything. Run it without `--reap` for a +read-only audit: + +```bash +export OSWORLD_RUN_ID=my-osworld-run +.venv/bin/python benchmarks/osworld/tools/cleanup_opensandbox_run.py +``` diff --git a/benchmarks/osworld/tools/cleanup_opensandbox_run.py b/benchmarks/osworld/tools/cleanup_opensandbox_run.py new file mode 100644 index 0000000000..ca9970d466 --- /dev/null +++ b/benchmarks/osworld/tools/cleanup_opensandbox_run.py @@ -0,0 +1,153 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Audit or reap OpenSandbox instances owned by one exact OSWorld run ID.""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import time +from datetime import timedelta +from typing import Any + + +RUN_METADATA_KEYS = ("run-id", "nemo-gym.nvidia.com/run") +RUN_ID_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]*$") + + +def _positive_float(value: str) -> float: + parsed = float(value) + if parsed <= 0: + raise argparse.ArgumentTypeError("must be greater than zero") + return parsed + + +def _require_sdk() -> tuple[Any, Any, Any]: + try: + from opensandbox import SandboxManagerSync + from opensandbox.config import ConnectionConfigSync + from opensandbox.models.sandboxes import SandboxFilter + except ImportError as error: + raise RuntimeError( + "OpenSandbox cleanup requires the Gym sandbox extra; " + "run `uv sync --extra sandbox` or set GYM_PYTHON accordingly" + ) from error + return SandboxManagerSync, ConnectionConfigSync, SandboxFilter + + +def _list_exact_ids(manager: Any, sandbox_filter: Any, run_id: str) -> list[str]: + """Return IDs whose returned metadata exactly matches either Gym run key.""" + + matched: set[str] = set() + for metadata_key in RUN_METADATA_KEYS: + page = 1 + while True: + result = manager.list_sandbox_infos( + sandbox_filter( + metadata={metadata_key: run_id}, + page=page, + page_size=200, + ) + ) + for info in result.sandbox_infos: + metadata = info.metadata or {} + if metadata.get(metadata_key) == run_id: + matched.add(str(info.id)) + if not result.pagination.has_next_page: + break + page += 1 + return sorted(matched) + + +def _reap_exact_ids( + manager: Any, + sandbox_filter: Any, + run_id: str, + *, + timeout_s: float, + poll_s: float, +) -> dict[str, Any]: + """Kill all current exact matches and wait until the list API reports none.""" + + matched_ids = _list_exact_ids(manager, sandbox_filter, run_id) + kill_errors: dict[str, str] = {} + for sandbox_id in matched_ids: + try: + manager.kill_sandbox(sandbox_id) + except Exception as error: # noqa: BLE001 - report every fleet operation and continue + kill_errors[sandbox_id] = type(error).__name__ + + deadline = time.monotonic() + timeout_s + remaining_ids = _list_exact_ids(manager, sandbox_filter, run_id) + while remaining_ids and time.monotonic() < deadline: + time.sleep(min(poll_s, max(0.0, deadline - time.monotonic()))) + remaining_ids = _list_exact_ids(manager, sandbox_filter, run_id) + + return { + "run_id": run_id, + "matched_ids": matched_ids, + "kill_errors": kill_errors, + "remaining_ids": remaining_ids, + "all_gone": not remaining_ids, + } + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Audit or reap OpenSandbox instances matching one exact OSWorld run ID." + ) + parser.add_argument("--run-id", default=os.environ.get("OSWORLD_RUN_ID")) + parser.add_argument( + "--reap", + action="store_true", + help="Terminate exact matches; without this flag the command is read-only.", + ) + parser.add_argument("--timeout-seconds", type=_positive_float, default=120.0) + parser.add_argument("--poll-seconds", type=_positive_float, default=2.0) + args = parser.parse_args() + if not args.run_id: + parser.error("--run-id or OSWORLD_RUN_ID is required") + if not RUN_ID_PATTERN.fullmatch(args.run_id): + parser.error("run ID must match [A-Za-z0-9][A-Za-z0-9_.-]*") + return args + + +def main() -> int: + args = _parse_args() + base_url = os.environ.get("OPENSANDBOX_BASE_URL", "").strip() + api_key = os.environ.get("OPENSANDBOX_API_KEY", "").strip() + protocol = os.environ.get("OPENSANDBOX_PROTOCOL", "http").strip().lower() + if not base_url or not api_key: + raise SystemExit("Set both OPENSANDBOX_BASE_URL and OPENSANDBOX_API_KEY") + + SandboxManagerSync, ConnectionConfigSync, SandboxFilter = _require_sdk() + config = ConnectionConfigSync( + domain=base_url, + api_key=api_key, + protocol=protocol, + request_timeout=timedelta(seconds=60), + ) + manager = SandboxManagerSync.create(connection_config=config) + try: + if args.reap: + report = _reap_exact_ids( + manager, + SandboxFilter, + args.run_id, + timeout_s=args.timeout_seconds, + poll_s=args.poll_seconds, + ) + print(json.dumps(report, sort_keys=True)) + return 0 if report["all_gone"] else 1 + + matched_ids = _list_exact_ids(manager, SandboxFilter, args.run_id) + print(json.dumps({"run_id": args.run_id, "matched_ids": matched_ids}, sort_keys=True)) + return 0 + finally: + manager.close() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/osworld/tools/cleanup_run.sh b/benchmarks/osworld/tools/cleanup_run.sh index f5af5599bb..3629ec8067 100755 --- a/benchmarks/osworld/tools/cleanup_run.sh +++ b/benchmarks/osworld/tools/cleanup_run.sh @@ -3,6 +3,9 @@ set -Eeuo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" GYM_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" +GYM_BIN=${GYM_BIN:-${GYM_ROOT}/.venv/bin/gym} +GYM_PYTHON=${GYM_PYTHON:-$(dirname "${GYM_BIN}")/python} +OPENSANDBOX_CLEANUP=${SCRIPT_DIR}/cleanup_opensandbox_run.py RUN_ROOT=${1:-${OSWORLD_RUN_ROOT:-${GYM_ROOT}}} RUN_ID=${OSWORLD_RUN_ID:?set OSWORLD_RUN_ID} STATE_DIR=${RUN_ROOT}/run/osworld/${RUN_ID} @@ -85,6 +88,19 @@ stop_role() { stop_role eval "eval run" stop_role control "env start" +if [[ -n "${OPENSANDBOX_BASE_URL:-}" || -n "${OPENSANDBOX_API_KEY:-}" ]]; then + if [[ -z "${OPENSANDBOX_BASE_URL:-}" || -z "${OPENSANDBOX_API_KEY:-}" ]]; then + echo "Set both OPENSANDBOX_BASE_URL and OPENSANDBOX_API_KEY for OpenSandbox cleanup" >&2 + exit 2 + fi + if [[ ! -x "${GYM_PYTHON}" ]]; then + echo "Gym Python is not executable: ${GYM_PYTHON}" >&2 + echo "Set GYM_BIN or GYM_PYTHON to the environment installed with the sandbox extra" >&2 + exit 2 + fi + "${GYM_PYTHON}" "${OPENSANDBOX_CLEANUP}" --run-id "${RUN_ID}" --reap +fi + if command -v docker >/dev/null 2>&1; then container_ids=$(docker ps -aq \ --filter "label=nemo-gym.sandbox=1" \ diff --git a/benchmarks/osworld/tools/start_control.sh b/benchmarks/osworld/tools/start_control.sh index 7f236e83bc..7d58ffd5f2 100755 --- a/benchmarks/osworld/tools/start_control.sh +++ b/benchmarks/osworld/tools/start_control.sh @@ -52,6 +52,7 @@ exec >>"${RUN_ROOT}/logs/control-${RUN_ID}.log" 2>&1 export PYTHONPATH="${GYM_ROOT}${PYTHONPATH:+:${PYTHONPATH}}" export OSWORLD_RUN_ID=${RUN_ID} +export NEMO_GYM_RUN_ID=${NEMO_GYM_RUN_ID:-${RUN_ID}} export OSWORLD_TASK_ARTIFACT_ROOT=${OSWORLD_TASK_ARTIFACT_ROOT:-${RUN_ROOT}/results/${RUN_ID}/tasks} export OSWORLD_RESOURCES_IO_LOG=${OSWORLD_RESOURCES_IO_LOG:-${RUN_ROOT}/results/${RUN_ID}/resources-io.jsonl} export OSWORLD_VM_EXEC_LOG=${OSWORLD_VM_EXEC_LOG:-${RUN_ROOT}/results/${RUN_ID}/vm-exec.jsonl} diff --git a/responses_api_agents/osworld_agent/pyproject.toml b/responses_api_agents/osworld_agent/pyproject.toml deleted file mode 100644 index fe76257dc1..0000000000 --- a/responses_api_agents/osworld_agent/pyproject.toml +++ /dev/null @@ -1,76 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -[project] -name = "osworld-agent" -version = "0.0.0" -requires-python = ">=3.13.14" -dependencies = [ - # The managed agent server gets its own venv. Include the sandbox extra - # here so OpenSandbox-backed runs have the SDK in that isolated runtime. - "nemo-gym[dev,sandbox]", - # JeffPengCoder/OSWorld nv-gym at an immutable commit: upstream main - # 83e85344 plus the nv-gym provider overlay and explicit proxy-runtime - # repair. Use an archive because uv recursively initializes git - # submodules, while optional SurferH submodules are not needed here. - "osworld @ https://github.com/JeffPengCoder/OSWorld/archive/31b76bf1c4d4e589238b314caa91470afc52651e.tar.gz", - "numkong==7.7.0", - "docker==7.1.0", - # Needed by OSWorld's mm_agents.pointer implementation. - "anthropic", - "boto3", - "parallel-web", - "requests-toolbelt~=1.0.0", - # OSWorld currently pins requests<2.32; prevent RequestsDependencyWarning - # when transitive deps pull chardet 7.x. - "chardet<6", - # OSWorld imports these at runtime, but the repository-level Gym - # pyproject excludes them globally to keep other server venvs lean. Keeping - # this server on an agent-local pyproject lets uv install them here without - # changing the global exclude-dependencies policy. - "cryptography", - "cffi", - "scipy", - "numpy<2", - "pynput", - "ag2", - "opencv-python-headless~=4.8.1.78", - "Pillow~=11.0.0", - "scikit-learn", - "matplotlib~=3.7.4", - "flask~=3.0.0", - "func-timeout", -] - -[build-system] -build-backend = "setuptools.build_meta" -requires = ["setuptools>=61"] - -[tool.setuptools.packages.find] -where = ["../.."] -include = ["responses_api_agents.osworld_agent*"] - -[tool.uv.sources] -nemo-gym = { path = "../..", editable = true } - -[tool.uv] -# Upstream's broad development dependency set includes the GUI OpenCV wheel -# and a SurferH-only git submodule. The OSWorld Gym server is headless and does -# not load SurferH, so keep those optional pieces outside the runtime while -# supplying opencv-python-headless explicitly above. -exclude-dependencies = ["opencv-python", "opencv-contrib-python", "agp-client"] -# osworld pins torch<2.6 but torchvision 0.20.x (for torch 2.5.x) has no cp313 -# wheels. Override to 2.12.0 which has cp313 wheels for both torch and torchvision. -override-dependencies = ["torch>=2.12.0", "torchvision>=0.27.0", "opencv-python-headless~=4.8.1.78"] diff --git a/responses_api_agents/osworld_agent/requirements.txt b/responses_api_agents/osworld_agent/requirements.txt new file mode 100644 index 0000000000..b53e7ac386 --- /dev/null +++ b/responses_api_agents/osworld_agent/requirements.txt @@ -0,0 +1,42 @@ +-e nemo-gym[dev] @ ../../ + +# OSWorld supports Gym's OpenSandbox backend without installing every optional +# Sandbox provider SDK into this managed server environment. +opensandbox>=0.1.15 +tenacity>=9.1.4 + +# JeffPengCoder/OSWorld nv-gym at an immutable commit: upstream main +# 83e85344 plus the nv-gym provider overlay, proxy-runtime repair, and VLC +# gateway-auth fallback. Use an archive because uv recursively initializes +# git submodules, while optional SurferH submodules are not needed here. +osworld @ https://github.com/JeffPengCoder/OSWorld/archive/dc23424e9f6316b181bde149e0dc9bc3c5ff78c9.tar.gz + +numkong==7.7.0 +docker==7.1.0 + +# Needed by OSWorld's mm_agents.pointer implementation. +anthropic +boto3 +parallel-web +requests-toolbelt~=1.0.0 + +# OSWorld currently pins requests<2.32; prevent RequestsDependencyWarning +# when transitive dependencies pull chardet 7.x. +chardet<6 + +# OSWorld imports these packages at runtime but does not declare all of them. +cryptography +cffi +scipy +numpy<2 +pynput +ag2 + +# The root resolver policy removes OSWorld's GUI OpenCV dependency edges; this +# server supplies the compatible headless wheel instead. +opencv-python-headless~=4.8.1.78 +Pillow~=11.0.0 +scikit-learn +matplotlib~=3.7.4 +flask~=3.0.0 +func-timeout diff --git a/responses_api_agents/osworld_agent/tests/test_sandbox_provider.py b/responses_api_agents/osworld_agent/tests/test_sandbox_provider.py index d3c7e34d02..7b0b3bda72 100644 --- a/responses_api_agents/osworld_agent/tests/test_sandbox_provider.py +++ b/responses_api_agents/osworld_agent/tests/test_sandbox_provider.py @@ -245,7 +245,7 @@ def do_GET(self) -> None: monkeypatch.setenv("NO_PROXY", "") forwarder, port = start_forwarder( f"http://127.0.0.1:{upstream.server_address[1]}/proxy/9222", - {"X-Route": "cell2"}, + {"X-Route": "gateway"}, ) try: with requests.Session() as session: @@ -257,7 +257,7 @@ def do_GET(self) -> None: assert response.status_code == 200 assert seen == { "path": "/proxy/9222/json/version", - "route": "cell2", + "route": "gateway", } assert response.json()["webSocketDebuggerUrl"] == ( f"ws://127.0.0.1:{port}/devtools/browser/test" diff --git a/tests/unit_tests/test_opensandbox_provider.py b/tests/unit_tests/test_opensandbox_provider.py index 889c8d6c40..278b7869db 100644 --- a/tests/unit_tests/test_opensandbox_provider.py +++ b/tests/unit_tests/test_opensandbox_provider.py @@ -239,7 +239,9 @@ async def test_direct_create_passes_image_auth_to_sdk_create( assert image.image == "registry.example/repo:tag" assert image.auth.username == "user" assert image.auth.password == TEST_REGISTRY_PASSWORD -async def test_pool_create_uses_cell2_auth_without_execd_connect( + + +async def test_pool_create_uses_api_key_auth_without_execd_connect( fake_opensandbox_sdk: None, monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -268,7 +270,7 @@ async def fake_rest_request( provider = opensandbox_provider.OpenSandboxProvider( connection={ "domain": "http://sandbox.example", - "api_key": "cell2-key", # pragma: allowlist secret + "api_key": "pool-api-key", # pragma: allowlist secret "request_timeout_s": 30, }, create={"request_timeout_s": 120, "timeout_s": 30}, @@ -288,7 +290,7 @@ async def fake_rest_request( assert calls[0]["method"] == "POST" assert calls[0]["url"] == "http://sandbox.example/v1/sandboxes" assert calls[0]["headers"] == { - "OPEN-SANDBOX-API-KEY": "cell2-key" # pragma: allowlist secret + "OPEN-SANDBOX-API-KEY": "pool-api-key" # pragma: allowlist secret } assert calls[0]["json_body"]["timeout"] == 1800 assert calls[0]["json_body"]["extensions"] == {"poolRef": "osworld-kvm"} @@ -426,12 +428,12 @@ async def test_pool_create_requires_pool_ref( await provider._create_once(SandboxSpec()) -async def test_endpoint_normalizes_cell2_scheme() -> None: +async def test_endpoint_normalizes_missing_scheme() -> None: class FakeRaw: async def get_endpoint(self, port: int) -> Any: assert port == 5000 return SimpleNamespace( - endpoint="100.100.232.228:5000", + endpoint="10.0.0.22:5000", headers={"X-Route": "sandbox"}, ) @@ -449,7 +451,7 @@ async def get_endpoint(self, port: int) -> Any: 5000, ) - assert resolved.endpoint == "http://100.100.232.228:5000" + assert resolved.endpoint == "http://10.0.0.22:5000" assert resolved.headers == {"X-Route": "sandbox"} From c012533817dffb94104d542be686e5dcaf8c9392 Mon Sep 17 00:00:00 2001 From: Jeff Peng Date: Wed, 12 Aug 2026 12:48:40 +0800 Subject: [PATCH 03/10] fix(osworld): align pool flow with OpenSandbox SDK Remove the duplicate image-less REST lifecycle and allocate OSWorld Pool VMs through the SDK compatibility-image path. Keep excluded runtime dependencies scoped to the agent with overrides and an explicit opt-in installer. Signed-off-by: Jeff Peng --- benchmarks/osworld/README.md | 26 +- benchmarks/osworld/prepare.py | 8 +- benchmarks/osworld/tests/test_prepare.py | 5 +- benchmarks/osworld/tests/test_run_scripts.py | 23 +- .../sandbox/providers/opensandbox/provider.py | 278 ++---------------- responses_api_agents/osworld_agent/client.py | 5 - .../install_optional_runtime_deps.sh | 34 +++ .../osworld_agent/local_forwarder.py | 4 +- .../osworld_agent/overrides.txt | 22 ++ .../osworld_agent/requirements.txt | 11 +- .../osworld_agent/sandbox_provider.py | 21 +- .../osworld_agent/tests/test_client.py | 10 +- .../tests/test_sandbox_provider.py | 27 +- tests/unit_tests/test_cli_setup_command.py | 12 + tests/unit_tests/test_opensandbox_provider.py | 269 ++++++++--------- 15 files changed, 304 insertions(+), 451 deletions(-) create mode 100755 responses_api_agents/osworld_agent/install_optional_runtime_deps.sh create mode 100644 responses_api_agents/osworld_agent/overrides.txt diff --git a/benchmarks/osworld/README.md b/benchmarks/osworld/README.md index 8ec148d783..4e0fb3db52 100644 --- a/benchmarks/osworld/README.md +++ b/benchmarks/osworld/README.md @@ -211,9 +211,11 @@ The `gym_opensandbox` backend keeps the same OSWorld agent, task setup, action, and evaluator path while replacing the local or remote Docker lifecycle with an allocation from a server-managed KVM Pool. The Pool operator owns its VM image, entrypoint, and capacity. Consequently, clients provide neither a -container image nor `--vm-path`; they only name a pre-provisioned Pool. The -checked-in default is `osworld-kvm`, overridable with -`OPENSANDBOX_POOL_REF`. +VM image nor `--vm-path`; they name a pre-provisioned Pool and pass a small +compatibility image solely because the OpenSandbox SDK requires one. The Pool +supplies the actual OSWorld VM and does not use that compatibility image. The +checked-in defaults are `osworld-kvm` and `busybox:1.36`, overridable with +`OPENSANDBOX_POOL_REF` and `OPENSANDBOX_COMPAT_IMAGE`. Install Gym with the OpenSandbox SDK and configure the management endpoint. Keep credentials in the environment; `prepare.py` does not write the API key @@ -251,6 +253,24 @@ python3 prepare.py \ --force-env ``` +The managed OSWorld agent's default `requirements.txt` respects Gym's global +security and codec exclusions. After `prepare.py` writes `env.yaml`, pre-create +its isolated environment and explicitly install the two codec-bearing runtime +packages that OSWorld imports but Gym does not ship in packages or containers: + +```bash +gym env prefetch +bash ../../responses_api_agents/osworld_agent/install_optional_runtime_deps.sh \ + /absolute/run/root/server-venvs/responses_api_agents/osworld_agent/.venv +``` + +The script uses `--no-config` only for that named venv and installs +cryptography, headless OpenCV, and the matching torchvision wheel. These are +runtime imports for OSWorld's desktop stack, but repository policy excludes +them from managed package and container resolution. `skip_venv_if_present: +true` in the generated config then lets `gym env start` reuse the prepared +environment. + OpenSandbox may return path-based gateway endpoints with required routing headers rather than directly routable Pod addresses. The adapter creates client-local forwarders that preserve those paths and headers for OSWorld's diff --git a/benchmarks/osworld/prepare.py b/benchmarks/osworld/prepare.py index 865b7d2cec..9b779887f0 100644 --- a/benchmarks/osworld/prepare.py +++ b/benchmarks/osworld/prepare.py @@ -42,6 +42,7 @@ OSWORLD_PROVIDER_CONFIG = BENCHMARK_DIR / "configs" / "osworld_docker_pinned.yaml" OPENSANDBOX_CONFIG = BENCHMARK_DIR / "configs" / "osworld_opensandbox.yaml" OPENSANDBOX_VM_SENTINEL = "/opensandbox/Ubuntu.qcow2" +OPENSANDBOX_COMPAT_IMAGE = "busybox:1.36" PROFILE_CONFIGS: dict[str, tuple[Path, ...]] = { "default": (DEFAULT_CONFIG,), @@ -300,9 +301,7 @@ def write_env( "gym_opensandbox": "osworld_opensandbox", }.get(execution_backend) emitted_vm_path: str | Path | None = ( - OPENSANDBOX_VM_SENTINEL - if execution_backend == "gym_opensandbox" - else resolved_vm_path + OPENSANDBOX_VM_SENTINEL if execution_backend == "gym_opensandbox" else resolved_vm_path ) contents = "\n".join( [ @@ -348,7 +347,8 @@ def write_env( " sandbox_require_kvm: false", " sandbox_ready_timeout_s: 600.0", " sandbox_spec:", - " image: null", + " # Required by the SDK; poolRef supplies the actual OSWorld VM.", + f" image: ${{oc.env:OPENSANDBOX_COMPAT_IMAGE,{OPENSANDBOX_COMPAT_IMAGE}}}", " ttl_s: 14400", " ready_timeout_s: 1200", " provider_options:", diff --git a/benchmarks/osworld/tests/test_prepare.py b/benchmarks/osworld/tests/test_prepare.py index e8391f196d..24e1c95eef 100644 --- a/benchmarks/osworld/tests/test_prepare.py +++ b/benchmarks/osworld/tests/test_prepare.py @@ -262,7 +262,7 @@ def test_write_env_rejects_sandbox_without_explicit_vm(tmp_path: Path) -> None: ) -def test_write_env_configures_image_less_opensandbox_pool(tmp_path: Path) -> None: +def test_write_env_configures_sdk_compatibility_image_for_opensandbox_pool(tmp_path: Path) -> None: env_path = tmp_path / "run" / "env.yaml" assert write_env( @@ -285,7 +285,7 @@ def test_write_env_configures_image_less_opensandbox_pool(tmp_path: Path) -> Non assert agent["sandbox_provider"] == "osworld_opensandbox" assert agent["vm_path"] == OPENSANDBOX_VM_SENTINEL assert agent["sandbox_require_kvm"] is False - assert agent["sandbox_spec"]["image"] is None + assert agent["sandbox_spec"]["image"] == ("${oc.env:OPENSANDBOX_COMPAT_IMAGE,busybox:1.36}") assert agent["sandbox_spec"]["ttl_s"] == 14400 assert agent["sandbox_spec"]["provider_options"]["extensions"]["poolRef"] == ( "${oc.env:OPENSANDBOX_POOL_REF,osworld-kvm}" @@ -320,6 +320,7 @@ def test_opensandbox_env_composes_with_strict_inherited_sandbox_spec(tmp_path: P GlobalConfigDictParser()._recursively_swap_keys(config) agent = config["osworld_nano_omni_agent"]["responses_api_agents"]["osworld_agent"] + assert agent["sandbox_spec"]["image"] == "busybox:1.36" assert agent["sandbox_spec"]["ttl_s"] == 14400 assert agent["sandbox_spec"]["provider_options"]["extensions"]["poolRef"] == "osworld-kvm" diff --git a/benchmarks/osworld/tests/test_run_scripts.py b/benchmarks/osworld/tests/test_run_scripts.py index ac56b3cb25..a2b6f7ef30 100644 --- a/benchmarks/osworld/tests/test_run_scripts.py +++ b/benchmarks/osworld/tests/test_run_scripts.py @@ -17,11 +17,20 @@ OPENSANDBOX_CLEANUP_SCRIPT = REPO_ROOT / "benchmarks/osworld/tools/cleanup_opensandbox_run.py" OSWORLD_AGENT_CONFIG = REPO_ROOT / "responses_api_agents/osworld_agent/configs/osworld_agent.yaml" OSWORLD_AGENT_REQUIREMENTS = REPO_ROOT / "responses_api_agents/osworld_agent/requirements.txt" +OSWORLD_AGENT_OVERRIDES = REPO_ROOT / "responses_api_agents/osworld_agent/overrides.txt" +OSWORLD_RUNTIME_DEPS_SCRIPT = REPO_ROOT / "responses_api_agents/osworld_agent/install_optional_runtime_deps.sh" @pytest.mark.parametrize( "script", - [VM_PREPARE_SCRIPT, CHECK_ENVIRONMENT_SCRIPT, START_CONTROL_SCRIPT, RUN_EVAL_SCRIPT, CLEANUP_RUN_SCRIPT], + [ + VM_PREPARE_SCRIPT, + CHECK_ENVIRONMENT_SCRIPT, + START_CONTROL_SCRIPT, + RUN_EVAL_SCRIPT, + CLEANUP_RUN_SCRIPT, + OSWORLD_RUNTIME_DEPS_SCRIPT, + ], ) def test_public_host_setup_scripts_are_syntax_valid_and_portable(script: Path) -> None: subprocess.run(["bash", "-n", str(script)], check=True) @@ -52,10 +61,22 @@ def test_start_control_preflights_native_build_toolchain() -> None: def test_managed_osworld_agent_installs_opensandbox_sdk() -> None: requirements = OSWORLD_AGENT_REQUIREMENTS.read_text(encoding="utf-8").splitlines() + overrides = OSWORLD_AGENT_OVERRIDES.read_text(encoding="utf-8").splitlines() + runtime_script = OSWORLD_RUNTIME_DEPS_SCRIPT.read_text(encoding="utf-8") assert "-e nemo-gym[dev] @ ../../" in requirements assert "opensandbox>=0.1.15" in requirements assert "tenacity>=9.1.4" in requirements + assert not any(line.startswith("cryptography") for line in requirements) + assert not any(line.startswith("flask") for line in requirements) + assert not any(line.startswith("opencv-") for line in requirements) + assert "torch==2.11.0" in overrides + assert "matplotlib==3.10.6" in overrides + assert "agp-client; sys_platform == 'never'" in overrides + assert "--no-config" in runtime_script + assert "cryptography~=46.0" in runtime_script + assert "opencv-python-headless~=4.8.1.78" in runtime_script + assert "torchvision==0.26.0" in runtime_script def test_remote_docker_requires_a_reachable_publish_host() -> None: diff --git a/nemo_gym/sandbox/providers/opensandbox/provider.py b/nemo_gym/sandbox/providers/opensandbox/provider.py index 68e48967c6..76bddbf24b 100644 --- a/nemo_gym/sandbox/providers/opensandbox/provider.py +++ b/nemo_gym/sandbox/providers/opensandbox/provider.py @@ -15,16 +15,15 @@ """OpenSandbox provider implementation.""" import asyncio -import json import logging import re import shlex -import uuid from collections.abc import Mapping from dataclasses import dataclass, field, replace from datetime import timedelta from pathlib import Path from typing import Any, Awaitable, Callable +from urllib.parse import urlsplit from nemo_gym.sandbox.attribution import RUN_KEY, log_attribution_once, resolve_attribution, resolve_run_id from nemo_gym.sandbox.providers.base import ( @@ -65,7 +64,6 @@ class SandboxBackendUnreachableError(RuntimeError): RETRYABLE_HTTP_STATUS_CODES = {408, 409, 425, 429, 500, 502, 503, 504} -POOLED_CREATE_MARKER_KEY = "nemo-gym-create-id" RETRYABLE_ERROR_MARKERS = ( "all connection attempts failed", "connection refused", @@ -289,29 +287,6 @@ def _log_operation_retry(retry_state: Any, *, operation: str = "?", sandbox_id: ) -async def _rest_request( - method: str, - url: str, - *, - json_body: Any | None = None, - headers: Mapping[str, str] | None = None, - timeout_s: float = 300.0, -) -> tuple[int, str]: - """Send one lifecycle request without initializing Gym's global config.""" - - import aiohttp # noqa: PLC0415 - - timeout = aiohttp.ClientTimeout(total=timeout_s) - async with aiohttp.ClientSession(timeout=timeout, trust_env=False) as session: - async with session.request( - method, - url, - json=json_body, - headers=dict(headers or {}), - ) as response: - return response.status, await response.text() - - def _string_map(values: Mapping[str, Any]) -> dict[str, str]: return {str(key): str(value) for key, value in values.items()} @@ -656,7 +631,9 @@ def _connection_config( _, ConnectionConfig, _, _, _ = _require_opensandbox_sdk() kwargs: dict[str, Any] = {} if self._connection.domain is not None: - kwargs["domain"] = self._connection.domain + # OpenSandbox SDK 0.1.15 appends ``/v1`` directly. Normalizing here + # prevents a configured trailing slash from producing ``//v1``. + kwargs["domain"] = self._connection.domain.rstrip("/") if self._connection.api_key is not None: kwargs["api_key"] = self._connection.api_key if self._connection.protocol is not None: @@ -975,213 +952,6 @@ async def _connect_after_create(self, handle: SandboxHandle, spec: SandboxSpec) if sleep_s > 0: await asyncio.sleep(sleep_s) - def _rest_base_url(self) -> str: - domain = self._connection.domain - if not domain: - raise ValueError("OpenSandbox connection.domain is required for pool-mode create") - if domain.startswith(("http://", "https://")): - return domain.rstrip("/") - return f"{self._connection.protocol or 'http'}://{domain}".rstrip("/") - - def _rest_headers(self) -> dict[str, str]: - if self._connection.api_key: - return {"OPEN-SANDBOX-API-KEY": self._connection.api_key} - return {} - - async def _rest_create_pooled( - self, - spec: SandboxSpec, - options: OpenSandboxProviderOptions, - ) -> str: - """Allocate from a server-side Pool using an image-less lifecycle POST.""" - - create_id = uuid.uuid4().hex - body: dict[str, Any] = { - "extensions": dict(options.extensions), - "metadata": { - **(spec.metadata or {}), - POOLED_CREATE_MARKER_KEY: create_id, - }, - } - if spec.ttl_s is not None: - body["timeout"] = int(spec.ttl_s) - - try: - status, text = await _rest_request( - "POST", - f"{self._rest_base_url()}/v1/sandboxes", - json_body=body, - headers=self._rest_headers(), - timeout_s=float(self._create.request_timeout_s or self._connection.request_timeout_s or 300), - ) - except BaseException: - await self._reap_pooled_create_marker(create_id) - raise - - if status not in {200, 201, 202}: - raise OpenSandboxCreateError(f"OpenSandbox pool-mode create failed (HTTP {status}): {text[:300]}") - try: - sandbox_id = json.loads(text)["id"] - except (json.JSONDecodeError, KeyError, TypeError) as error: - raise OpenSandboxCreateError( - f"OpenSandbox pool-mode create returned an unexpected body: {text[:300]}" - ) from error - return str(sandbox_id) - - async def _reap_pooled_create_marker(self, create_id: str) -> None: - """Delete only a lost create carrying this client's exact marker.""" - - base = self._rest_base_url() - timeout_s = float(self._operations.close_timeout_s or 30.0) - loop = asyncio.get_running_loop() - deadline = loop.time() + timeout_s - page = 1 - page_size = 200 - - def remaining_timeout_s() -> float: - return max(deadline - loop.time(), 0.0) - - try: - while page <= 100: - remaining_s = remaining_timeout_s() - if remaining_s <= 0: - LOGGER.warning( - "Timed out scanning OpenSandbox sandboxes for abandoned " - "create marker %s after %s pages", - create_id, - page - 1, - ) - return - status, text = await _rest_request( - "GET", - f"{base}/v1/sandboxes?page={page}&pageSize={page_size}", - headers=self._rest_headers(), - timeout_s=remaining_s, - ) - if status != 200: - LOGGER.warning( - "Could not list OpenSandbox sandboxes to reap abandoned " - "create marker %s: page=%s HTTP %s", - create_id, - page, - status, - ) - return - data = json.loads(text) - items = ( - data - if isinstance(data, list) - else data.get("items") or data.get("sandboxes") or [] - ) - for item in items: - if not isinstance(item, Mapping): - continue - metadata = item.get("metadata") or {} - if ( - not isinstance(metadata, Mapping) - or metadata.get(POOLED_CREATE_MARKER_KEY) != create_id - ): - continue - sandbox_id = str(item.get("id") or "") - if not sandbox_id: - continue - remaining_s = remaining_timeout_s() - if remaining_s <= 0: - LOGGER.warning( - "Timed out before deleting abandoned OpenSandbox " - "pooled create %s (marker %s)", - sandbox_id, - create_id, - ) - return - delete_status, _ = await _rest_request( - "DELETE", - f"{base}/v1/sandboxes/{sandbox_id}", - headers=self._rest_headers(), - timeout_s=remaining_s, - ) - LOGGER.warning( - "Reaped abandoned OpenSandbox pooled create %s " - "(marker %s, DELETE HTTP %s)", - sandbox_id, - create_id, - delete_status, - ) - return - - if isinstance(data, list): - has_next_page = len(items) >= page_size - else: - pagination = data.get("pagination") or {} - has_next_page = bool(pagination.get("hasNextPage")) - total_pages = pagination.get("totalPages") - if isinstance(total_pages, int): - has_next_page = has_next_page or page < total_pages - if not has_next_page: - return - page += 1 - LOGGER.warning( - "Stopped scanning OpenSandbox sandboxes for abandoned create " - "marker %s after %s pages", - create_id, - page - 1, - ) - except Exception as error: # noqa: BLE001 - LOGGER.warning( - "Failed to reap abandoned OpenSandbox pooled create (marker %s): %r", - create_id, - error, - ) - - async def _cleanup_failed_pooled_create(self, handle: SandboxHandle) -> None: - try: - await _rest_request( - "DELETE", - f"{self._rest_base_url()}/v1/sandboxes/{handle.sandbox_id}", - headers=self._rest_headers(), - timeout_s=float(self._operations.close_timeout_s or 30.0), - ) - except Exception as error: # noqa: BLE001 - LOGGER.warning( - "Failed to clean up pooled sandbox after create failure; sandbox_id=%s; error=%r", - handle.sandbox_id, - error, - ) - - async def _create_pooled( - self, - spec: SandboxSpec, - options: OpenSandboxProviderOptions, - ) -> SandboxHandle: - if "poolRef" not in options.extensions: - raise ValueError( - "OpenSandbox create without image/snapshot_id requires provider_options.extensions.poolRef" - ) - sandbox_id = await self._rest_create_pooled(spec, options) - rest_handle = _PooledRestSandbox( - sandbox_id=sandbox_id, - base_url=self._rest_base_url(), - headers=self._rest_headers(), - protocol=self._connection.protocol or "http", - timeout_s=float( - self._connection.request_timeout_s - or self._create.request_timeout_s - or 300.0 - ), - use_server_proxy=self._connection.use_server_proxy, - ) - created_handle = SandboxHandle( - sandbox_id=sandbox_id, - provider_name=self.name, - raw=rest_handle, - ) - try: - await self._verify_created_handle(created_handle) - except BaseException: - await self._cleanup_failed_pooled_create(created_handle) - raise - return created_handle - async def endpoint( self, handle: SandboxHandle, @@ -1189,32 +959,44 @@ async def endpoint( ) -> SandboxEndpoint: """Resolve one client-reachable direct or server-proxied service URL.""" - if handle.raw is None: - raise RuntimeError(f"OpenSandbox handle for {handle.sandbox_id!r} has no SDK object") - endpoint = await self._await_sdk_operation( - lambda: handle.raw.get_endpoint(port), - operation=f"get_endpoint({port})", + get_endpoint = getattr(handle.raw, "get_endpoint", None) + if get_endpoint is None: + raise NotImplementedError( + "The installed opensandbox SDK does not expose Sandbox.get_endpoint; " + "sandbox service endpoints require opensandbox>=0.1.15" + ) + resolved = await self._await_sdk_operation( + lambda: get_endpoint(port), + operation="get_endpoint", sandbox_id=handle.sandbox_id, timeout_s=( float(self._connection.request_timeout_s) if self._connection.request_timeout_s is not None else None ), ) - url = str(getattr(endpoint, "endpoint", "") or "") - if not url: + endpoint_url = str(getattr(resolved, "endpoint", "") or "") + if not endpoint_url: raise RuntimeError(f"OpenSandbox returned an empty endpoint for sandbox {handle.sandbox_id!r} port {port}") - if "://" not in url: - url = f"{self._connection.protocol or 'http'}://{url}" - headers = dict(getattr(endpoint, "headers", None) or {}) - return SandboxEndpoint(endpoint=url, headers=headers) + if "://" not in endpoint_url: + domain = str(self._connection.domain or "") + # urlsplit("host.example:8080") treats the hostname as a scheme. + # Only read a scheme from a domain that actually contains ``://``; + # otherwise use ConnectionConfig.protocol just as the SDK does. + domain_scheme = urlsplit(domain).scheme if "://" in domain else "" + scheme = domain_scheme or self._connection.protocol or "http" + endpoint_url = f"{scheme}://{endpoint_url.lstrip('/')}" + headers = dict(getattr(resolved, "headers", None) or {}) + if self._connection.use_server_proxy and self._connection.api_key: + # Proxy mode terminates at the trusted OpenSandbox gateway. Direct + # endpoints terminate in untrusted workloads and must never receive + # the management API key. + headers.setdefault("OPEN-SANDBOX-API-KEY", str(self._connection.api_key)) + return SandboxEndpoint(endpoint=endpoint_url, headers=headers) async def _create_once(self, spec: SandboxSpec) -> SandboxHandle: """Create a sandbox through ``opensandbox.Sandbox.create``.""" Sandbox, _, _, _, _ = _require_opensandbox_sdk() options = OpenSandboxProviderOptions.from_mapping(spec.provider_options) - if spec.image is None and options.snapshot_id is None: - return await self._create_pooled(spec, options) - kwargs: dict[str, Any] = { "env": spec.env, "metadata": spec.metadata, diff --git a/responses_api_agents/osworld_agent/client.py b/responses_api_agents/osworld_agent/client.py index bffd6dce6f..b237eaa992 100644 --- a/responses_api_agents/osworld_agent/client.py +++ b/responses_api_agents/osworld_agent/client.py @@ -1719,11 +1719,6 @@ def proxy_precondition_failure(reason: str, message: str) -> RolloutResult: sandbox_provider_name = str(next(iter(sandbox_provider_config or {}), "")).lower().strip() if sandbox_provider_name == "docker": effective_sandbox_spec.setdefault("image", container_image) - elif sandbox_provider_name == "opensandbox": - # An image-less create allocates a prebuilt QEMU guest from - # the server-side Pool. Do not let the reusable Docker - # profile's image select the SDK's container-create path. - effective_sandbox_spec.pop("image", None) env_kwargs.update( { "sandbox_provider": dict(sandbox_provider_config or {}), diff --git a/responses_api_agents/osworld_agent/install_optional_runtime_deps.sh b/responses_api_agents/osworld_agent/install_optional_runtime_deps.sh new file mode 100755 index 0000000000..ba69d96568 --- /dev/null +++ b/responses_api_agents/osworld_agent/install_optional_runtime_deps.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Install OSWorld runtime packages that NeMo Gym deliberately excludes from +# shipped packages and containers. Run this only after `gym env prefetch` has +# created the managed OSWorld agent venv. +set -euo pipefail + +if [[ $# -ne 1 ]]; then + echo "Usage: $0 /absolute/path/to/osworld-agent-venv" >&2 + exit 2 +fi + +venv_path=$1 +venv_python="${venv_path}/bin/python" +if [[ ! -x "${venv_python}" ]]; then + echo "OSWorld agent Python is not executable: ${venv_python}" >&2 + exit 2 +fi + +if "${venv_python}" -c "import cryptography, cv2, torchvision" 2>/dev/null; then + echo "[osworld-runtime-deps] Already installed, skipping." + exit 0 +fi + +echo "[osworld-runtime-deps] Installing opt-in runtime dependencies..." +uv pip install --no-config --python "${venv_python}" \ + "cryptography~=46.0" \ + "opencv-python-headless~=4.8.1.78" \ + "torchvision==0.26.0" + +"${venv_python}" -c "import cryptography, cv2, torchvision" +echo "[osworld-runtime-deps] Done." diff --git a/responses_api_agents/osworld_agent/local_forwarder.py b/responses_api_agents/osworld_agent/local_forwarder.py index 3f4bc29c1c..0b295b064f 100644 --- a/responses_api_agents/osworld_agent/local_forwarder.py +++ b/responses_api_agents/osworld_agent/local_forwarder.py @@ -139,9 +139,7 @@ def _forward(self) -> None: length = int(self.headers.get("Content-Length", 0) or 0) body = self.rfile.read(length) if length else None headers = { - key: value - for key, value in self.headers.items() - if key.lower() not in _HOP_BY_HOP_REQUEST_HEADERS + key: value for key, value in self.headers.items() if key.lower() not in _HOP_BY_HOP_REQUEST_HEADERS } # Guest authentication headers (for example VLC Basic auth) win # over route headers with the same name. diff --git a/responses_api_agents/osworld_agent/overrides.txt b/responses_api_agents/osworld_agent/overrides.txt new file mode 100644 index 0000000000..2831314f96 --- /dev/null +++ b/responses_api_agents/osworld_agent/overrides.txt @@ -0,0 +1,22 @@ +# Keep the managed OSWorld agent compatible with Python 3.13 without changing +# repository-wide dependency policy or the independently maintained OSWorld +# package metadata. + +# OSWorld pins torch 2.5.x, which has no CPython 3.13 wheels. Match the +# repository's tested Python 3.13 line instead of floating to a newer release. +torch==2.11.0 + +# OSWorld pins matplotlib 3.7.x, which also predates CPython 3.13 wheels and +# falls back to a fragile source build. Use the version already exercised by +# another Gym server on the repository's Python 3.13 line. +matplotlib==3.10.6 + +# SurferH is not loaded by this Gym adapter, and agp-client is not published on +# the configured package indexes. +agp-client; sys_platform == 'never' + +# Codec-bearing wheels remain excluded from the default managed environment. +# Install the explicitly documented opt-in runtime dependencies before an +# OSWorld run that needs them. +opencv-python; sys_platform == 'never' +opencv-contrib-python; sys_platform == 'never' diff --git a/responses_api_agents/osworld_agent/requirements.txt b/responses_api_agents/osworld_agent/requirements.txt index b53e7ac386..41dcea31d4 100644 --- a/responses_api_agents/osworld_agent/requirements.txt +++ b/responses_api_agents/osworld_agent/requirements.txt @@ -24,19 +24,18 @@ requests-toolbelt~=1.0.0 # when transitive dependencies pull chardet 7.x. chardet<6 -# OSWorld imports these packages at runtime but does not declare all of them. -cryptography +# The Gym adapter imports these OSWorld paths at runtime. Repository policy +# intentionally excludes cryptography, codec-bearing OpenCV, and torchvision +# wheels from managed server resolution. Install those explicit opt-in runtime +# dependencies after `gym env prefetch`; see +# install_optional_runtime_deps.sh and benchmarks/osworld/README.md. cffi scipy numpy<2 pynput ag2 -# The root resolver policy removes OSWorld's GUI OpenCV dependency edges; this -# server supplies the compatible headless wheel instead. -opencv-python-headless~=4.8.1.78 Pillow~=11.0.0 scikit-learn matplotlib~=3.7.4 -flask~=3.0.0 func-timeout diff --git a/responses_api_agents/osworld_agent/sandbox_provider.py b/responses_api_agents/osworld_agent/sandbox_provider.py index 1c8c978675..c802b0a15a 100644 --- a/responses_api_agents/osworld_agent/sandbox_provider.py +++ b/responses_api_agents/osworld_agent/sandbox_provider.py @@ -138,9 +138,10 @@ def _build_spec(self, path_to_vm: str, *, headless: bool, os_type: str) -> Sandb values["metadata"] = metadata if self._sandbox_provider_name == "opensandbox": - if values.get("image"): + if not values.get("image"): raise ValueError( - "OpenSandbox OSWorld Pool allocation must be image-less; the Pool owns the QEMU image" + "OpenSandbox OSWorld Pool allocation requires sandbox_spec.image " + "for SDK validation; the Pool still supplies the actual OSWorld VM" ) provider_options = dict(values.get("provider_options") or {}) extensions = dict(provider_options.get("extensions") or {}) @@ -150,12 +151,14 @@ def _build_spec(self, path_to_vm: str, *, headless: bool, os_type: str) -> Sandb values["provider_options"] = provider_options values.setdefault("ttl_s", 7200) values.setdefault("ready_timeout_s", self._ready_timeout_s) - # The reusable OSWorld profile also carries Docker/QEMU defaults. - # They are intentionally discarded because the server-side Pool - # owns the image, entrypoint, environment, and resources. + # The SDK requires an image argument even for Pool allocation, but + # poolRef supplies the actual prebuilt OSWorld VM. The reusable + # profile's entrypoint, environment, and resources remain Docker- + # specific and are intentionally discarded. pool_fields = { key: values[key] for key in ( + "image", "ttl_s", "ready_timeout_s", "ports", @@ -205,10 +208,7 @@ def _build_spec(self, path_to_vm: str, *, headless: bool, os_type: str) -> Sandb return SandboxSpec(**values) def _resolve_service_endpoints(self, sandbox: Sandbox) -> tuple[str, dict[int, int]]: - endpoints = { - container_port: sandbox.endpoint(container_port) - for container_port in OSWORLD_SERVICE_PORTS - } + endpoints = {container_port: sandbox.endpoint(container_port) for container_port in OSWORLD_SERVICE_PORTS} # Preserve the zero-hop path for local Docker or a routed Pod network. try: @@ -222,8 +222,7 @@ def _resolve_service_endpoints(self, sandbox: Sandbox) -> tuple[str, dict[int, i hosts = {host for host, _ in direct.values()} if len(hosts) == 1: return hosts.pop(), { - container_port: endpoint_port - for container_port, (_, endpoint_port) in direct.items() + container_port: endpoint_port for container_port, (_, endpoint_port) in direct.items() } # OpenSandbox's externally reachable endpoint is a path-based gateway diff --git a/responses_api_agents/osworld_agent/tests/test_client.py b/responses_api_agents/osworld_agent/tests/test_client.py index 606e9d955d..aa9a4c7cf4 100644 --- a/responses_api_agents/osworld_agent/tests/test_client.py +++ b/responses_api_agents/osworld_agent/tests/test_client.py @@ -360,7 +360,7 @@ def test_gym_sandbox_backend_is_passed_as_plain_env_configuration(monkeypatch) - assert kwargs["sandbox_ready_poll_s"] == 0.25 -def test_opensandbox_pool_backend_discards_inherited_docker_image(monkeypatch) -> None: +def test_opensandbox_pool_backend_preserves_sdk_compatibility_image(monkeypatch) -> None: _patch_client_for_fake_runtime(monkeypatch) result = osworld_client.run_osworld_task( @@ -369,7 +369,7 @@ def test_opensandbox_pool_backend_discards_inherited_docker_image(monkeypatch) - env_class_path="fake.FakeEnv", sandbox_provider_config={"opensandbox": {"connection": {}}}, sandbox_spec={ - "image": "docker://inherited-osworld-image", + "image": "busybox:1.36", "provider_options": { "extensions": {"poolRef": "osworld-kvm"}, }, @@ -385,10 +385,8 @@ def test_opensandbox_pool_backend_discards_inherited_docker_image(monkeypatch) - assert kwargs["sandbox_provider"] == { "opensandbox": {"connection": {}}, } - assert "image" not in kwargs["sandbox_spec"] - assert kwargs["sandbox_spec"]["provider_options"]["extensions"]["poolRef"] == ( - "osworld-kvm" - ) + assert kwargs["sandbox_spec"]["image"] == "busybox:1.36" + assert kwargs["sandbox_spec"]["provider_options"]["extensions"]["poolRef"] == ("osworld-kvm") assert kwargs["path_to_vm"] == "/opensandbox/Ubuntu.qcow2" assert kwargs["sandbox_require_kvm"] is False diff --git a/responses_api_agents/osworld_agent/tests/test_sandbox_provider.py b/responses_api_agents/osworld_agent/tests/test_sandbox_provider.py index 7b0b3bda72..aa5530724c 100644 --- a/responses_api_agents/osworld_agent/tests/test_sandbox_provider.py +++ b/responses_api_agents/osworld_agent/tests/test_sandbox_provider.py @@ -117,7 +117,7 @@ def test_build_spec_docker_tcg_mode_does_not_map_kvm(tmp_path) -> None: assert osworld_sandbox._has_option(spec.provider_options["run_args"], "--cap-add", "NET_ADMIN") -def test_build_spec_uses_image_less_opensandbox_pool(monkeypatch) -> None: +def test_build_spec_uses_sdk_compatibility_image_for_opensandbox_pool(monkeypatch) -> None: monkeypatch.setenv("OSWORLD_RUN_ID", "opensandbox-run") provider = osworld_sandbox.GymSandboxDesktopProvider( { @@ -130,7 +130,7 @@ def test_build_spec_uses_image_less_opensandbox_pool(monkeypatch) -> None: }, { "ttl_s": 1800, - "image": None, + "image": "busybox:1.36", "entrypoint": ["/run/entry.sh"], "env": {"KVM": "Y"}, "resources": {"cpu": 4, "memory_mib": 16384}, @@ -144,7 +144,7 @@ def test_build_spec_uses_image_less_opensandbox_pool(monkeypatch) -> None: os_type="Ubuntu", ) - assert spec.image is None + assert spec.image == "busybox:1.36" assert spec.ttl_s == 1800 assert spec.ports == osworld_sandbox.OSWORLD_SERVICE_PORTS assert spec.provider_options == {"extensions": {"poolRef": "osworld-kvm"}} @@ -160,7 +160,7 @@ def test_build_spec_rejects_invalid_opensandbox_pool_spec() -> None: {"opensandbox": {}}, {"provider_options": {"extensions": {}}}, ) - with pytest.raises(ValueError, match="poolRef"): + with pytest.raises(ValueError, match="requires sandbox_spec.image"): provider._build_spec( "/opensandbox/Ubuntu.qcow2", headless=True, @@ -170,11 +170,11 @@ def test_build_spec_rejects_invalid_opensandbox_pool_spec() -> None: provider = osworld_sandbox.GymSandboxDesktopProvider( {"opensandbox": {}}, { - "image": "docker://osworld:latest", - "provider_options": {"extensions": {"poolRef": "osworld-kvm"}}, + "image": "busybox:1.36", + "provider_options": {"extensions": {}}, }, ) - with pytest.raises(ValueError, match="image-less"): + with pytest.raises(ValueError, match="poolRef"): provider._build_spec( "/opensandbox/Ubuntu.qcow2", headless=True, @@ -229,10 +229,7 @@ def log_message(self, *args: object) -> None: def do_GET(self) -> None: seen["path"] = self.path seen["route"] = self.headers.get("X-Route", "") - content = ( - b'{"webSocketDebuggerUrl":' - b'"ws://100.100.1.2:9222/devtools/browser/test"}' - ) + content = b'{"webSocketDebuggerUrl":"ws://100.100.1.2:9222/devtools/browser/test"}' self.send_response(200) self.send_header("Content-Type", "application/json") self.send_header("Content-Length", str(len(content))) @@ -259,9 +256,7 @@ def do_GET(self) -> None: "path": "/proxy/9222/json/version", "route": "gateway", } - assert response.json()["webSocketDebuggerUrl"] == ( - f"ws://127.0.0.1:{port}/devtools/browser/test" - ) + assert response.json()["webSocketDebuggerUrl"] == (f"ws://127.0.0.1:{port}/devtools/browser/test") finally: forwarder.shutdown() forwarder.server_close() @@ -312,9 +307,7 @@ def endpoint(self, port: int) -> SandboxEndpoint: monkeypatch.setattr( osworld_sandbox, "start_forwarder", - lambda *_args, **_kwargs: (_ for _ in ()).throw( - RuntimeError("forwarder failed") - ), + lambda *_args, **_kwargs: (_ for _ in ()).throw(RuntimeError("forwarder failed")), ) with pytest.raises(RuntimeError, match="forwarder failed"): diff --git a/tests/unit_tests/test_cli_setup_command.py b/tests/unit_tests/test_cli_setup_command.py index a48a7d6276..5b7f21a7d2 100644 --- a/tests/unit_tests/test_cli_setup_command.py +++ b/tests/unit_tests/test_cli_setup_command.py @@ -54,6 +54,18 @@ def test_sanity(self, tmp_path: Path) -> None: expected_command = f"cd {server_dir} && uv venv --seed --allow-existing --python test python version {server_dir}/.venv > >(sed 's/^/(my server name) /') 2> >(sed 's/^/(my server name) /' >&2) && source {server_dir}/.venv/bin/activate && uv pip install -r requirements.txt ray[default]==test ray version openai==test openai version > >(sed 's/^/(my server name) /') 2> >(sed 's/^/(my server name) /' >&2)" assert expected_command == actual_command + def test_requirements_uses_server_local_overrides(self, tmp_path: Path) -> None: + server_dir = self._setup_server_dir(tmp_path) + (server_dir / "overrides.txt").write_text("dependency==2\n") + + actual_command = setup_env_command( + dir_path=server_dir, + global_config_dict=self._debug_global_config_dict(tmp_path), + prefix="my server name", + ) + + assert "uv pip install --override overrides.txt -r requirements.txt" in actual_command + def test_skips_install_when_venv_present(self, tmp_path: Path) -> None: server_dir = self._setup_server_dir(tmp_path) diff --git a/tests/unit_tests/test_opensandbox_provider.py b/tests/unit_tests/test_opensandbox_provider.py index 278b7869db..b1c9b46688 100644 --- a/tests/unit_tests/test_opensandbox_provider.py +++ b/tests/unit_tests/test_opensandbox_provider.py @@ -241,168 +241,88 @@ async def test_direct_create_passes_image_auth_to_sdk_create( assert image.auth.password == TEST_REGISTRY_PASSWORD -async def test_pool_create_uses_api_key_auth_without_execd_connect( +async def test_pool_create_uses_sdk_compatibility_image_and_proxy_auth( fake_opensandbox_sdk: None, - monkeypatch: pytest.MonkeyPatch, ) -> None: - calls: list[dict[str, Any]] = [] - - async def fake_rest_request( - method: str, - url: str, - *, - json_body: Any | None = None, - headers: dict[str, str] | None = None, - timeout_s: float = 300.0, - ) -> tuple[int, str]: - calls.append( - { - "method": method, - "url": url, - "json_body": json_body, - "headers": headers, - "timeout_s": timeout_s, - } - ) - return 201, '{"id": "pooled-sandbox-1"}' - - monkeypatch.setattr(opensandbox_provider, "_rest_request", fake_rest_request) provider = opensandbox_provider.OpenSandboxProvider( connection={ - "domain": "http://sandbox.example", + "domain": "http://sandbox.example/", "api_key": "pool-api-key", # pragma: allowlist secret "request_timeout_s": 30, + "use_server_proxy": True, + }, + create={ + "request_timeout_s": 120, + "timeout_s": 30, + "skip_health_check": True, }, - create={"request_timeout_s": 120, "timeout_s": 30}, probe={"command": None}, ) handle = await provider.create( SandboxSpec( + image="busybox:1.36", ttl_s=1800, metadata={"purpose": "osworld"}, provider_options={"extensions": {"poolRef": "osworld-kvm"}}, ) ) - assert handle.sandbox_id == "pooled-sandbox-1" - assert isinstance(handle.raw, opensandbox_provider._PooledRestSandbox) - assert FakeSandbox.connected_args == () - assert calls[0]["method"] == "POST" - assert calls[0]["url"] == "http://sandbox.example/v1/sandboxes" - assert calls[0]["headers"] == { + assert handle.sandbox_id == "sandbox-1" + assert FakeSandbox.created_kwargs["image"] == "busybox:1.36" + assert FakeSandbox.created_kwargs["timeout"] == timedelta(seconds=1800) + assert FakeSandbox.created_kwargs["extensions"]["poolRef"] == "osworld-kvm" + assert FakeSandbox.created_kwargs["metadata"]["purpose"] == "osworld" + assert FakeSandbox.created_kwargs["skip_health_check"] is True + create_connection = FakeSandbox.created_kwargs["connection_config"] + assert create_connection.kwargs["domain"] == "http://sandbox.example" + assert create_connection.kwargs["headers"] == { "OPEN-SANDBOX-API-KEY": "pool-api-key" # pragma: allowlist secret } - assert calls[0]["json_body"]["timeout"] == 1800 - assert calls[0]["json_body"]["extensions"] == {"poolRef": "osworld-kvm"} - assert calls[0]["json_body"]["metadata"]["purpose"] == "osworld" - assert calls[0]["json_body"]["metadata"][opensandbox_provider.POOLED_CREATE_MARKER_KEY] + assert FakeSandbox.connected_args == ("sandbox-1",) + assert FakeSandbox.connected_kwargs["skip_health_check"] is True -async def test_pool_connect_cancellation_deletes_known_sandbox( +async def test_pool_connect_cancellation_destroys_known_sdk_sandbox( monkeypatch: pytest.MonkeyPatch, ) -> None: - calls: list[tuple[str, str]] = [] - - async def fake_rest_request( - method: str, - url: str, - **_kwargs: Any, - ) -> tuple[int, str]: - calls.append((method, url)) - if method == "POST": - return 201, '{"id": "pooled-cancelled-connect"}' - if method == "DELETE": - return 204, "" - raise AssertionError(f"unexpected request: {method} {url}") - - monkeypatch.setattr(opensandbox_provider, "_rest_request", fake_rest_request) - provider = opensandbox_provider.OpenSandboxProvider( - connection={"domain": "http://sandbox.example"}, - probe={"command": None}, - ) - - async def cancelled_verify(_handle: Any) -> None: - raise asyncio.CancelledError + lifecycle: list[str] = [] - monkeypatch.setattr(provider, "_verify_created_handle", cancelled_verify) + class CancellingSandbox: + def __init__(self) -> None: + self.id = "pooled-cancelled-connect" - with pytest.raises(asyncio.CancelledError): - await provider._create_once( - SandboxSpec( - ready_timeout_s=30, - provider_options={ - "extensions": { - "poolRef": "osworld-kvm", - } - }, - ) - ) + @classmethod + async def create(cls, **_kwargs: Any) -> "CancellingSandbox": + lifecycle.append("create") + return cls() - assert calls == [ - ("POST", "http://sandbox.example/v1/sandboxes"), - ( - "DELETE", - "http://sandbox.example/v1/sandboxes/pooled-cancelled-connect", - ), - ] + @classmethod + async def connect(cls, *_args: Any, **_kwargs: Any) -> "CancellingSandbox": + lifecycle.append("connect") + raise asyncio.CancelledError + async def kill(self) -> None: + lifecycle.append("kill") -async def test_pool_post_cancellation_reaps_exact_create_marker( - monkeypatch: pytest.MonkeyPatch, -) -> None: - calls: list[tuple[str, str]] = [] - create_marker = "" - - async def fake_rest_request( - method: str, - url: str, - *, - json_body: Any | None = None, - **_kwargs: Any, - ) -> tuple[int, str]: - nonlocal create_marker - calls.append((method, url)) - if method == "POST": - create_marker = str( - json_body["metadata"][ - opensandbox_provider.POOLED_CREATE_MARKER_KEY - ] - ) - raise asyncio.CancelledError - if method == "GET": - if "page=1&" in url: - return ( - 200, - ( - '{"items":[{"id":"some-other-create",' - '"metadata":{"nemo-gym-create-id":"other"}}],' - '"pagination":{"page":1,"pageSize":200,' - '"totalPages":2,"hasNextPage":true}}' - ), - ) - return ( - 200, - ( - '{"items":[{"id":"lost-create",' - f'"metadata":{{"{opensandbox_provider.POOLED_CREATE_MARKER_KEY}":' - f'"{create_marker}"}}}}],' - '"pagination":{"page":2,"pageSize":200,' - '"totalPages":2,"hasNextPage":false}}' - ), - ) - if method == "DELETE": - return 204, "" - raise AssertionError(f"unexpected request: {method} {url}") + async def close(self) -> None: + lifecycle.append("close") - monkeypatch.setattr(opensandbox_provider, "_rest_request", fake_rest_request) + monkeypatch.setattr( + opensandbox_provider, + "_require_opensandbox_sdk", + lambda: (CancellingSandbox, FakeConnectionConfig, object, FakePlatformSpec, FakeVolume), + ) provider = opensandbox_provider.OpenSandboxProvider( connection={"domain": "http://sandbox.example"}, + create={"skip_health_check": True}, probe={"command": None}, ) with pytest.raises(asyncio.CancelledError): await provider._create_once( SandboxSpec( + image="busybox:1.36", + ready_timeout_s=30, provider_options={ "extensions": { "poolRef": "osworld-kvm", @@ -411,24 +331,10 @@ async def fake_rest_request( ) ) - assert create_marker - assert calls == [ - ("POST", "http://sandbox.example/v1/sandboxes"), - ("GET", "http://sandbox.example/v1/sandboxes?page=1&pageSize=200"), - ("GET", "http://sandbox.example/v1/sandboxes?page=2&pageSize=200"), - ("DELETE", "http://sandbox.example/v1/sandboxes/lost-create"), - ] + assert lifecycle == ["create", "connect", "kill", "close"] -async def test_pool_create_requires_pool_ref( - fake_opensandbox_sdk: None, -) -> None: - provider = opensandbox_provider.OpenSandboxProvider(probe={"command": None}) - with pytest.raises(ValueError, match="poolRef"): - await provider._create_once(SandboxSpec()) - - -async def test_endpoint_normalizes_missing_scheme() -> None: +async def test_endpoint_normalizes_missing_scheme_and_adds_proxy_auth() -> None: class FakeRaw: async def get_endpoint(self, port: int) -> Any: assert port == 5000 @@ -438,7 +344,11 @@ async def get_endpoint(self, port: int) -> Any: ) provider = opensandbox_provider.OpenSandboxProvider( - connection={"protocol": "http"}, + connection={ + "domain": "https://sandbox.example/", + "api_key": "pool-api-key", # pragma: allowlist secret + "use_server_proxy": True, + }, operations={"retries": 0}, probe={"command": None}, ) @@ -451,8 +361,77 @@ async def get_endpoint(self, port: int) -> Any: 5000, ) - assert resolved.endpoint == "http://10.0.0.22:5000" - assert resolved.headers == {"X-Route": "sandbox"} + assert resolved.endpoint == "https://10.0.0.22:5000" + assert resolved.headers == { + "X-Route": "sandbox", + "OPEN-SANDBOX-API-KEY": "pool-api-key", # pragma: allowlist secret + } + + +async def test_endpoint_uses_configured_protocol_for_domain_without_scheme() -> None: + class FakeRaw: + async def get_endpoint(self, _port: int) -> Any: + return SimpleNamespace(endpoint="sandbox.example:5000", headers={}) + + provider = opensandbox_provider.OpenSandboxProvider( + connection={ + "domain": "gateway.example:8080/", + "protocol": "https", + }, + operations={"retries": 0}, + probe={"command": None}, + ) + resolved = await provider.endpoint( + opensandbox_provider.SandboxHandle( + sandbox_id="sandbox-1", + provider_name="opensandbox", + raw=FakeRaw(), + ), + 5000, + ) + + assert resolved.endpoint == "https://sandbox.example:5000" + + +async def test_direct_endpoint_never_receives_management_api_key() -> None: + class FakeRaw: + async def get_endpoint(self, _port: int) -> Any: + return SimpleNamespace(endpoint="http://10.0.0.22:5000", headers={}) + + provider = opensandbox_provider.OpenSandboxProvider( + connection={ + "api_key": "pool-api-key", # pragma: allowlist secret + "use_server_proxy": False, + }, + operations={"retries": 0}, + probe={"command": None}, + ) + resolved = await provider.endpoint( + opensandbox_provider.SandboxHandle( + sandbox_id="sandbox-1", + provider_name="opensandbox", + raw=FakeRaw(), + ), + 5000, + ) + + assert resolved.headers == {} + + +async def test_endpoint_requires_sdk_get_endpoint() -> None: + provider = opensandbox_provider.OpenSandboxProvider( + operations={"retries": 0}, + probe={"command": None}, + ) + with pytest.raises(NotImplementedError, match="opensandbox>=0.1.15"): + await provider.endpoint( + opensandbox_provider.SandboxHandle( + sandbox_id="sandbox-1", + provider_name="opensandbox", + raw=object(), + ), + 5000, + ) def test_provider_validation_and_retry_helpers() -> None: @@ -549,7 +528,7 @@ def test_provider_options_from_mapping() -> None: def test_connection_config_and_image_policy(fake_opensandbox_sdk: None) -> None: provider = opensandbox_provider.OpenSandboxProvider( connection={ - "domain": "sandbox.example", + "domain": "sandbox.example/", "api_key": "key", # pragma: allowlist secret "protocol": "https", "request_timeout_s": 10, From 2584f15330e010eced8e5d1f7754b57aaba18dc5 Mon Sep 17 00:00:00 2001 From: Jeff Peng Date: Wed, 12 Aug 2026 15:05:04 +0800 Subject: [PATCH 04/10] fix(osworld): validate Chrome CDP relay setup Signed-off-by: Jeff Peng --- benchmarks/osworld/README.md | 19 +++++++++ benchmarks/osworld/prepare.py | 48 ++++++++++++++++++++++ benchmarks/osworld/tests/test_prepare.py | 52 ++++++++++++++++++++++++ 3 files changed, 119 insertions(+) diff --git a/benchmarks/osworld/README.md b/benchmarks/osworld/README.md index 4e0fb3db52..791f800d70 100644 --- a/benchmarks/osworld/README.md +++ b/benchmarks/osworld/README.md @@ -41,6 +41,25 @@ Docker SSH setup below. Start the model first, verify the environment second, then prepare and run Gym. Neither the operator workstation nor a persistent interactive SSH session is part of runtime communication. +### Chrome CDP port ownership + +OSWorld task setup, rather than a deployment script or the Gym adapter, owns +the guest processes that expose Chrome DevTools. Canonical tasks launch Chrome +with `--remote-debugging-port=1337` and then launch +`socat tcp-listen:9222,fork tcp:localhost:1337`. `DesktopEnv.reset()` executes +both commands from `verifier_metadata.osworld_task.config` for each fresh VM. + +The Docker image or OpenSandbox Pool must include `socat` and publish guest +port 9222; Gym forwards that published HTTP/WebSocket endpoint to OSWorld. A +user running canonical OSWorld inputs does not need to run either command +manually. Authors of custom inputs must include the relay whenever their task +setup starts Chrome CDP on port 1337. `prepare.py` checks this contract and +fails early with the missing setup command instead of allowing a later 502. + +A standalone Sandbox API smoke test that bypasses `DesktopEnv.reset()` must +start both Chrome and the relay before probing port 9222. Screenshot-only +checks against the OSWorld service on port 5000 do not require this relay. + ## Requirements - Linux x86_64 with Docker 20+ and access to the local Docker daemon. diff --git a/benchmarks/osworld/prepare.py b/benchmarks/osworld/prepare.py index 9b779887f0..f2e18ba1e8 100644 --- a/benchmarks/osworld/prepare.py +++ b/benchmarks/osworld/prepare.py @@ -21,6 +21,7 @@ import hashlib import json import os +import re from collections.abc import Sequence from pathlib import Path from typing import Any @@ -67,6 +68,52 @@ ) +def _setup_command_texts(task: dict[str, Any]) -> tuple[str, ...]: + """Return commands that OSWorld will execute during the task setup.""" + + config = task.get("config") + if not isinstance(config, list): + return () + + commands: list[str] = [] + for setup_item in config: + if not isinstance(setup_item, dict): + continue + parameters = setup_item.get("parameters") + if not isinstance(parameters, dict): + continue + command = parameters.get("command") + if isinstance(command, str): + commands.append(command) + elif isinstance(command, list): + commands.append(" ".join(str(argument) for argument in command)) + return tuple(commands) + + +def _validate_chrome_cdp_relay(task: dict[str, Any], *, line_number: int) -> None: + """Require OSWorld's guest relay when task setup starts Chrome CDP.""" + + commands = _setup_command_texts(task) + starts_chrome_cdp = any( + re.search(r"--remote-debugging-port(?:=|\s+)1337(?!\d)", command, flags=re.IGNORECASE) + for command in commands + ) + has_cdp_relay = any( + re.search(r"\bsocat\b", command, flags=re.IGNORECASE) + and re.search(r"\blisten:9222(?!\d)", command, flags=re.IGNORECASE) + and re.search(r"\b(?:localhost|127\.0\.0\.1):1337(?!\d)", command, flags=re.IGNORECASE) + for command in commands + ) + if starts_chrome_cdp and not has_cdp_relay: + task_id = str(task.get("id") or "") + raise ValueError( + f"OSWorld row {line_number} task {task_id!r} starts Chrome CDP on guest port 1337 " + "but verifier_metadata.osworld_task.config does not launch the required guest relay. " + "Add ['socat', 'tcp-listen:9222,fork', 'tcp:localhost:1337']; Gym publishes and " + "forwards port 9222 but does not start this task-owned process." + ) + + def prepare(input_jsonl: Path = DEFAULT_INPUT) -> Path: """Validate and return an OSWorld JSONL suitable for rollout collection.""" @@ -88,6 +135,7 @@ def prepare(input_jsonl: Path = DEFAULT_INPUT) -> Path: metadata = row.get("verifier_metadata") if not isinstance(metadata, dict) or not isinstance(metadata.get("osworld_task"), dict): raise ValueError(f"OSWorld row {line_number} must contain verifier_metadata.osworld_task") + _validate_chrome_cdp_relay(metadata["osworld_task"], line_number=line_number) row_count += 1 if row_count == 0: diff --git a/benchmarks/osworld/tests/test_prepare.py b/benchmarks/osworld/tests/test_prepare.py index 24e1c95eef..5c593e47ac 100644 --- a/benchmarks/osworld/tests/test_prepare.py +++ b/benchmarks/osworld/tests/test_prepare.py @@ -32,6 +32,58 @@ def test_prepare_validates_committed_example() -> None: assert prepare() == DEFAULT_INPUT.resolve() +def test_prepare_rejects_chrome_cdp_without_guest_relay(tmp_path: Path) -> None: + input_jsonl = tmp_path / "missing-cdp-relay.jsonl" + input_jsonl.write_text( + json.dumps( + { + "verifier_metadata": { + "osworld_task": { + "id": "chrome-without-relay", + "config": [ + { + "type": "launch", + "parameters": {"command": "google-chrome --remote-debugging-port 1337"}, + } + ], + } + } + } + ) + + "\n", + encoding="utf-8", + ) + + with pytest.raises(ValueError) as exc_info: + prepare(input_jsonl) + + message = str(exc_info.value) + assert "OSWorld row 1" in message + assert "chrome-without-relay" in message + assert "tcp-listen:9222" in message + assert "task-owned process" in message + + +def test_prepare_does_not_require_cdp_relay_without_chrome_cdp(tmp_path: Path) -> None: + input_jsonl = tmp_path / "non-chrome.jsonl" + input_jsonl.write_text( + json.dumps( + { + "verifier_metadata": { + "osworld_task": { + "id": "non-chrome-task", + "config": [{"type": "launch", "parameters": {"command": ["libreoffice"]}}], + } + } + } + ) + + "\n", + encoding="utf-8", + ) + + assert prepare(input_jsonl) == input_jsonl.resolve() + + @pytest.mark.parametrize( "shard_args", [[], ["--num-shards", "1", "--shard-index", "0"]], From 56521ebc6542497cb9e40a3e8c70907a183673df Mon Sep 17 00:00:00 2001 From: Jeff Peng Date: Wed, 12 Aug 2026 15:40:48 +0800 Subject: [PATCH 05/10] docs(osworld): close OpenSandbox lifecycle review Signed-off-by: Jeff Peng --- benchmarks/osworld/README.md | 10 ++++++++++ benchmarks/osworld/prepare.py | 3 +-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/benchmarks/osworld/README.md b/benchmarks/osworld/README.md index 791f800d70..66725ae765 100644 --- a/benchmarks/osworld/README.md +++ b/benchmarks/osworld/README.md @@ -320,6 +320,16 @@ root without `--reap`: --run-id my-osworld-opensandbox-run ``` +There is one unavoidable create-timeout boundary: the OpenSandbox server may +accept `Sandbox.create()` while its response is lost or delayed, so the SDK +caller can time out before receiving the sandbox ID. Gym cannot immediately +call `kill()` on an ID it has never observed. This is an SDK/server lifecycle +window, not a reason to maintain a second REST lifecycle in Gym. Attribution +metadata is included in the original create request, so keep a stable, +run-unique `OSWORLD_RUN_ID` and run the exact-ID cleanup above after an abnormal +exit or create timeout. Once the SDK returns a handle, Gym performs normal +`kill()` and local `close()` cleanup directly. + ## Multi-environment runs Set concurrency and data selection during preparation, then use the same two diff --git a/benchmarks/osworld/prepare.py b/benchmarks/osworld/prepare.py index f2e18ba1e8..3a7740d333 100644 --- a/benchmarks/osworld/prepare.py +++ b/benchmarks/osworld/prepare.py @@ -95,8 +95,7 @@ def _validate_chrome_cdp_relay(task: dict[str, Any], *, line_number: int) -> Non commands = _setup_command_texts(task) starts_chrome_cdp = any( - re.search(r"--remote-debugging-port(?:=|\s+)1337(?!\d)", command, flags=re.IGNORECASE) - for command in commands + re.search(r"--remote-debugging-port(?:=|\s+)1337(?!\d)", command, flags=re.IGNORECASE) for command in commands ) has_cdp_relay = any( re.search(r"\bsocat\b", command, flags=re.IGNORECASE) From 7c3765a25053a539b6e160dabe7ce894405742c3 Mon Sep 17 00:00:00 2001 From: Jeff Peng Date: Wed, 12 Aug 2026 18:45:40 +0800 Subject: [PATCH 06/10] fix(deps): update yappi for Python 3.14 Signed-off-by: Jeff Peng --- pyproject.toml | 2 +- uv.lock | 31 +++++++++++++++++++++---------- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index b3be211285..76ab182897 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -166,7 +166,7 @@ dependencies = [ "aiohttp>=3.14.1", # yappi: profiling tool - # Updated Mon Sep 22, 2025 with yappi==1.6.10 + # Updated Wed Aug 12, 2026 with yappi==1.7.6 for CPython 3.14 support # License: MIT https://github.com/sumerc/yappi/blob/1d3f7501701e1f050b6dcd6a86fd36aec08185c7/LICENSE "yappi", diff --git a/uv.lock b/uv.lock index 97a480baca..f85222643c 100644 --- a/uv.lock +++ b/uv.lock @@ -5832,17 +5832,28 @@ wheels = [ [[package]] name = "yappi" -version = "1.6.10" +version = "1.7.6" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/02/5b/cfde09baf28f7046194b98f1c4907e172c48e7c1b2db35a918fc8a57727a/yappi-1.6.10.tar.gz", hash = "sha256:463b822727658937bd95a7d80ca9758605b8cd0014e004e9e520ec9cb4db0c92", size = 59379, upload-time = "2024-11-12T11:24:38.351Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2c/33/9ca066f48c7fb21e0ab16fd5e1c99771275a8cec435ef7ac1840d13252f0/yappi-1.6.10-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:944df9ebc6b283d6591a6b5f4c586d0eb9c6131c915f1b20fb36127ade83720d", size = 32924, upload-time = "2024-11-12T11:23:53.435Z" }, - { url = "https://files.pythonhosted.org/packages/cc/ef/a81fac59ca7a13fd26321d59a54841f70f76ce91b5884c001d77f534b3b1/yappi-1.6.10-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3736ea6458edbabd96918d88e2963594823e4ab4c58d62a52ef81f6b5839ec19", size = 77308, upload-time = "2024-11-12T11:23:55.393Z" }, - { url = "https://files.pythonhosted.org/packages/62/59/8fdcb2a660388a7778c52cdfa0c52654955cf7953f85efacd8fd771f8da0/yappi-1.6.10-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f27bbc3311a3662231cff395d38061683fac5c538f3bab6796ff05511d2cce43", size = 81347, upload-time = "2024-11-12T11:23:56.926Z" }, - { url = "https://files.pythonhosted.org/packages/f5/28/62d8f97a62eafc443bb057442ae75b7f4741230c2dd774c5b7002bc05a4e/yappi-1.6.10-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:354cf94d659302b421b13c03487f2f1bce969b97b85fba88afb11f2ef83c35f3", size = 76239, upload-time = "2024-11-12T11:23:57.93Z" }, - { url = "https://files.pythonhosted.org/packages/5d/aa/ea0dbf6e00c7dcb81b4d84d35f6e0584c448674fc19533ddb3198533d41b/yappi-1.6.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1d82839835ae2c291b88fb56d82f80c88c00d76df29f3c1ed050db73b553bef0", size = 78712, upload-time = "2024-11-12T11:23:59.171Z" }, - { url = "https://files.pythonhosted.org/packages/88/72/81acfc73b5d66031284c7b4d384200d016f96e26038466269ed139114e98/yappi-1.6.10-cp313-cp313-win32.whl", hash = "sha256:fc84074575afcc5a2a712e132c0b51541b7434b3099be99f573964ef3b6064a8", size = 32026, upload-time = "2024-11-12T11:24:00.305Z" }, - { url = "https://files.pythonhosted.org/packages/23/71/47f12130412703a6816dba27ebd0aa853612ea6fbe3f93f7698c3520ea92/yappi-1.6.10-cp313-cp313-win_amd64.whl", hash = "sha256:334b31dfefae02bc28b7cd50953aaaae3292e40c15efb613792e4a587281a161", size = 34471, upload-time = "2024-11-12T11:24:01.378Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/9f/47/f7ec7744dff1104560d6276f951a8182f5b805e8d86ece591aebd0512845/yappi-1.7.6.tar.gz", hash = "sha256:c94281936af77c00c6ac2306a0e7f85a67e354d717120df85fcc5dfb9243d4dd", size = 62639, upload-time = "2026-03-17T22:31:40.928Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/15/b0/9a10f3a22290b67e23f339318fd368c173547478e0896f89363fb9cf190b/yappi-1.7.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:072df6fa8b4cfb5159c261dd0df8e8b85de0adbadbc5e953e1183da193674bc4", size = 33299, upload-time = "2026-03-17T22:31:06.04Z" }, + { url = "https://files.pythonhosted.org/packages/aa/ca/f36ccb82d7c96dee3858d26ed08e67de1767c309f285dbb2f76eceeaba48/yappi-1.7.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e4643d431656ec63e83455605ba29d1609d36b2fe14412e6939a223c323a7aee", size = 33193, upload-time = "2026-03-17T22:31:07.293Z" }, + { url = "https://files.pythonhosted.org/packages/17/04/078db90359b39496f9192e375cd97831b138794cf456ad43bd8c7b65a4e3/yappi-1.7.6-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b27541c7f77ef2f76b2e0bb5da6dce5dc5fcdc7e500b4756e7a3e077d499ac25", size = 83096, upload-time = "2026-03-17T22:31:08.205Z" }, + { url = "https://files.pythonhosted.org/packages/f0/52/24e214e5d4093e7b137fac95958afe289d1153ad35e6556be348c55a0b6a/yappi-1.7.6-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6e100b6c36b922fc407078ed74f08b2463f46efc1fb440387eb493966e4ec434", size = 82639, upload-time = "2026-03-17T22:31:09.121Z" }, + { url = "https://files.pythonhosted.org/packages/6d/d9/19b43be0e0f2a72518ec4907138614d4f98027839c10cd6b9b3a607cca2a/yappi-1.7.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5beecd15ff133c93fc505669754cb7caadd7fb19e87a71af133dfd1410e17aff", size = 80278, upload-time = "2026-03-17T22:31:10.039Z" }, + { url = "https://files.pythonhosted.org/packages/68/9e/9fa404fee5eb4942ad36409b5d00e3783bd573982aa84f22c8a2646a7125/yappi-1.7.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f3b5742d39c1ebe8909db0dec4a5b724a5a6167161864280021298f7ef4e76a1", size = 80337, upload-time = "2026-03-17T22:31:11.299Z" }, + { url = "https://files.pythonhosted.org/packages/92/2a/a42901c467259e10193c66a24bff410f041896ecdd3cb7b42dd515a54b2a/yappi-1.7.6-cp313-cp313-win32.whl", hash = "sha256:c9e3a92a04d9d6199fa0d157139beff1ca7eea7389e0e6b46b1353d8ffeec6a3", size = 32897, upload-time = "2026-03-17T22:31:12.219Z" }, + { url = "https://files.pythonhosted.org/packages/a1/6c/dede83e0ca33701681acdb06854e492010257ae83bd9dda8e953983fab3a/yappi-1.7.6-cp313-cp313-win_amd64.whl", hash = "sha256:95f9f326483d111b768f630a2d60689de7defff777f016b1f0dab9e93f36beb5", size = 35215, upload-time = "2026-03-17T22:31:13.084Z" }, + { url = "https://files.pythonhosted.org/packages/3a/b0/dec448196d207b2e3b4e6b27dd74d0f1714b645af4f25cfe7dfd564ec14f/yappi-1.7.6-cp313-cp313-win_arm64.whl", hash = "sha256:4981a243c5dbf105f6e1415197935ca36fde2b28adf26d2feceb95b5f1f77f06", size = 32861, upload-time = "2026-03-17T22:31:14.292Z" }, + { url = "https://files.pythonhosted.org/packages/9e/b3/d3fc45ea2c23c798887e1897a0aac92f8680d109a2381dbfefc70228cbb9/yappi-1.7.6-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:8bf3595e8c1c0326b8012591bc96b72625c7424d4d9fbe4b640b0aafd81f88dc", size = 33342, upload-time = "2026-03-17T22:31:15.116Z" }, + { url = "https://files.pythonhosted.org/packages/2a/9d/eb1298c95b00891ed1c62262779034bb109d5dea66c4db8546106f698602/yappi-1.7.6-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e9b018df48bc061248ae1fc36e161e9b4fb2cbbc50a8a0dfb68b9db4608bc9da", size = 33200, upload-time = "2026-03-17T22:31:16.289Z" }, + { url = "https://files.pythonhosted.org/packages/3e/d5/7b5fb53dff4f9361c88161bd1cd6e47388d57aaba8ae22ead354d37f8ecc/yappi-1.7.6-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:06c0487ab02e3a9722524c8d034feeadbdc2070d6530c38f7483291bf978b800", size = 82974, upload-time = "2026-03-17T22:31:17.125Z" }, + { url = "https://files.pythonhosted.org/packages/24/d0/0c55c25d74bd4bbe46031fc316927e93fc4b438005db66313ddc02d23bdb/yappi-1.7.6-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dedd28687f48607db40874629a47bc93d16f1b9c93045f34961620bda76df9d7", size = 82409, upload-time = "2026-03-17T22:31:18.032Z" }, + { url = "https://files.pythonhosted.org/packages/f6/ff/9a6a783840a595ada5c35355c7a1452846ecc69e44ba192e7c2a1236239e/yappi-1.7.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:1e3ef62417c598474a359de6aef92e13ea623416bb0ff45fa4b97e6569120549", size = 80189, upload-time = "2026-03-17T22:31:18.96Z" }, + { url = "https://files.pythonhosted.org/packages/86/2b/dbb6c82cc6f2b4d642af2b612f8930cb0948f4ede5d0307a4b45cb676932/yappi-1.7.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2b44e7a3187290615877d039bb2f4e232e1b7a5858b314ef6b011bd90447b537", size = 80171, upload-time = "2026-03-17T22:31:19.868Z" }, + { url = "https://files.pythonhosted.org/packages/a2/37/58c6601a43b9aa69f6c05cc2538ece44b73380349458d5fd397a737514cb/yappi-1.7.6-cp314-cp314-win32.whl", hash = "sha256:5d1d7ba37477da04cc1005784036a535ec5e053cfa09aec7d20e5bc436aedb8c", size = 33481, upload-time = "2026-03-17T22:31:21.074Z" }, + { url = "https://files.pythonhosted.org/packages/1b/d2/b468708803dcfead2b9c0415189ae89d0c17215c22715ffbc65372c0eccd/yappi-1.7.6-cp314-cp314-win_amd64.whl", hash = "sha256:53b8b8b6ad4f42cb82107c9fa96d103de33f76785e0ce84f5a326e66efc80f64", size = 35816, upload-time = "2026-03-17T22:31:21.95Z" }, + { url = "https://files.pythonhosted.org/packages/cb/88/5d9bea42f502a3916cd73934a7e4d522856e019a55e3364901c457e9e530/yappi-1.7.6-cp314-cp314-win_arm64.whl", hash = "sha256:b6a189c4b666933218d4bd4b7e1e22d03123120dcba3af4d6c2748ba7efba9ac", size = 33421, upload-time = "2026-03-17T22:31:22.825Z" }, ] [[package]] From 0986585a957f052f3766621a27c7f3979a9a6081 Mon Sep 17 00:00:00 2001 From: Jeff Peng Date: Wed, 12 Aug 2026 20:22:47 +0800 Subject: [PATCH 07/10] fix(osworld): preflight opt-in runtime dependencies Signed-off-by: Jeff Peng --- benchmarks/osworld/README.md | 30 ++- benchmarks/osworld/prepare.py | 22 +- benchmarks/osworld/tests/test_prepare.py | 11 +- benchmarks/osworld/tests/test_run_scripts.py | 19 ++ benchmarks/osworld/tools/README.md | 11 +- benchmarks/osworld/tools/start_control.sh | 27 +++ responses_api_agents/osworld_agent/README.md | 15 ++ responses_api_agents/osworld_agent/app.py | 3 + .../install_optional_runtime_deps.sh | 13 +- .../osworld_agent/runtime_dependencies.py | 204 ++++++++++++++++++ .../tests/test_runtime_dependencies.py | 109 ++++++++++ 11 files changed, 446 insertions(+), 18 deletions(-) create mode 100644 responses_api_agents/osworld_agent/runtime_dependencies.py create mode 100644 responses_api_agents/osworld_agent/tests/test_runtime_dependencies.py diff --git a/benchmarks/osworld/README.md b/benchmarks/osworld/README.md index 66725ae765..7435ed949d 100644 --- a/benchmarks/osworld/README.md +++ b/benchmarks/osworld/README.md @@ -139,7 +139,9 @@ prefetches setup and evaluator files, and writes a private, gitignored settings. Hugging Face assets use the official client cache and `HF_TOKEN` when configured. It keeps an existing env file unless `--force-env` is supplied. Python component dependencies are installed by `gym env start` from the agent -and model server project files. +and model server project files, except for the OSWorld agent's explicitly +opted-in runtime packages. `prepare.py` prints the exact prefetch and install +commands for the selected managed-agent venv before its normal start commands. Asset preparation is idempotent: `gym env start` checks the same selected JSONL and shared cache at server startup without contacting the remote source for task @@ -273,9 +275,10 @@ python3 prepare.py \ ``` The managed OSWorld agent's default `requirements.txt` respects Gym's global -security and codec exclusions. After `prepare.py` writes `env.yaml`, pre-create -its isolated environment and explicitly install the two codec-bearing runtime -packages that OSWorld imports but Gym does not ship in packages or containers: +security and codec exclusions. After `prepare.py` writes `env.yaml`, run the +exact path-aware next steps that it prints. They pre-create the isolated agent +environment and explicitly install the runtime packages that OSWorld imports +but Gym does not ship in packages or containers: ```bash gym env prefetch @@ -283,12 +286,19 @@ bash ../../responses_api_agents/osworld_agent/install_optional_runtime_deps.sh \ /absolute/run/root/server-venvs/responses_api_agents/osworld_agent/.venv ``` -The script uses `--no-config` only for that named venv and installs -cryptography, headless OpenCV, and the matching torchvision wheel. These are -runtime imports for OSWorld's desktop stack, but repository policy excludes -them from managed package and container resolution. `skip_venv_if_present: -true` in the generated config then lets `gym env start` reuse the prepared -environment. +The script uses `--no-config` only for that named agent venv and installs +cryptography, headless OpenCV, and the matching torchvision wheel. It is +idempotent, but skips installation only when the required versions are both +present and importable. It also reasserts the normal agent's `numpy<2` +constraint because the OpenCV 4.8 wheel uses NumPy's 1.x ABI. These are runtime +imports for OSWorld's desktop stack, +but repository policy excludes them from managed package and container +resolution. `skip_venv_if_present: true` in the generated config then lets +`gym env start` reuse the prepared environment. `tools/start_control.sh` checks +the same venv and exits before starting Gym with copyable remediation commands +if the explicit step was skipped; it never installs the packages itself. The +OSWorld agent entrypoint repeats the non-mutating check, so a direct +`gym env start` fails early with the scoped installer command as well. OpenSandbox may return path-based gateway endpoints with required routing headers rather than directly routable Pod addresses. The adapter creates diff --git a/benchmarks/osworld/prepare.py b/benchmarks/osworld/prepare.py index 3a7740d333..d3c8b028dd 100644 --- a/benchmarks/osworld/prepare.py +++ b/benchmarks/osworld/prepare.py @@ -22,11 +22,13 @@ import json import os import re +import shlex from collections.abc import Sequence from pathlib import Path from typing import Any from benchmarks.osworld.assets import DEFAULT_SETUP_CACHE, ensure_osworld_assets +from responses_api_agents.osworld_agent.runtime_dependencies import managed_agent_venv_path BENCHMARK_DIR = Path(__file__).resolve().parent @@ -35,6 +37,9 @@ DEFAULT_INPUT = BENCHMARK_DIR / "data" / "example.jsonl" DEFAULT_OUTPUT = REPO_ROOT / "results" / "osworld" / "rollouts.jsonl" DEFAULT_ENV = BENCHMARK_DIR / "env.yaml" +OSWORLD_RUNTIME_DEPS_INSTALLER = ( + REPO_ROOT / "responses_api_agents" / "osworld_agent" / "install_optional_runtime_deps.sh" +) BASE_AGENT_CONFIG = REPO_ROOT / "responses_api_agents" / "osworld_agent" / "configs" / "osworld_agent.yaml" OPENAI_MODEL_CONFIG = REPO_ROOT / "responses_api_models" / "openai_model" / "configs" / "openai_model.yaml" @@ -599,8 +604,21 @@ def main() -> None: force=args.force_env, ) - print("\nNext steps:") - print(f" cd {args.env_file.expanduser().resolve().parent}") + agent_venv = managed_agent_venv_path(REPO_ROOT, args.server_venv_root) + env_dir = args.env_file.expanduser().resolve().parent + print("\nNext steps (the OSWorld runtime package install is an explicit opt-in):") + print(f" cd {shlex.quote(str(env_dir))}") + print(" gym env prefetch") + print( + " " + + shlex.join( + [ + "bash", + str(OSWORLD_RUNTIME_DEPS_INSTALLER), + str(agent_venv), + ] + ) + ) print(" gym env start") print(" gym eval run --no-serve") diff --git a/benchmarks/osworld/tests/test_prepare.py b/benchmarks/osworld/tests/test_prepare.py index 5c593e47ac..0cf036af0e 100644 --- a/benchmarks/osworld/tests/test_prepare.py +++ b/benchmarks/osworld/tests/test_prepare.py @@ -189,7 +189,7 @@ def test_opensandbox_backend_adds_pool_provider_config() -> None: ) -def test_main_writes_complete_nano_omni_profile(monkeypatch, tmp_path: Path) -> None: +def test_main_writes_complete_nano_omni_profile(monkeypatch, tmp_path: Path, capsys) -> None: vm_path = tmp_path / "Ubuntu.qcow2" vm_path.write_bytes(b"qcow2-base") env_path = tmp_path / "env.yaml" @@ -227,6 +227,15 @@ def test_main_writes_complete_nano_omni_profile(monkeypatch, tmp_path: Path) -> assert agent["sandbox_provider"] == "osworld_sandbox" assert agent["vm_path"] == str(vm_path.resolve()) + output = capsys.readouterr().out + managed_venv = tmp_path / "server-venvs/responses_api_agents/osworld_agent/.venv" + assert "the OSWorld runtime package install is an explicit opt-in" in output + assert "gym env prefetch" in output + assert "install_optional_runtime_deps.sh" in output + assert str(managed_venv) in output + assert output.index("gym env prefetch") < output.index("install_optional_runtime_deps.sh") + assert output.index("install_optional_runtime_deps.sh") < output.index("gym env start") + def test_write_task_shards_are_disjoint_complete_and_manifested(tmp_path: Path) -> None: source = tmp_path / "tasks.jsonl" diff --git a/benchmarks/osworld/tests/test_run_scripts.py b/benchmarks/osworld/tests/test_run_scripts.py index a2b6f7ef30..e6dc10b8a8 100644 --- a/benchmarks/osworld/tests/test_run_scripts.py +++ b/benchmarks/osworld/tests/test_run_scripts.py @@ -16,9 +16,11 @@ CLEANUP_RUN_SCRIPT = REPO_ROOT / "benchmarks/osworld/tools/cleanup_run.sh" OPENSANDBOX_CLEANUP_SCRIPT = REPO_ROOT / "benchmarks/osworld/tools/cleanup_opensandbox_run.py" OSWORLD_AGENT_CONFIG = REPO_ROOT / "responses_api_agents/osworld_agent/configs/osworld_agent.yaml" +OSWORLD_AGENT_APP = REPO_ROOT / "responses_api_agents/osworld_agent/app.py" OSWORLD_AGENT_REQUIREMENTS = REPO_ROOT / "responses_api_agents/osworld_agent/requirements.txt" OSWORLD_AGENT_OVERRIDES = REPO_ROOT / "responses_api_agents/osworld_agent/overrides.txt" OSWORLD_RUNTIME_DEPS_SCRIPT = REPO_ROOT / "responses_api_agents/osworld_agent/install_optional_runtime_deps.sh" +OSWORLD_RUNTIME_DEPS_CHECKER = REPO_ROOT / "responses_api_agents/osworld_agent/runtime_dependencies.py" @pytest.mark.parametrize( @@ -59,6 +61,17 @@ def test_start_control_preflights_native_build_toolchain() -> None: assert "python3-dev" in text +def test_start_control_requires_explicit_osworld_runtime_setup() -> None: + text = START_CONTROL_SCRIPT.read_text(encoding="utf-8") + + assert "runtime_dependencies.py" in text + assert "OSWORLD_AGENT_VENV" in text + assert "gym env prefetch" in text + assert "install_optional_runtime_deps.sh" in text + assert "uv pip install" not in text + assert "require_optional_runtime_dependencies()" in OSWORLD_AGENT_APP.read_text(encoding="utf-8") + + def test_managed_osworld_agent_installs_opensandbox_sdk() -> None: requirements = OSWORLD_AGENT_REQUIREMENTS.read_text(encoding="utf-8").splitlines() overrides = OSWORLD_AGENT_OVERRIDES.read_text(encoding="utf-8").splitlines() @@ -74,6 +87,7 @@ def test_managed_osworld_agent_installs_opensandbox_sdk() -> None: assert "matplotlib==3.10.6" in overrides assert "agp-client; sys_platform == 'never'" in overrides assert "--no-config" in runtime_script + assert '"numpy<2"' in runtime_script assert "cryptography~=46.0" in runtime_script assert "opencv-python-headless~=4.8.1.78" in runtime_script assert "torchvision==0.26.0" in runtime_script @@ -100,6 +114,11 @@ def test_role_checks_cover_environment_and_model_contracts() -> None: assert "/models" in model_text assert "/chat/completions" in model_text compile(model_text, str(MODEL_PROBE_SCRIPT), "exec") + compile( + OSWORLD_RUNTIME_DEPS_CHECKER.read_text(encoding="utf-8"), + str(OSWORLD_RUNTIME_DEPS_CHECKER), + "exec", + ) def test_cleanup_is_scoped_to_the_run_id() -> None: diff --git a/benchmarks/osworld/tools/README.md b/benchmarks/osworld/tools/README.md index d45362e957..7656847886 100644 --- a/benchmarks/osworld/tools/README.md +++ b/benchmarks/osworld/tools/README.md @@ -7,7 +7,7 @@ configuration entry point; host checks and lifecycle wrappers live here: ```text model host -> probe_model_endpoint.py environment host -> check_environment.sh -agent/control -> prepare.py -> start_control.sh -> run_eval.sh +agent/control -> prepare.py -> prefetch/opt-in deps -> start_control.sh -> run_eval.sh abnormal recovery -> cleanup_run.sh -> cleanup_opensandbox_run.py ``` @@ -15,7 +15,7 @@ abnormal recovery -> cleanup_run.sh -> cleanup_opensandbox_run.py | --- | --- | | `probe_model_endpoint.py` | Require the configured model identity and optionally exercise the one- or three-image chat-completions request shape | | `check_environment.sh` | Validate local or SSH-reached Linux/Docker/KVM/qcow2 environment-host readiness | -| `start_control.sh` | Preflight the agent/control build toolchain, then run `gym env start` | +| `start_control.sh` | Preflight the build toolchain and explicitly prepared OSWorld agent venv, then run `gym env start` | | `run_eval.sh` | Supervisor-friendly wrapper around `gym eval run --no-serve` | | `cleanup_run.sh` | Recovery-only cleanup for stale processes or labeled Sandbox containers after abnormal termination | | `cleanup_opensandbox_run.py` | Read-only audit or opt-in reaping of OpenSandbox instances matching one exact run ID | @@ -56,6 +56,13 @@ Both runtime wrappers require `OSWORLD_RUN_ID`. Set address. Their optional positional argument selects the root for logs and results; it defaults to the Gym repository root. +`prepare.py` prints the exact `gym env prefetch` and +`install_optional_runtime_deps.sh` commands for the configured managed agent +venv. The install remains an explicit opt-in because these packages are +excluded from Gym's shipped environments. `start_control.sh` validates their +versions and imports and prints the same remediation if they are not ready; it +does not install anything automatically. + For a split-host Gym Docker deployment, run the wrappers on the agent/control host and point its normal Docker CLI at the OSWorld environment host. The identical qcow2 path and non-interactive SSH authorization are one-time host diff --git a/benchmarks/osworld/tools/start_control.sh b/benchmarks/osworld/tools/start_control.sh index 7d58ffd5f2..21c80197be 100755 --- a/benchmarks/osworld/tools/start_control.sh +++ b/benchmarks/osworld/tools/start_control.sh @@ -9,12 +9,18 @@ CONTROL_HOST=${NEMO_GYM_CONTROL_HOST:-127.0.0.1} GYM_BIN=${GYM_BIN:-${GYM_ROOT}/.venv/bin/gym} GYM_PYTHON=${GYM_PYTHON:-$(dirname "${GYM_BIN}")/python} ENV_FILE=${GYM_ROOT}/benchmarks/osworld/env.yaml +RUNTIME_DEPS_CHECKER=${GYM_ROOT}/responses_api_agents/osworld_agent/runtime_dependencies.py +RUNTIME_DEPS_INSTALLER=${GYM_ROOT}/responses_api_agents/osworld_agent/install_optional_runtime_deps.sh STATE_DIR=${RUN_ROOT}/run/osworld/${RUN_ID} PID_FILE=${STATE_DIR}/control.pid [[ -x "${GYM_BIN}" ]] || { echo "Gym executable is not available: ${GYM_BIN}" >&2; exit 2; } [[ -x "${GYM_PYTHON}" ]] || { echo "Gym Python is not available: ${GYM_PYTHON}" >&2; exit 2; } [[ -r "${ENV_FILE}" ]] || { echo "prepared Gym environment is not readable: ${ENV_FILE}" >&2; exit 2; } +[[ -r "${RUNTIME_DEPS_CHECKER}" ]] || { + echo "OSWorld runtime dependency checker is not readable: ${RUNTIME_DEPS_CHECKER}" >&2 + exit 2 +} command -v cc >/dev/null 2>&1 || { echo "A C compiler is required to build the OSWorld agent environment" >&2 exit 2 @@ -25,6 +31,27 @@ python_include=$("${GYM_PYTHON}" -c 'import sysconfig; print(sysconfig.get_path( exit 2 } +if [[ -z "${OSWORLD_AGENT_VENV:-}" ]]; then + OSWORLD_AGENT_VENV=$("${GYM_PYTHON}" "${RUNTIME_DEPS_CHECKER}" resolve-venv \ + --gym-root "${GYM_ROOT}" --env-file "${ENV_FILE}") +fi +osworld_agent_python=${OSWORLD_AGENT_VENV}/bin/python +print_runtime_setup() { + echo "Prepare the OSWorld agent venv and explicitly opt in to its runtime packages:" >&2 + printf ' cd %q\n' "$(dirname "${ENV_FILE}")" >&2 + echo " gym env prefetch" >&2 + printf ' bash %q %q\n' "${RUNTIME_DEPS_INSTALLER}" "${OSWORLD_AGENT_VENV}" >&2 +} +if [[ ! -x "${osworld_agent_python}" ]]; then + echo "Managed OSWorld agent Python is not executable: ${osworld_agent_python}" >&2 + print_runtime_setup + exit 2 +fi +if ! "${osworld_agent_python}" "${RUNTIME_DEPS_CHECKER}" check; then + print_runtime_setup + exit 2 +fi + case "${DOCKER_HOST:-}" in ssh://*|tcp://*) [[ -n "${OSWORLD_SANDBOX_PUBLISH_HOST:-}" ]] || { diff --git a/responses_api_agents/osworld_agent/README.md b/responses_api_agents/osworld_agent/README.md index 426725f60f..c2da713c02 100644 --- a/responses_api_agents/osworld_agent/README.md +++ b/responses_api_agents/osworld_agent/README.md @@ -43,6 +43,7 @@ service endpoints, status, and cleanup. | `adapter_agents.py` | Gym-owned model scaffolds, including `NemotronV3NanoOmniAgent` | | `action_parser.py` | Gym pyautogui/control-action parsing and validation | | `proxy.py` | Explicit proxy-task configuration validation and non-secret provenance | +| `runtime_dependencies.py` | Version/import readiness check and explicit-install remediation for excluded packages | | `sandbox_desktop_env.py` | Scoped `DesktopEnv` compatibility wiring for the Gym Sandbox backend | | `sandbox_provider.py` | OSWorld provider contract backed by Gym Sandbox lifecycle and endpoints | @@ -160,6 +161,12 @@ python3 prepare.py \ --execution-backend gym_sandbox \ --vm-path /absolute/path/to/Ubuntu.qcow2 +# Explicitly opt in to packages excluded from Gym's shipped environments. +# prepare.py prints these commands with the exact configured venv path. +gym env prefetch +bash ../../responses_api_agents/osworld_agent/install_optional_runtime_deps.sh \ + ../../responses_api_agents/osworld_agent/.venv + # Terminal 1: start configured servers. gym env start @@ -167,6 +174,14 @@ gym env start gym eval run --no-serve ``` +The installer targets only the managed OSWorld agent venv. It does not modify +the system Python, Gym's root venv, the model server, or the OSWorld VM. The +public `benchmarks/osworld/tools/start_control.sh` wrapper checks that the +required package versions are importable and fails with the exact setup +commands when this explicit step has been omitted. The agent entrypoint repeats +that non-mutating check so a direct `gym env start` also fails early and +actionably; neither path installs packages automatically. + Choose a model-specific agent composition during preparation. For example: ```bash diff --git a/responses_api_agents/osworld_agent/app.py b/responses_api_agents/osworld_agent/app.py index 04d1097c05..f3b912676c 100644 --- a/responses_api_agents/osworld_agent/app.py +++ b/responses_api_agents/osworld_agent/app.py @@ -1029,4 +1029,7 @@ def _empty_response( if __name__ == "__main__": + from responses_api_agents.osworld_agent.runtime_dependencies import require_optional_runtime_dependencies + + require_optional_runtime_dependencies() OSWorldAgent.run_webserver() diff --git a/responses_api_agents/osworld_agent/install_optional_runtime_deps.sh b/responses_api_agents/osworld_agent/install_optional_runtime_deps.sh index ba69d96568..5240ed6870 100755 --- a/responses_api_agents/osworld_agent/install_optional_runtime_deps.sh +++ b/responses_api_agents/osworld_agent/install_optional_runtime_deps.sh @@ -14,21 +14,28 @@ fi venv_path=$1 venv_python="${venv_path}/bin/python" +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +runtime_checker="${script_dir}/runtime_dependencies.py" if [[ ! -x "${venv_python}" ]]; then echo "OSWorld agent Python is not executable: ${venv_python}" >&2 exit 2 fi +if [[ ! -r "${runtime_checker}" ]]; then + echo "OSWorld runtime dependency checker is not readable: ${runtime_checker}" >&2 + exit 2 +fi -if "${venv_python}" -c "import cryptography, cv2, torchvision" 2>/dev/null; then - echo "[osworld-runtime-deps] Already installed, skipping." +if "${venv_python}" "${runtime_checker}" check --quiet; then + echo "[osworld-runtime-deps] Required versions are already installed and importable; skipping." exit 0 fi echo "[osworld-runtime-deps] Installing opt-in runtime dependencies..." uv pip install --no-config --python "${venv_python}" \ + "numpy<2" \ "cryptography~=46.0" \ "opencv-python-headless~=4.8.1.78" \ "torchvision==0.26.0" -"${venv_python}" -c "import cryptography, cv2, torchvision" +"${venv_python}" "${runtime_checker}" check echo "[osworld-runtime-deps] Done." diff --git a/responses_api_agents/osworld_agent/runtime_dependencies.py b/responses_api_agents/osworld_agent/runtime_dependencies.py new file mode 100644 index 0000000000..49d246014a --- /dev/null +++ b/responses_api_agents/osworld_agent/runtime_dependencies.py @@ -0,0 +1,204 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Validate the OSWorld agent's explicitly installed runtime dependencies.""" + +from __future__ import annotations + +import argparse +import importlib +import importlib.metadata +import shlex +import sys +from collections.abc import Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import yaml +from packaging.specifiers import SpecifierSet +from packaging.version import InvalidVersion, Version + + +OSWORLD_AGENT_RELATIVE_DIR = Path("responses_api_agents/osworld_agent") + + +@dataclass(frozen=True) +class RuntimeDependency: + """One dependency excluded from Gym's default managed environments.""" + + distribution: str + import_name: str + specifier: str + + @property + def requirement(self) -> str: + return f"{self.distribution}{self.specifier}" + + +OPTIONAL_RUNTIME_DEPENDENCIES = ( + # OpenCV 4.8 wheels use NumPy's 1.x ABI. This repeats the normal agent + # requirement so the opt-in installer can repair a drifted managed venv. + RuntimeDependency("numpy", "numpy", "<2"), + RuntimeDependency("cryptography", "cryptography", "~=46.0"), + RuntimeDependency("opencv-python-headless", "cv2", "~=4.8.1.78"), + RuntimeDependency("torchvision", "torchvision", "==0.26.0"), +) + + +def managed_agent_venv_path(gym_root: Path, venv_root: Path | None = None) -> Path: + """Return Gym's managed venv path for the OSWorld agent server.""" + + gym_root = gym_root.expanduser().resolve() + resolved_venv_root = gym_root if venv_root is None else venv_root.expanduser().resolve() + agent_dir = gym_root / OSWORLD_AGENT_RELATIVE_DIR + if resolved_venv_root == gym_root: + return agent_dir / ".venv" + return resolved_venv_root / OSWORLD_AGENT_RELATIVE_DIR / ".venv" + + +def managed_agent_venv_from_env(gym_root: Path, env_file: Path) -> Path: + """Resolve the OSWorld agent venv from a prepared Gym ``env.yaml``.""" + + env_file = env_file.expanduser().resolve() + payload: Any = yaml.safe_load(env_file.read_text(encoding="utf-8")) + if payload is None: + payload = {} + if not isinstance(payload, dict): + raise ValueError(f"Gym environment must be a YAML mapping: {env_file}") + + raw_venv_root = payload.get("uv_venv_dir") + if raw_venv_root is None: + return managed_agent_venv_path(gym_root) + if not isinstance(raw_venv_root, str) or not raw_venv_root.strip(): + raise ValueError(f"uv_venv_dir must be a non-empty path string in {env_file}") + if "${" in raw_venv_root: + raise ValueError( + f"uv_venv_dir contains an unresolved interpolation in {env_file}; " + "set OSWORLD_AGENT_VENV to the resolved agent venv path" + ) + + venv_root = Path(raw_venv_root).expanduser() + if not venv_root.is_absolute(): + venv_root = env_file.parent / venv_root + return managed_agent_venv_path(gym_root, venv_root) + + +def validate_optional_runtime_dependencies( + dependencies: Sequence[RuntimeDependency] = OPTIONAL_RUNTIME_DEPENDENCIES, +) -> tuple[str, ...]: + """Return actionable problems in the current Python environment.""" + + problems: list[str] = [] + version_ready: list[RuntimeDependency] = [] + invalid_distributions: set[str] = set() + for dependency in dependencies: + try: + installed_version = importlib.metadata.version(dependency.distribution) + except importlib.metadata.PackageNotFoundError: + problems.append(f"{dependency.requirement}: package is not installed") + invalid_distributions.add(dependency.distribution) + continue + + try: + installed = Version(installed_version) + except InvalidVersion: + problems.append( + f"{dependency.requirement}: installed version {installed_version!r} is not a valid version" + ) + invalid_distributions.add(dependency.distribution) + else: + if installed not in SpecifierSet(dependency.specifier): + problems.append( + f"{dependency.requirement}: installed version {installed_version!r} " + "does not satisfy the requirement" + ) + invalid_distributions.add(dependency.distribution) + else: + version_ready.append(dependency) + + for dependency in version_ready: + # Importing OpenCV against NumPy 2 prints a native ABI traceback before + # Python can catch the ImportError. The NumPy version error is already + # sufficient and more actionable, so avoid that noisy dependent import. + if dependency.import_name == "cv2" and "numpy" in invalid_distributions: + continue + try: + importlib.import_module(dependency.import_name) + except Exception as exc: # noqa: BLE001 - binary import errors must be reported as readiness failures. + problems.append( + f"{dependency.requirement}: import {dependency.import_name!r} failed ({type(exc).__name__}: {exc})" + ) + return tuple(problems) + + +def require_optional_runtime_dependencies( + *, + venv_path: Path | None = None, + installer: Path | None = None, +) -> None: + """Fail an actual OSWorld agent startup with copyable remediation.""" + + problems = validate_optional_runtime_dependencies() + if not problems: + return + + resolved_venv = Path(sys.prefix).resolve() if venv_path is None else venv_path.expanduser().resolve() + resolved_installer = ( + Path(__file__).resolve().with_name("install_optional_runtime_deps.sh") + if installer is None + else installer.expanduser().resolve() + ) + details = "\n".join(f" - {problem}" for problem in problems) + command = shlex.join(["bash", str(resolved_installer), str(resolved_venv)]) + raise RuntimeError( + "OSWorld optional runtime dependencies are not ready in this agent venv:\n" + f"{details}\n" + "Run the explicit opt-in installer, then start Gym again:\n" + f" {command}" + ) + + +def _build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + + resolve_parser = subparsers.add_parser("resolve-venv", help="print the managed OSWorld agent venv path") + resolve_parser.add_argument("--gym-root", type=Path, required=True) + resolve_parser.add_argument("--env-file", type=Path, required=True) + + check_parser = subparsers.add_parser("check", help="validate this interpreter's OSWorld runtime packages") + check_parser.add_argument("--quiet", action="store_true", help="suppress output when validation succeeds") + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = _build_parser().parse_args(argv) + if args.command == "resolve-venv": + try: + print(managed_agent_venv_from_env(args.gym_root, args.env_file)) + except (OSError, ValueError, yaml.YAMLError) as exc: + print(f"Cannot resolve the managed OSWorld agent venv: {exc}", file=sys.stderr) + return 2 + return 0 + + problems = validate_optional_runtime_dependencies() + if problems: + if not args.quiet: + print( + "OSWorld optional runtime dependencies are missing, incompatible, or unusable:", + file=sys.stderr, + ) + for problem in problems: + print(f" - {problem}", file=sys.stderr) + return 1 + if not args.quiet: + versions = ", ".join( + f"{dependency.distribution}={importlib.metadata.version(dependency.distribution)}" + for dependency in OPTIONAL_RUNTIME_DEPENDENCIES + ) + print(f"OSWorld optional runtime dependencies are ready: {versions}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/responses_api_agents/osworld_agent/tests/test_runtime_dependencies.py b/responses_api_agents/osworld_agent/tests/test_runtime_dependencies.py new file mode 100644 index 0000000000..d61268f52b --- /dev/null +++ b/responses_api_agents/osworld_agent/tests/test_runtime_dependencies.py @@ -0,0 +1,109 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import importlib.metadata +from pathlib import Path + +import pytest + +from responses_api_agents.osworld_agent import runtime_dependencies + + +def test_managed_agent_venv_matches_gym_layout(tmp_path: Path) -> None: + gym_root = tmp_path / "Gym" + + assert runtime_dependencies.managed_agent_venv_path(gym_root) == ( + gym_root / "responses_api_agents/osworld_agent/.venv" + ) + assert runtime_dependencies.managed_agent_venv_path(gym_root, tmp_path / "server-venvs") == ( + tmp_path / "server-venvs/responses_api_agents/osworld_agent/.venv" + ) + + +def test_managed_agent_venv_reads_relative_env_root(tmp_path: Path) -> None: + gym_root = tmp_path / "Gym" + env_file = gym_root / "benchmarks/osworld/env.yaml" + env_file.parent.mkdir(parents=True) + env_file.write_text("uv_venv_dir: server-venvs\n", encoding="utf-8") + + assert runtime_dependencies.managed_agent_venv_from_env(gym_root, env_file) == ( + env_file.parent / "server-venvs/responses_api_agents/osworld_agent/.venv" + ) + + +def test_runtime_dependency_validation_accepts_compatible_local_wheel_versions(monkeypatch) -> None: + versions = { + "numpy": "1.26.4", + "cryptography": "46.0.7", + "opencv-python-headless": "4.8.1.78", + "torchvision": "0.26.0+cu130", + } + imported: list[str] = [] + monkeypatch.setattr(runtime_dependencies.importlib.metadata, "version", versions.__getitem__) + monkeypatch.setattr(runtime_dependencies.importlib, "import_module", imported.append) + + assert runtime_dependencies.validate_optional_runtime_dependencies() == () + assert imported == ["numpy", "cryptography", "cv2", "torchvision"] + + +def test_runtime_dependency_validation_reports_missing_mismatched_and_broken_imports(monkeypatch) -> None: + versions = { + "numpy": "1.26.4", + "opencv-python-headless": "4.10.0.84", + "torchvision": "0.26.0", + } + + def installed_version(distribution: str) -> str: + if distribution == "cryptography": + raise importlib.metadata.PackageNotFoundError(distribution) + return versions[distribution] + + def import_module(import_name: str) -> None: + if import_name == "torchvision": + raise RuntimeError("operator ABI mismatch") + + monkeypatch.setattr(runtime_dependencies.importlib.metadata, "version", installed_version) + monkeypatch.setattr(runtime_dependencies.importlib, "import_module", import_module) + + problems = runtime_dependencies.validate_optional_runtime_dependencies() + + assert any("cryptography~=46.0: package is not installed" in problem for problem in problems) + assert any("opencv-python-headless~=4.8.1.78" in problem and "does not satisfy" in problem for problem in problems) + assert any("torchvision==0.26.0" in problem and "operator ABI mismatch" in problem for problem in problems) + + +def test_runtime_dependency_validation_rejects_numpy_2(monkeypatch) -> None: + dependencies = ( + runtime_dependencies.RuntimeDependency("numpy", "numpy", "<2"), + runtime_dependencies.RuntimeDependency("opencv-python-headless", "cv2", "~=4.8.1.78"), + ) + versions = {"numpy": "2.5.2", "opencv-python-headless": "4.8.1.78"} + imported: list[str] = [] + monkeypatch.setattr(runtime_dependencies.importlib.metadata, "version", versions.__getitem__) + monkeypatch.setattr(runtime_dependencies.importlib, "import_module", imported.append) + + assert runtime_dependencies.validate_optional_runtime_dependencies(dependencies) == ( + "numpy<2: installed version '2.5.2' does not satisfy the requirement", + ) + assert imported == [] + + +def test_runtime_dependency_startup_error_has_copyable_scoped_installer(monkeypatch, tmp_path: Path) -> None: + monkeypatch.setattr( + runtime_dependencies, + "validate_optional_runtime_dependencies", + lambda: ("torchvision==0.26.0: package is not installed",), + ) + installer = tmp_path / "Gym checkout/osworld_agent/install_optional_runtime_deps.sh" + agent_venv = tmp_path / "managed venv" + + with pytest.raises(RuntimeError) as exc_info: + runtime_dependencies.require_optional_runtime_dependencies( + venv_path=agent_venv, + installer=installer, + ) + + message = str(exc_info.value) + assert "this agent venv" in message + assert "torchvision==0.26.0" in message + assert f"bash '{installer.resolve()}' '{agent_venv.resolve()}'" in message From 7083d6b6cfe27d2fcc3bd6b9c471102df7c88ae0 Mon Sep 17 00:00:00 2001 From: Jeff Peng Date: Thu, 13 Aug 2026 04:47:08 +0800 Subject: [PATCH 08/10] fix(deps): declare yappi compatibility floor Signed-off-by: Jeff Peng --- pyproject.toml | 2 +- uv.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 76ab182897..931757f6bc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -168,7 +168,7 @@ dependencies = [ # yappi: profiling tool # Updated Wed Aug 12, 2026 with yappi==1.7.6 for CPython 3.14 support # License: MIT https://github.com/sumerc/yappi/blob/1d3f7501701e1f050b6dcd6a86fd36aec08185c7/LICENSE - "yappi", + "yappi>=1.7.6", # Ray: Used for distributed processing # Updated Tue Jul 29, 2026 with ray[default]>=2.56.1 diff --git a/uv.lock b/uv.lock index f85222643c..c3f6d21401 100644 --- a/uv.lock +++ b/uv.lock @@ -2690,7 +2690,7 @@ requires-dist = [ { name = "uvloop" }, { name = "vllm", marker = "extra == 'vllm'", specifier = "==0.24.0" }, { name = "wandb" }, - { name = "yappi" }, + { name = "yappi", specifier = ">=1.7.6" }, ] provides-extras = ["all", "vllm", "sandbox", "openshell", "dev"] From 48302914f84438b1a1ba00e42aa10cdd9bc175ac Mon Sep 17 00:00:00 2001 From: Jeff Peng Date: Thu, 13 Aug 2026 13:10:07 +0800 Subject: [PATCH 09/10] fix(osworld): narrow OpenSandbox lifecycle ownership Keep generic sync Sandbox cancellation behavior unchanged, scope health-check bypass to OSWorld Pool specs, and centralize endpoint scheme and header policy in the OpenSandbox provider. Signed-off-by: Jeff Peng --- .../osworld/configs/osworld_opensandbox.yaml | 1 - benchmarks/osworld/prepare.py | 1 + benchmarks/osworld/tests/test_prepare.py | 2 + nemo_gym/sandbox/api.py | 41 +------- .../sandbox/providers/opensandbox/provider.py | 25 ++--- .../osworld_agent/configs/osworld_agent.yaml | 1 + .../tests/test_sandbox_provider.py | 10 +- tests/unit_tests/test_opensandbox_provider.py | 93 ++++--------------- tests/unit_tests/test_sandbox.py | 22 ----- 9 files changed, 44 insertions(+), 152 deletions(-) diff --git a/benchmarks/osworld/configs/osworld_opensandbox.yaml b/benchmarks/osworld/configs/osworld_opensandbox.yaml index 148337c040..6942c4c661 100644 --- a/benchmarks/osworld/configs/osworld_opensandbox.yaml +++ b/benchmarks/osworld/configs/osworld_opensandbox.yaml @@ -19,7 +19,6 @@ osworld_opensandbox: create: request_timeout_s: 1200 timeout_s: 1200 - skip_health_check: true retries: 3 retry_delay_s: 5.0 retry_max_delay_s: 60.0 diff --git a/benchmarks/osworld/prepare.py b/benchmarks/osworld/prepare.py index d3c8b028dd..348c79a97f 100644 --- a/benchmarks/osworld/prepare.py +++ b/benchmarks/osworld/prepare.py @@ -404,6 +404,7 @@ def write_env( " ttl_s: 14400", " ready_timeout_s: 1200", " provider_options:", + " skip_health_check: true", " extensions:", " poolRef: ${oc.env:OPENSANDBOX_POOL_REF,osworld-kvm}", ] diff --git a/benchmarks/osworld/tests/test_prepare.py b/benchmarks/osworld/tests/test_prepare.py index 0cf036af0e..299af5c40e 100644 --- a/benchmarks/osworld/tests/test_prepare.py +++ b/benchmarks/osworld/tests/test_prepare.py @@ -348,6 +348,7 @@ def test_write_env_configures_sdk_compatibility_image_for_opensandbox_pool(tmp_p assert agent["sandbox_require_kvm"] is False assert agent["sandbox_spec"]["image"] == ("${oc.env:OPENSANDBOX_COMPAT_IMAGE,busybox:1.36}") assert agent["sandbox_spec"]["ttl_s"] == 14400 + assert agent["sandbox_spec"]["provider_options"]["skip_health_check"] is True assert agent["sandbox_spec"]["provider_options"]["extensions"]["poolRef"] == ( "${oc.env:OPENSANDBOX_POOL_REF,osworld-kvm}" ) @@ -383,6 +384,7 @@ def test_opensandbox_env_composes_with_strict_inherited_sandbox_spec(tmp_path: P agent = config["osworld_nano_omni_agent"]["responses_api_agents"]["osworld_agent"] assert agent["sandbox_spec"]["image"] == "busybox:1.36" assert agent["sandbox_spec"]["ttl_s"] == 14400 + assert agent["sandbox_spec"]["provider_options"]["skip_health_check"] is True assert agent["sandbox_spec"]["provider_options"]["extensions"]["poolRef"] == "osworld-kvm" diff --git a/nemo_gym/sandbox/api.py b/nemo_gym/sandbox/api.py index f54532cc84..cf46db25a2 100644 --- a/nemo_gym/sandbox/api.py +++ b/nemo_gym/sandbox/api.py @@ -15,7 +15,6 @@ """Provider-neutral public sandbox API.""" import asyncio -import logging import tempfile import threading from collections.abc import Awaitable, Callable, Mapping @@ -38,10 +37,8 @@ T = TypeVar("T") -LOGGER = logging.getLogger(__name__) SYNC_OPERATION_TIMEOUT_S = 3600.0 SYNC_LOOP_CLOSE_TIMEOUT_S = 5.0 -SYNC_CANCELLATION_TIMEOUT_S = 75.0 class AsyncSandbox: @@ -84,7 +81,7 @@ async def start( source_path = tmp_path / f"file-{index}" source_path.write_text(contents, encoding="utf-8") await self._provider.upload_file(handle, source_path, target_path) - except BaseException: + except Exception: await self._provider.close(handle) await self._provider.aclose() self._closed = True @@ -205,11 +202,9 @@ def __init__( *, wait_timeout_s: float = SYNC_OPERATION_TIMEOUT_S, close_timeout_s: float = SYNC_LOOP_CLOSE_TIMEOUT_S, - cancellation_timeout_s: float = SYNC_CANCELLATION_TIMEOUT_S, ) -> None: self._wait_timeout_s = wait_timeout_s self._close_timeout_s = close_timeout_s - self._cancellation_timeout_s = cancellation_timeout_s self._loop = asyncio.new_event_loop() self._ready = threading.Event() self._closed = False @@ -239,9 +234,6 @@ def _wait_for_result(self, operation: str, future: Future[T]) -> T: raise TimeoutError( f"Sandbox.{operation}() timed out waiting for the sync loop after {self._wait_timeout_s:g}s" ) from e - except BaseException: - future.cancel() - raise def call(self, operation: str, func: Callable[[], T]) -> T: self._ensure_can_block(operation) @@ -262,41 +254,14 @@ def invoke() -> None: def run(self, operation: str, awaitable_factory: Callable[[], Awaitable[T]]) -> T: self._ensure_can_block(operation) - completion = threading.Event() - awaitable = awaitable_factory() - - async def tracked_awaitable() -> T: - try: - return await awaitable - finally: - # ``concurrent.futures.Future.cancel()`` becomes done before - # the underlying asyncio Task has finished unwinding. Signal - # the caller only after provider-side cancellation cleanup. - completion.set() - - future = asyncio.run_coroutine_threadsafe(tracked_awaitable(), self._loop) - - def cancel_and_drain() -> None: - future.cancel() - if not completion.wait(timeout=self._cancellation_timeout_s): - LOGGER.warning( - "Sandbox.%s() cancellation cleanup did not finish within %ss", - operation, - self._cancellation_timeout_s, - ) - + future = asyncio.run_coroutine_threadsafe(awaitable_factory(), self._loop) try: return future.result(timeout=self._wait_timeout_s) except FutureTimeoutError as e: - cancel_and_drain() + future.cancel() raise TimeoutError( f"Sandbox.{operation}() timed out waiting for the sync loop after {self._wait_timeout_s:g}s" ) from e - except BaseException: - # Ctrl-C and other caller-side BaseExceptions must not leave the - # provider coroutine running invisibly on the sync loop. - cancel_and_drain() - raise def close(self) -> None: if self._closed: diff --git a/nemo_gym/sandbox/providers/opensandbox/provider.py b/nemo_gym/sandbox/providers/opensandbox/provider.py index 76bddbf24b..a1705cfcc2 100644 --- a/nemo_gym/sandbox/providers/opensandbox/provider.py +++ b/nemo_gym/sandbox/providers/opensandbox/provider.py @@ -959,21 +959,15 @@ async def endpoint( ) -> SandboxEndpoint: """Resolve one client-reachable direct or server-proxied service URL.""" - get_endpoint = getattr(handle.raw, "get_endpoint", None) - if get_endpoint is None: - raise NotImplementedError( - "The installed opensandbox SDK does not expose Sandbox.get_endpoint; " - "sandbox service endpoints require opensandbox>=0.1.15" - ) resolved = await self._await_sdk_operation( - lambda: get_endpoint(port), + lambda: handle.raw.get_endpoint(port), operation="get_endpoint", sandbox_id=handle.sandbox_id, timeout_s=( float(self._connection.request_timeout_s) if self._connection.request_timeout_s is not None else None ), ) - endpoint_url = str(getattr(resolved, "endpoint", "") or "") + endpoint_url = str(resolved.endpoint or "") if not endpoint_url: raise RuntimeError(f"OpenSandbox returned an empty endpoint for sandbox {handle.sandbox_id!r} port {port}") if "://" not in endpoint_url: @@ -984,12 +978,13 @@ async def endpoint( domain_scheme = urlsplit(domain).scheme if "://" in domain else "" scheme = domain_scheme or self._connection.protocol or "http" endpoint_url = f"{scheme}://{endpoint_url.lstrip('/')}" - headers = dict(getattr(resolved, "headers", None) or {}) - if self._connection.use_server_proxy and self._connection.api_key: - # Proxy mode terminates at the trusted OpenSandbox gateway. Direct - # endpoints terminate in untrusted workloads and must never receive - # the management API key. - headers.setdefault("OPEN-SANDBOX-API-KEY", str(self._connection.api_key)) + headers = dict(handle.raw.connection_config.headers) + # Match the SDK's service adapters: connection-wide headers apply to + # every request, while endpoint-specific routing or auth headers win. + # The upstream proxy-auth fix adds the management API key to + # ConnectionConfig.headers only in server-proxy mode, so direct + # sandbox endpoints never receive it. + headers.update(resolved.headers) return SandboxEndpoint(endpoint=endpoint_url, headers=headers) async def _create_once(self, spec: SandboxSpec) -> SandboxHandle: @@ -1061,7 +1056,7 @@ async def _create_once(self, spec: SandboxSpec) -> SandboxHandle: if self._create.skip_health_check: handle = await self._connect_after_create(created_handle, spec) await self._verify_created_handle(handle) - except BaseException: + except Exception: await self._cleanup_failed_create_handle(created_handle) raise return handle diff --git a/responses_api_agents/osworld_agent/configs/osworld_agent.yaml b/responses_api_agents/osworld_agent/configs/osworld_agent.yaml index c51a7c16e4..6c4b4708a3 100644 --- a/responses_api_agents/osworld_agent/configs/osworld_agent.yaml +++ b/responses_api_agents/osworld_agent/configs/osworld_agent.yaml @@ -29,6 +29,7 @@ osworld_simple_agent: memory_mib: 16384 disk_gib: 40 provider_options: + skip_health_check: null # OpenSandbox Pool specs set true; other providers ignore null extensions: poolRef: null vm_path: ${oc.env:OSWORLD_VM_PATH,${oc.env:OSWORLD_SANDBOX_VM_PATH,""}} # optional absolute qcow2 path diff --git a/responses_api_agents/osworld_agent/tests/test_sandbox_provider.py b/responses_api_agents/osworld_agent/tests/test_sandbox_provider.py index aa5530724c..49bb66bcb2 100644 --- a/responses_api_agents/osworld_agent/tests/test_sandbox_provider.py +++ b/responses_api_agents/osworld_agent/tests/test_sandbox_provider.py @@ -134,7 +134,10 @@ def test_build_spec_uses_sdk_compatibility_image_for_opensandbox_pool(monkeypatc "entrypoint": ["/run/entry.sh"], "env": {"KVM": "Y"}, "resources": {"cpu": 4, "memory_mib": 16384}, - "provider_options": {"extensions": {"poolRef": "osworld-kvm"}}, + "provider_options": { + "skip_health_check": True, + "extensions": {"poolRef": "osworld-kvm"}, + }, }, ) @@ -147,7 +150,10 @@ def test_build_spec_uses_sdk_compatibility_image_for_opensandbox_pool(monkeypatc assert spec.image == "busybox:1.36" assert spec.ttl_s == 1800 assert spec.ports == osworld_sandbox.OSWORLD_SERVICE_PORTS - assert spec.provider_options == {"extensions": {"poolRef": "osworld-kvm"}} + assert spec.provider_options == { + "skip_health_check": True, + "extensions": {"poolRef": "osworld-kvm"}, + } assert spec.metadata["osworld-provider"] == "gym-opensandbox-sandbox" assert spec.metadata["run-id"] == "opensandbox-run" assert spec.entrypoint is None diff --git a/tests/unit_tests/test_opensandbox_provider.py b/tests/unit_tests/test_opensandbox_provider.py index b1c9b46688..5521ec0df0 100644 --- a/tests/unit_tests/test_opensandbox_provider.py +++ b/tests/unit_tests/test_opensandbox_provider.py @@ -254,7 +254,6 @@ async def test_pool_create_uses_sdk_compatibility_image_and_proxy_auth( create={ "request_timeout_s": 120, "timeout_s": 30, - "skip_health_check": True, }, probe={"command": None}, ) @@ -263,7 +262,10 @@ async def test_pool_create_uses_sdk_compatibility_image_and_proxy_auth( image="busybox:1.36", ttl_s=1800, metadata={"purpose": "osworld"}, - provider_options={"extensions": {"poolRef": "osworld-kvm"}}, + provider_options={ + "skip_health_check": True, + "extensions": {"poolRef": "osworld-kvm"}, + }, ) ) @@ -278,69 +280,23 @@ async def test_pool_create_uses_sdk_compatibility_image_and_proxy_auth( assert create_connection.kwargs["headers"] == { "OPEN-SANDBOX-API-KEY": "pool-api-key" # pragma: allowlist secret } - assert FakeSandbox.connected_args == ("sandbox-1",) - assert FakeSandbox.connected_kwargs["skip_health_check"] is True - - -async def test_pool_connect_cancellation_destroys_known_sdk_sandbox( - monkeypatch: pytest.MonkeyPatch, -) -> None: - lifecycle: list[str] = [] + assert FakeSandbox.connected_args == () - class CancellingSandbox: - def __init__(self) -> None: - self.id = "pooled-cancelled-connect" - - @classmethod - async def create(cls, **_kwargs: Any) -> "CancellingSandbox": - lifecycle.append("create") - return cls() - - @classmethod - async def connect(cls, *_args: Any, **_kwargs: Any) -> "CancellingSandbox": - lifecycle.append("connect") - raise asyncio.CancelledError - - async def kill(self) -> None: - lifecycle.append("kill") - - async def close(self) -> None: - lifecycle.append("close") - - monkeypatch.setattr( - opensandbox_provider, - "_require_opensandbox_sdk", - lambda: (CancellingSandbox, FakeConnectionConfig, object, FakePlatformSpec, FakeVolume), - ) - provider = opensandbox_provider.OpenSandboxProvider( - connection={"domain": "http://sandbox.example"}, - create={"skip_health_check": True}, - probe={"command": None}, - ) - with pytest.raises(asyncio.CancelledError): - await provider._create_once( - SandboxSpec( - image="busybox:1.36", - ready_timeout_s=30, - provider_options={ - "extensions": { - "poolRef": "osworld-kvm", - } - }, - ) +async def test_endpoint_normalizes_missing_scheme_and_merges_sdk_headers() -> None: + class FakeRaw: + connection_config = SimpleNamespace( + headers={ + "OPEN-SANDBOX-API-KEY": "pool-api-key", # pragma: allowlist secret + "X-Shared": "connection", + } ) - assert lifecycle == ["create", "connect", "kill", "close"] - - -async def test_endpoint_normalizes_missing_scheme_and_adds_proxy_auth() -> None: - class FakeRaw: async def get_endpoint(self, port: int) -> Any: assert port == 5000 return SimpleNamespace( endpoint="10.0.0.22:5000", - headers={"X-Route": "sandbox"}, + headers={"X-Route": "sandbox", "X-Shared": "endpoint"}, ) provider = opensandbox_provider.OpenSandboxProvider( @@ -363,13 +319,16 @@ async def get_endpoint(self, port: int) -> Any: assert resolved.endpoint == "https://10.0.0.22:5000" assert resolved.headers == { - "X-Route": "sandbox", "OPEN-SANDBOX-API-KEY": "pool-api-key", # pragma: allowlist secret + "X-Shared": "endpoint", + "X-Route": "sandbox", } async def test_endpoint_uses_configured_protocol_for_domain_without_scheme() -> None: class FakeRaw: + connection_config = SimpleNamespace(headers={}) + async def get_endpoint(self, _port: int) -> Any: return SimpleNamespace(endpoint="sandbox.example:5000", headers={}) @@ -395,6 +354,8 @@ async def get_endpoint(self, _port: int) -> Any: async def test_direct_endpoint_never_receives_management_api_key() -> None: class FakeRaw: + connection_config = SimpleNamespace(headers={}) + async def get_endpoint(self, _port: int) -> Any: return SimpleNamespace(endpoint="http://10.0.0.22:5000", headers={}) @@ -418,22 +379,6 @@ async def get_endpoint(self, _port: int) -> Any: assert resolved.headers == {} -async def test_endpoint_requires_sdk_get_endpoint() -> None: - provider = opensandbox_provider.OpenSandboxProvider( - operations={"retries": 0}, - probe={"command": None}, - ) - with pytest.raises(NotImplementedError, match="opensandbox>=0.1.15"): - await provider.endpoint( - opensandbox_provider.SandboxHandle( - sandbox_id="sandbox-1", - provider_name="opensandbox", - raw=object(), - ), - 5000, - ) - - def test_provider_validation_and_retry_helpers() -> None: with pytest.raises(ValueError, match="image_pull_policy"): opensandbox_provider.validate_image_pull_policy("Sometimes") diff --git a/tests/unit_tests/test_sandbox.py b/tests/unit_tests/test_sandbox.py index 19d20ce056..02a63ba607 100644 --- a/tests/unit_tests/test_sandbox.py +++ b/tests/unit_tests/test_sandbox.py @@ -709,28 +709,6 @@ async def never_finishes() -> None: runner.close() -def test_sync_loop_runner_waits_for_async_cancellation_cleanup() -> None: - runner = _AsyncLoopRunner( - wait_timeout_s=0.01, - cancellation_timeout_s=1.0, - ) - cleanup_finished = threading.Event() - - async def needs_async_cleanup() -> None: - try: - await asyncio.get_running_loop().create_future() - finally: - await asyncio.sleep(0.05) - cleanup_finished.set() - - try: - with pytest.raises(TimeoutError, match="timed out waiting for the sync loop"): - runner.run("blocked", needs_async_cleanup) - assert cleanup_finished.is_set() - finally: - runner.close() - - def test_sync_sandbox_file_operations(tmp_path: Path) -> None: provider = FakeSandboxProvider() with Sandbox(provider) as sandbox: From a1edee0701416b4e75a2672590e27e41e5f20deb Mon Sep 17 00:00:00 2001 From: Jeff Peng Date: Thu, 13 Aug 2026 14:57:47 +0800 Subject: [PATCH 10/10] fix(opensandbox): derive endpoint scheme from SDK config Signed-off-by: Jeff Peng --- nemo_gym/sandbox/providers/opensandbox/provider.py | 9 +++------ tests/unit_tests/test_opensandbox_provider.py | 14 +++++++------- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/nemo_gym/sandbox/providers/opensandbox/provider.py b/nemo_gym/sandbox/providers/opensandbox/provider.py index a1705cfcc2..e96f8fb2fb 100644 --- a/nemo_gym/sandbox/providers/opensandbox/provider.py +++ b/nemo_gym/sandbox/providers/opensandbox/provider.py @@ -971,12 +971,9 @@ async def endpoint( if not endpoint_url: raise RuntimeError(f"OpenSandbox returned an empty endpoint for sandbox {handle.sandbox_id!r} port {port}") if "://" not in endpoint_url: - domain = str(self._connection.domain or "") - # urlsplit("host.example:8080") treats the hostname as a scheme. - # Only read a scheme from a domain that actually contains ``://``; - # otherwise use ConnectionConfig.protocol just as the SDK does. - domain_scheme = urlsplit(domain).scheme if "://" in domain else "" - scheme = domain_scheme or self._connection.protocol or "http" + # Use the SDK handle's effective configuration so environment- + # resolved domains and protocols match the lifecycle request. + scheme = urlsplit(handle.raw.connection_config.get_base_url()).scheme or "http" endpoint_url = f"{scheme}://{endpoint_url.lstrip('/')}" headers = dict(handle.raw.connection_config.headers) # Match the SDK's service adapters: connection-wide headers apply to diff --git a/tests/unit_tests/test_opensandbox_provider.py b/tests/unit_tests/test_opensandbox_provider.py index 5521ec0df0..4f625b57b3 100644 --- a/tests/unit_tests/test_opensandbox_provider.py +++ b/tests/unit_tests/test_opensandbox_provider.py @@ -286,10 +286,11 @@ async def test_pool_create_uses_sdk_compatibility_image_and_proxy_auth( async def test_endpoint_normalizes_missing_scheme_and_merges_sdk_headers() -> None: class FakeRaw: connection_config = SimpleNamespace( + get_base_url=lambda: "https://sandbox.example/v1", headers={ "OPEN-SANDBOX-API-KEY": "pool-api-key", # pragma: allowlist secret "X-Shared": "connection", - } + }, ) async def get_endpoint(self, port: int) -> Any: @@ -325,18 +326,17 @@ async def get_endpoint(self, port: int) -> Any: } -async def test_endpoint_uses_configured_protocol_for_domain_without_scheme() -> None: +async def test_endpoint_uses_effective_sdk_scheme_when_provider_input_is_unset() -> None: class FakeRaw: - connection_config = SimpleNamespace(headers={}) + connection_config = SimpleNamespace( + get_base_url=lambda: "https://gateway.example/v1", + headers={}, + ) async def get_endpoint(self, _port: int) -> Any: return SimpleNamespace(endpoint="sandbox.example:5000", headers={}) provider = opensandbox_provider.OpenSandboxProvider( - connection={ - "domain": "gateway.example:8080/", - "protocol": "https", - }, operations={"retries": 0}, probe={"command": None}, )