diff --git a/packages/sandboxed_gym/src/sandboxed_gym/host/opensandbox.py b/packages/sandboxed_gym/src/sandboxed_gym/host/opensandbox.py index 3ad82eda55..2b4e07d78b 100644 --- a/packages/sandboxed_gym/src/sandboxed_gym/host/opensandbox.py +++ b/packages/sandboxed_gym/src/sandboxed_gym/host/opensandbox.py @@ -193,11 +193,15 @@ async def wait_ready(self, handle: "GymHostHandle[OpenSandboxDriver]", timeout_s while asyncio.get_running_loop().time() < deadline: try: body = await asyncio.to_thread(self._get_json, handle.health_url, handle.headers) + except Exception as exc: + last_error = exc + else: if body.get("status") == "ready": return + error = body.get("error") + if isinstance(error, Mapping) and error.get("code") == "bootstrap_failed": + raise RuntimeError(f"job host {handle.host_id} failed during bootstrap: {error.get('message')}") last_error = RuntimeError(f"host not ready: {body!r}") - except Exception as exc: - last_error = exc await asyncio.sleep(_HEALTH_POLL_S) raise TimeoutError( f"job host {handle.host_id} at {handle.health_url} did not become ready within {timeout_s:g}s" @@ -210,8 +214,14 @@ def _get_json(self, url: str, headers: Mapping[str, str]) -> dict[str, Any]: with urlopen(request, timeout=10) as response: payload = response.read() except HTTPError as exc: - if exc.code == 503: - return {"status": "starting"} + # The Gym host uses an HTTP error response for both a transient bootstrap state and a + # terminal bootstrap failure. Preserve a JSON error envelope so wait_ready() can + # distinguish them; re-raise unrelated/non-JSON proxy errors. + if exc.code in {500, 503}: + try: + return json.loads(exc.read().decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError): + pass raise except URLError: raise diff --git a/packages/sandboxed_gym/src/sandboxed_gym/runtime/gym_host_runtime.py b/packages/sandboxed_gym/src/sandboxed_gym/runtime/gym_host_runtime.py index 4178fc76dc..40a000746f 100644 --- a/packages/sandboxed_gym/src/sandboxed_gym/runtime/gym_host_runtime.py +++ b/packages/sandboxed_gym/src/sandboxed_gym/runtime/gym_host_runtime.py @@ -16,6 +16,7 @@ import socket import subprocess import sys +import traceback from http.server import BaseHTTPRequestHandler, HTTPServer from typing import Any @@ -52,6 +53,7 @@ _DEFAULT_HTTP_PORT = 8080 _READY: bool = False +_BOOTSTRAP_ERROR: str | None = None _RUN_HELPER: Any = None _HEAD_SERVER_CONFIG: Any = None _ROLLOUT_HELPER: Any = None @@ -440,7 +442,10 @@ def do_GET(self) -> None: self.send_response(404) self.end_headers() return - if not _READY: + if _BOOTSTRAP_ERROR is not None: + body = json.dumps(_runtime_error("bootstrap_failed", _BOOTSTRAP_ERROR)).encode("utf-8") + self.send_response(500) + elif not _READY: body = json.dumps({"status": "starting"}).encode("utf-8") self.send_response(503) else: @@ -538,13 +543,21 @@ def log_message(self, format: str, *args: Any) -> None: def main() -> None: - global _READY, _RUN_HELPER, _HEAD_SERVER_CONFIG, _ROLLOUT_HELPER + global _BOOTSTRAP_ERROR, _READY, _RUN_HELPER, _HEAD_SERVER_CONFIG, _ROLLOUT_HELPER Handler.max_request_bytes = _env_int("NMP_MAX_REQUEST_BYTES", Handler.max_request_bytes) Handler.max_response_bytes = _env_int("NMP_MAX_RESPONSE_BYTES", Handler.max_response_bytes) - _RUN_HELPER, _HEAD_SERVER_CONFIG, _ROLLOUT_HELPER = bootstrap_gym_host() - _READY = True + try: + _RUN_HELPER, _HEAD_SERVER_CONFIG, _ROLLOUT_HELPER = bootstrap_gym_host() + _READY = True + except Exception as exc: + # OpenSandbox adds a long-running egress sidecar. If this process exits during bootstrap, + # Kubernetes leaves that sidecar running and the aggregate BatchSandbox remains Pending, + # hiding the real failure from the orchestrator. Keep only the diagnostic HTTP endpoint + # alive; wait_ready() reads this terminal response and immediately destroys the sandbox. + traceback.print_exc() + _BOOTSTRAP_ERROR = f"{type(exc).__name__}: {exc}" port = _env_int("NMP_RUNTIME_HTTP_PORT", _DEFAULT_HTTP_PORT) HTTPServer(("0.0.0.0", port), Handler).serve_forever() diff --git a/packages/sandboxed_gym/tests/conftest.py b/packages/sandboxed_gym/tests/conftest.py index 5e262c47ae..0b39f33c64 100644 --- a/packages/sandboxed_gym/tests/conftest.py +++ b/packages/sandboxed_gym/tests/conftest.py @@ -18,3 +18,10 @@ def isolated_gym_host_process_state(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.delenv("PYTHONPATH", raising=False) # Let the helper mutate an isolated list, then restore the interpreter's original sys.path object. monkeypatch.setattr(runtime.sys, "path", runtime.sys.path.copy()) + + +@pytest.fixture(autouse=True) +def reset_gym_host_server_state(monkeypatch: pytest.MonkeyPatch) -> None: + """Do not let module-level health state leak between HTTP handler tests.""" + monkeypatch.setattr(runtime, "_READY", False) + monkeypatch.setattr(runtime, "_BOOTSTRAP_ERROR", None) diff --git a/packages/sandboxed_gym/tests/test_gym_host_runtime.py b/packages/sandboxed_gym/tests/test_gym_host_runtime.py index 361cf8096e..5a683cb956 100644 --- a/packages/sandboxed_gym/tests/test_gym_host_runtime.py +++ b/packages/sandboxed_gym/tests/test_gym_host_runtime.py @@ -66,6 +66,54 @@ def test_health_not_ready(): server.server_close() +def test_health_reports_terminal_bootstrap_failure(): + runtime._BOOTSTRAP_ERROR = "ConfigPathNotFoundError: qa_unknown_model_type was not found" + server = HTTPServer(("127.0.0.1", 0), runtime.Handler) + port = server.server_address[1] + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + import urllib.error + import urllib.request + + with pytest.raises(urllib.error.HTTPError) as exc: + urllib.request.urlopen(f"http://127.0.0.1:{port}/health", timeout=5) + assert exc.value.code == 500 + body = json.loads(exc.value.read().decode()) + assert body == { + "error": { + "code": "bootstrap_failed", + "message": "ConfigPathNotFoundError: qa_unknown_model_type was not found", + } + } + finally: + server.shutdown() + server.server_close() + + +def test_main_keeps_diagnostic_endpoint_alive_after_bootstrap_failure(monkeypatch): + served = [] + + class FakeServer: + def __init__(self, address, handler): + served.append((address, handler)) + + def serve_forever(self): + served.append("served") + + def fail_bootstrap(): + raise FileNotFoundError("qa_no_such_resources_server") + + monkeypatch.setattr(runtime, "bootstrap_gym_host", fail_bootstrap) + monkeypatch.setattr(runtime, "HTTPServer", FakeServer) + + runtime.main() + + assert runtime._READY is False + assert runtime._BOOTSTRAP_ERROR == "FileNotFoundError: qa_no_such_resources_server" + assert served == [(("0.0.0.0", 8080), runtime.Handler), "served"] + + def test_health_ready(ready_server): import urllib.request diff --git a/packages/sandboxed_gym/tests/test_sandbox_host_entrypoint.py b/packages/sandboxed_gym/tests/test_sandbox_host_entrypoint.py index b03d772af1..b1e1231132 100644 --- a/packages/sandboxed_gym/tests/test_sandbox_host_entrypoint.py +++ b/packages/sandboxed_gym/tests/test_sandbox_host_entrypoint.py @@ -1,6 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +import asyncio import importlib.util import os import subprocess @@ -95,6 +96,28 @@ def test_opensandbox_host_provider_uses_configured_protocol_for_bare_endpoints() assert provider._absolute_url("10.244.6.40:8080") == "http://10.244.6.40:8080" +@requires_opensandbox +def test_opensandbox_host_provider_surfaces_terminal_bootstrap_failure(monkeypatch): + from sandboxed_gym.host.models import GymHostHandle + from sandboxed_gym.host.opensandbox import OpenSandboxGymHostProvider + + provider = OpenSandboxGymHostProvider(connection={"protocol": "http"}) + monkeypatch.setattr( + provider, + "_get_json", + lambda url, headers: { + "error": { + "code": "bootstrap_failed", + "message": "ConfigPathNotFoundError: qa_no_such_resources_server was not found", + } + }, + ) + handle = GymHostHandle(host_id="sandbox-1", health_url="http://host/health", rollout_url="http://host/run") + + with pytest.raises(RuntimeError, match="qa_no_such_resources_server"): + asyncio.run(provider.wait_ready(handle, timeout_s=5)) + + def _gym_host_spec(*, entrypoint: tuple[str, ...] | None = None) -> GymHostSpec: return GymHostSpec( job_id="job-1", diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/gym_sandbox.py b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/gym_sandbox.py index 63fc30ec97..65a62d329c 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/gym_sandbox.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/gym_sandbox.py @@ -368,6 +368,12 @@ def serve_config( "environment_path": "/job/environment" if fileset_environment else None, "sandbox": { "image": plan.runtime_image, + # Preserve the public runner timeout contract in sandboxed mode. Without this the + # host silently falls back to sandboxed-gym's 15-minute readiness default, so a + # submitted ``startup_timeout_s=120`` can remain active long after the caller's + # requested deadline when the runtime fails before opening its health endpoint. + "ready_timeout_s": target.startup_timeout_s, + **({"rollout_timeout_s": target.collection_timeout_s} if target.collection_timeout_s is not None else {}), # One claim, two sub-paths. The environment mount is read-only and the workspace is not, # so they must not resolve to the same directory. "environment_pvc_claim": environment_pvc_claim, diff --git a/plugins/nemo-evaluator/tests/test_gym_sandbox.py b/plugins/nemo-evaluator/tests/test_gym_sandbox.py index 1cef3ef1f5..da9f962c4f 100644 --- a/plugins/nemo-evaluator/tests/test_gym_sandbox.py +++ b/plugins/nemo-evaluator/tests/test_gym_sandbox.py @@ -170,6 +170,17 @@ def test_serve_config_takes_cluster_facts_from_the_deployment_not_the_job() -> N assert payload["gym_global_config"]["config_paths"] +def test_serve_config_preserves_runner_timeouts_in_sandboxed_mode() -> None: + payload = serve_config( + target(startup_timeout_s=123.0, collection_timeout_s=456.0), + capable_plan(), + job_id="job-7", + ) + + assert payload["sandbox"]["ready_timeout_s"] == 123.0 + assert payload["sandbox"]["rollout_timeout_s"] == 456.0 + + def test_sandbox_server_protocol_reaches_the_opensandbox_host_provider() -> None: plan = resolve_sandbox_plan( capable_config(),