NVIDIA-NeMo · puhuim · Sep 7, 2026 · coderabbitai · Sep 7, 2026 · coderabbitai
@@ -193,11 +193,15 @@ async def wait_ready(self, handle: "GymHostHandle[OpenSandboxDriver]", timeout_s
         while asyncio.get_running_loop().time() < deadline:
             try:
                 body = await asyncio.to_thread(self._get_json, handle.health_url, handle.headers)
+            except Exception as exc:
+                last_error = exc
+            else:
                 if body.get("status") == "ready":
                     return
+                error = body.get("error")
+                if isinstance(error, Mapping) and error.get("code") == "bootstrap_failed":
+                    raise RuntimeError(f"job host {handle.host_id} failed during bootstrap: {error.get('message')}")
                 last_error = RuntimeError(f"host not ready: {body!r}")
-            except Exception as exc:
-                last_error = exc
             await asyncio.sleep(_HEALTH_POLL_S)
         raise TimeoutError(
             f"job host {handle.host_id} at {handle.health_url} did not become ready within {timeout_s:g}s"
@@ -210,8 +214,14 @@ def _get_json(self, url: str, headers: Mapping[str, str]) -> dict[str, Any]:
             with urlopen(request, timeout=10) as response:
                 payload = response.read()
         except HTTPError as exc:
-            if exc.code == 503:
-                return {"status": "starting"}
+            # The Gym host uses an HTTP error response for both a transient bootstrap state and a
+            # terminal bootstrap failure. Preserve a JSON error envelope so wait_ready() can
+            # distinguish them; re-raise unrelated/non-JSON proxy errors.
+            if exc.code in {500, 503}:
+                try:
+                    return json.loads(exc.read().decode("utf-8"))
+                except (UnicodeDecodeError, json.JSONDecodeError):
+                    pass
             raise
         except URLError:
             raise

@@ -16,6 +16,7 @@
 import socket
 import subprocess
 import sys
+import traceback
 from http.server import BaseHTTPRequestHandler, HTTPServer
 from typing import Any
 
@@ -52,6 +53,7 @@
 
 _DEFAULT_HTTP_PORT = 8080
 _READY: bool = False
+_BOOTSTRAP_ERROR: str | None = None
 _RUN_HELPER: Any = None
 _HEAD_SERVER_CONFIG: Any = None
 _ROLLOUT_HELPER: Any = None
@@ -440,7 +442,10 @@ def do_GET(self) -> None:
             self.send_response(404)
             self.end_headers()
             return
-        if not _READY:
+        if _BOOTSTRAP_ERROR is not None:
+            body = json.dumps(_runtime_error("bootstrap_failed", _BOOTSTRAP_ERROR)).encode("utf-8")
+            self.send_response(500)
+        elif not _READY:
             body = json.dumps({"status": "starting"}).encode("utf-8")
             self.send_response(503)
         else:
@@ -538,13 +543,21 @@ def log_message(self, format: str, *args: Any) -> None:
 
 
 def main() -> None:
-    global _READY, _RUN_HELPER, _HEAD_SERVER_CONFIG, _ROLLOUT_HELPER
+    global _BOOTSTRAP_ERROR, _READY, _RUN_HELPER, _HEAD_SERVER_CONFIG, _ROLLOUT_HELPER
 
     Handler.max_request_bytes = _env_int("NMP_MAX_REQUEST_BYTES", Handler.max_request_bytes)
     Handler.max_response_bytes = _env_int("NMP_MAX_RESPONSE_BYTES", Handler.max_response_bytes)
 
-    _RUN_HELPER, _HEAD_SERVER_CONFIG, _ROLLOUT_HELPER = bootstrap_gym_host()
-    _READY = True
+    try:
+        _RUN_HELPER, _HEAD_SERVER_CONFIG, _ROLLOUT_HELPER = bootstrap_gym_host()
+        _READY = True
+    except Exception as exc:
+        # OpenSandbox adds a long-running egress sidecar. If this process exits during bootstrap,
+        # Kubernetes leaves that sidecar running and the aggregate BatchSandbox remains Pending,
+        # hiding the real failure from the orchestrator. Keep only the diagnostic HTTP endpoint
+        # alive; wait_ready() reads this terminal response and immediately destroys the sandbox.
+        traceback.print_exc()
+        _BOOTSTRAP_ERROR = f"{type(exc).__name__}: {exc}"
 
     port = _env_int("NMP_RUNTIME_HTTP_PORT", _DEFAULT_HTTP_PORT)
     HTTPServer(("0.0.0.0", port), Handler).serve_forever()

@@ -18,3 +18,10 @@ def isolated_gym_host_process_state(monkeypatch: pytest.MonkeyPatch) -> None:
     monkeypatch.delenv("PYTHONPATH", raising=False)
     # Let the helper mutate an isolated list, then restore the interpreter's original sys.path object.
     monkeypatch.setattr(runtime.sys, "path", runtime.sys.path.copy())
+
+
+@pytest.fixture(autouse=True)
+def reset_gym_host_server_state(monkeypatch: pytest.MonkeyPatch) -> None:
+    """Do not let module-level health state leak between HTTP handler tests."""
+    monkeypatch.setattr(runtime, "_READY", False)
+    monkeypatch.setattr(runtime, "_BOOTSTRAP_ERROR", None)
@@ -66,6 +66,54 @@ def test_health_not_ready():
         server.server_close()
 
 
+def test_health_reports_terminal_bootstrap_failure():
+    runtime._BOOTSTRAP_ERROR = "ConfigPathNotFoundError: qa_unknown_model_type was not found"
+    server = HTTPServer(("127.0.0.1", 0), runtime.Handler)
+    port = server.server_address[1]
+    thread = threading.Thread(target=server.serve_forever, daemon=True)
+    thread.start()
+    try:
+        import urllib.error
+        import urllib.request
+
+        with pytest.raises(urllib.error.HTTPError) as exc:
+            urllib.request.urlopen(f"http://127.0.0.1:{port}/health", timeout=5)
+        assert exc.value.code == 500
+        body = json.loads(exc.value.read().decode())
+        assert body == {
+            "error": {
+                "code": "bootstrap_failed",
+                "message": "ConfigPathNotFoundError: qa_unknown_model_type was not found",
+            }
+        }
+    finally:
+        server.shutdown()
+        server.server_close()
+
+
+def test_main_keeps_diagnostic_endpoint_alive_after_bootstrap_failure(monkeypatch):
+    served = []
+
+    class FakeServer:
+        def __init__(self, address, handler):
+            served.append((address, handler))
+
+        def serve_forever(self):
+            served.append("served")
+
+    def fail_bootstrap():
+        raise FileNotFoundError("qa_no_such_resources_server")
+
+    monkeypatch.setattr(runtime, "bootstrap_gym_host", fail_bootstrap)
+    monkeypatch.setattr(runtime, "HTTPServer", FakeServer)
+
+    runtime.main()
+
+    assert runtime._READY is False
+    assert runtime._BOOTSTRAP_ERROR == "FileNotFoundError: qa_no_such_resources_server"
+    assert served == [(("0.0.0.0", 8080), runtime.Handler), "served"]
+
+
 def test_health_ready(ready_server):
     import urllib.request
 

@@ -1,6 +1,7 @@
 # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 
+import asyncio
 import importlib.util
 import os
 import subprocess
@@ -95,6 +96,28 @@ def test_opensandbox_host_provider_uses_configured_protocol_for_bare_endpoints()
     assert provider._absolute_url("10.244.6.40:8080") == "http://10.244.6.40:8080"
 
 
+@requires_opensandbox
+def test_opensandbox_host_provider_surfaces_terminal_bootstrap_failure(monkeypatch):
+    from sandboxed_gym.host.models import GymHostHandle
+    from sandboxed_gym.host.opensandbox import OpenSandboxGymHostProvider
+
+    provider = OpenSandboxGymHostProvider(connection={"protocol": "http"})
+    monkeypatch.setattr(
+        provider,
+        "_get_json",
+        lambda url, headers: {
+            "error": {
+                "code": "bootstrap_failed",
+                "message": "ConfigPathNotFoundError: qa_no_such_resources_server was not found",
+            }
+        },
+    )
+    handle = GymHostHandle(host_id="sandbox-1", health_url="http://host/health", rollout_url="http://host/run")
+
+    with pytest.raises(RuntimeError, match="qa_no_such_resources_server"):
+        asyncio.run(provider.wait_ready(handle, timeout_s=5))
+
+
 def _gym_host_spec(*, entrypoint: tuple[str, ...] | None = None) -> GymHostSpec:
     return GymHostSpec(
         job_id="job-1",

@@ -368,6 +368,12 @@ def serve_config(
         "environment_path": "/job/environment" if fileset_environment else None,
         "sandbox": {
             "image": plan.runtime_image,
+            # Preserve the public runner timeout contract in sandboxed mode. Without this the
+            # host silently falls back to sandboxed-gym's 15-minute readiness default, so a
+            # submitted ``startup_timeout_s=120`` can remain active long after the caller's
+            # requested deadline when the runtime fails before opening its health endpoint.
+            "ready_timeout_s": target.startup_timeout_s,
+            **({"rollout_timeout_s": target.collection_timeout_s} if target.collection_timeout_s is not None else {}),
             # One claim, two sub-paths. The environment mount is read-only and the workspace is not,
             # so they must not resolve to the same directory.
             "environment_pvc_claim": environment_pvc_claim,

@@ -170,6 +170,17 @@ def test_serve_config_takes_cluster_facts_from_the_deployment_not_the_job() -> N
     assert payload["gym_global_config"]["config_paths"]
 
 
+def test_serve_config_preserves_runner_timeouts_in_sandboxed_mode() -> None:
+    payload = serve_config(
+        target(startup_timeout_s=123.0, collection_timeout_s=456.0),
+        capable_plan(),
+        job_id="job-7",
+    )
+
+    assert payload["sandbox"]["ready_timeout_s"] == 123.0
+    assert payload["sandbox"]["rollout_timeout_s"] == 456.0
+
+
 def test_sandbox_server_protocol_reaches_the_opensandbox_host_provider() -> None:
     plan = resolve_sandbox_plan(
         capable_config(),