fix(osworld): validate Chrome CDP relay setup

Author: JeffPengCoder

benchmarks/osworld/README.md

@@ -41,6 +41,25 @@ Docker SSH setup below. Start the model first, verify the environment second,
 then prepare and run Gym. Neither the operator workstation nor a persistent
 interactive SSH session is part of runtime communication.
 
+### Chrome CDP port ownership
+
+OSWorld task setup, rather than a deployment script or the Gym adapter, owns
+the guest processes that expose Chrome DevTools. Canonical tasks launch Chrome
+with `--remote-debugging-port=1337` and then launch
+`socat tcp-listen:9222,fork tcp:localhost:1337`. `DesktopEnv.reset()` executes
+both commands from `verifier_metadata.osworld_task.config` for each fresh VM.
+
+The Docker image or OpenSandbox Pool must include `socat` and publish guest
+port 9222; Gym forwards that published HTTP/WebSocket endpoint to OSWorld. A
+user running canonical OSWorld inputs does not need to run either command
+manually. Authors of custom inputs must include the relay whenever their task
+setup starts Chrome CDP on port 1337. `prepare.py` checks this contract and
+fails early with the missing setup command instead of allowing a later 502.
+
+A standalone Sandbox API smoke test that bypasses `DesktopEnv.reset()` must
+start both Chrome and the relay before probing port 9222. Screenshot-only
+checks against the OSWorld service on port 5000 do not require this relay.
+
 ## Requirements
 
 - Linux x86_64 with Docker 20+ and access to the local Docker daemon.

benchmarks/osworld/prepare.py

@@ -21,6 +21,7 @@
 import hashlib
 import json
 import os
+import re
 from collections.abc import Sequence
 from pathlib import Path
 from typing import Any
@@ -67,6 +68,52 @@
 )
 
 
+def _setup_command_texts(task: dict[str, Any]) -> tuple[str, ...]:
+    """Return commands that OSWorld will execute during the task setup."""
+
+    config = task.get("config")
+    if not isinstance(config, list):
+        return ()
+
+    commands: list[str] = []
+    for setup_item in config:
+        if not isinstance(setup_item, dict):
+            continue
+        parameters = setup_item.get("parameters")
+        if not isinstance(parameters, dict):
+            continue
+        command = parameters.get("command")
+        if isinstance(command, str):
+            commands.append(command)
+        elif isinstance(command, list):
+            commands.append(" ".join(str(argument) for argument in command))
+    return tuple(commands)
+
+
+def _validate_chrome_cdp_relay(task: dict[str, Any], *, line_number: int) -> None:
+    """Require OSWorld's guest relay when task setup starts Chrome CDP."""
+
+    commands = _setup_command_texts(task)
+    starts_chrome_cdp = any(
+        re.search(r"--remote-debugging-port(?:=|\s+)1337(?!\d)", command, flags=re.IGNORECASE)
+        for command in commands
+    )
+    has_cdp_relay = any(
+        re.search(r"\bsocat\b", command, flags=re.IGNORECASE)
+        and re.search(r"\blisten:9222(?!\d)", command, flags=re.IGNORECASE)
+        and re.search(r"\b(?:localhost|127\.0\.0\.1):1337(?!\d)", command, flags=re.IGNORECASE)
+        for command in commands
+    )
+    if starts_chrome_cdp and not has_cdp_relay:
+        task_id = str(task.get("id") or "<unknown>")
+        raise ValueError(
+            f"OSWorld row {line_number} task {task_id!r} starts Chrome CDP on guest port 1337 "
+            "but verifier_metadata.osworld_task.config does not launch the required guest relay. "
+            "Add ['socat', 'tcp-listen:9222,fork', 'tcp:localhost:1337']; Gym publishes and "
+            "forwards port 9222 but does not start this task-owned process."
+        )
+
+
 def prepare(input_jsonl: Path = DEFAULT_INPUT) -> Path:
     """Validate and return an OSWorld JSONL suitable for rollout collection."""
 
@@ -88,6 +135,7 @@ def prepare(input_jsonl: Path = DEFAULT_INPUT) -> Path:
             metadata = row.get("verifier_metadata")
             if not isinstance(metadata, dict) or not isinstance(metadata.get("osworld_task"), dict):
                 raise ValueError(f"OSWorld row {line_number} must contain verifier_metadata.osworld_task")
+            _validate_chrome_cdp_relay(metadata["osworld_task"], line_number=line_number)
             row_count += 1
 
     if row_count == 0:

benchmarks/osworld/tests/test_prepare.py

@@ -32,6 +32,58 @@ def test_prepare_validates_committed_example() -> None:
     assert prepare() == DEFAULT_INPUT.resolve()
 
 
+def test_prepare_rejects_chrome_cdp_without_guest_relay(tmp_path: Path) -> None:
+    input_jsonl = tmp_path / "missing-cdp-relay.jsonl"
+    input_jsonl.write_text(
+        json.dumps(
+            {
+                "verifier_metadata": {
+                    "osworld_task": {
+                        "id": "chrome-without-relay",
+                        "config": [
+                            {
+                                "type": "launch",
+                                "parameters": {"command": "google-chrome --remote-debugging-port 1337"},
+                            }
+                        ],
+                    }
+                }
+            }
+        )
+        + "\n",
+        encoding="utf-8",
+    )
+
+    with pytest.raises(ValueError) as exc_info:
+        prepare(input_jsonl)
+
+    message = str(exc_info.value)
+    assert "OSWorld row 1" in message
+    assert "chrome-without-relay" in message
+    assert "tcp-listen:9222" in message
+    assert "task-owned process" in message
+
+
+def test_prepare_does_not_require_cdp_relay_without_chrome_cdp(tmp_path: Path) -> None:
+    input_jsonl = tmp_path / "non-chrome.jsonl"
+    input_jsonl.write_text(
+        json.dumps(
+            {
+                "verifier_metadata": {
+                    "osworld_task": {
+                        "id": "non-chrome-task",
+                        "config": [{"type": "launch", "parameters": {"command": ["libreoffice"]}}],
+                    }
+                }
+            }
+        )
+        + "\n",
+        encoding="utf-8",
+    )
+
+    assert prepare(input_jsonl) == input_jsonl.resolve()
+
+
 @pytest.mark.parametrize(
     "shard_args",
     [[], ["--num-shards", "1", "--shard-index", "0"]],