diff --git a/sandbox.py b/sandbox.py index b79e5e8..86073ef 100644 --- a/sandbox.py +++ b/sandbox.py @@ -71,15 +71,25 @@ def _safe_target(workdir: str, filename: str) -> str: raise ValueError("filename must stay inside the sandbox") return str(target) - def run(self, files: Dict[str, str], script: str) -> Dict[str, Optional[object]]: - if not isinstance(files, dict) or len(files) > MAX_FILES: - raise ValueError(f"at most {MAX_FILES} files are allowed") - if not isinstance(script, str) or len(script.encode("utf-8")) > MAX_SCRIPT_BYTES: - raise ValueError("sandbox script is too large") - workdir = tempfile.mkdtemp(prefix="lux-sandbox-") + def run( + self, + files: Dict[str, str], + script: str, + puzzle_id: Optional[str] = None, + ) -> Dict[str, Optional[object]]: + workdir = None job_id = str(uuid.uuid4()) started = time.monotonic() + runtime_name = os.getenv("LUX_DOCKER_RUNTIME", "runc") + try: + if not isinstance(files, dict) or len(files) > MAX_FILES: + raise ValueError(f"at most {MAX_FILES} files are allowed") + if not isinstance(script, str) or len(script.encode("utf-8")) > MAX_SCRIPT_BYTES: + raise ValueError("sandbox script is too large") + + workdir = tempfile.mkdtemp(prefix="lux-sandbox-") + for filename, content in files.items(): target = self._safe_target(workdir, filename) if not isinstance(content, str) or len(content.encode("utf-8")) > MAX_FILE_BYTES: @@ -90,10 +100,11 @@ def run(self, files: Dict[str, str], script: str) -> Dict[str, Optional[object]] with open(os.path.join(workdir, "run.sh"), "w", encoding="utf-8") as handle: handle.write(script) cmd = self.build_command(workdir) + try: completed = subprocess.run(cmd, capture_output=True, timeout=EXECUTION_TIMEOUT) - timed_out = False except subprocess.TimeoutExpired as error: + duration_ms = max(0, round((time.monotonic() - started) * 1000)) result = { "passed": False, "exit_code": None, @@ -101,41 +112,97 @@ def run(self, files: Dict[str, str], script: str) -> Dict[str, Optional[object]] "stderr": "sandbox execution timed out", "command": cmd, "timed_out": True, + "error": "timeout", } _record_audit( { "job_id": job_id, - "runtime": os.getenv("LUX_DOCKER_RUNTIME", "runc"), - "duration_ms": round((time.monotonic() - started) * 1000), + "puzzle_id": puzzle_id, + "runtime": runtime_name, + "duration_ms": duration_ms, "exit_code": None, "timed_out": True, + "docker_failure": False, + "passed": False, + "result": "timeout", + } + ) + return result + except (subprocess.SubprocessError, OSError) as error: + duration_ms = max(0, round((time.monotonic() - started) * 1000)) + result = { + "passed": False, + "exit_code": None, + "stdout": "", + "stderr": f"docker execution failed: {error}", + "command": cmd, + "timed_out": False, + "docker_failure": True, + "error": "docker_failure", + } + _record_audit( + { + "job_id": job_id, + "puzzle_id": puzzle_id, + "runtime": runtime_name, + "duration_ms": duration_ms, + "exit_code": None, + "timed_out": False, + "docker_failure": True, "passed": False, + "result": "docker_failure", } ) return result + + duration_ms = max(0, round((time.monotonic() - started) * 1000)) stdout = completed.stdout.decode("utf-8", errors="ignore") stderr = completed.stderr.decode("utf-8", errors="ignore") + passed = completed.returncode == 0 + result_status = "success" if passed else "nonzero_exit" + result = { - "passed": completed.returncode == 0, + "passed": passed, "exit_code": completed.returncode, "stdout": stdout, "stderr": stderr, "command": cmd, - "timed_out": timed_out, + "timed_out": False, + "docker_failure": False, } _record_audit( { "job_id": job_id, - "runtime": os.getenv("LUX_DOCKER_RUNTIME", "runc"), - "duration_ms": round((time.monotonic() - started) * 1000), + "puzzle_id": puzzle_id, + "runtime": runtime_name, + "duration_ms": duration_ms, "exit_code": completed.returncode, - "timed_out": timed_out, - "passed": result["passed"], + "timed_out": False, + "docker_failure": False, + "passed": passed, + "result": result_status, } ) return result + except ValueError as err: + duration_ms = max(0, round((time.monotonic() - started) * 1000)) + _record_audit( + { + "job_id": job_id, + "puzzle_id": puzzle_id, + "runtime": runtime_name, + "duration_ms": duration_ms, + "exit_code": None, + "timed_out": False, + "docker_failure": False, + "passed": False, + "result": "validation_error", + } + ) + raise err finally: - shutil.rmtree(workdir, ignore_errors=True) + if workdir and os.path.exists(workdir): + shutil.rmtree(workdir, ignore_errors=True) _RUNTIME: Optional[DockerSandbox] = None diff --git a/server.py b/server.py index 4ad8d6c..c8c3c33 100644 --- a/server.py +++ b/server.py @@ -810,8 +810,8 @@ def build_docker_command(source_dir): return DockerSandbox().build_command(source_dir) -def run_docker(files, script): - return get_runtime().run(files, script) +def run_docker(files, script, puzzle_id=None, **kwargs): + return get_runtime().run(files, script, puzzle_id=puzzle_id) def validate(puzzle, attempt, files): @@ -827,7 +827,10 @@ def validate(puzzle, attempt, files): if v == "script": try: - result = run_docker(files, puzzle["test_script"]) + try: + result = run_docker(files, puzzle["test_script"], puzzle_id=puzzle.get("id")) + except TypeError: + result = run_docker(files, puzzle["test_script"]) return response(True, correct=result["passed"], output=result) except Exception as e: return response(False, error=str(e)), 500 diff --git a/tests/test_server_sandbox.py b/tests/test_server_sandbox.py index 2063e72..975ce78 100644 --- a/tests/test_server_sandbox.py +++ b/tests/test_server_sandbox.py @@ -1,4 +1,6 @@ import json +import os +import subprocess import pytest @@ -47,7 +49,11 @@ def fake_run(cmd, capture_output, timeout): monkeypatch.setattr(sandbox.subprocess, "run", fake_run) - result = server.run_docker({"answer.c": "int main(void){return 0;}"}, "echo ok") + result = server.run_docker( + {"answer.c": "int main(void){return 0;}"}, + "echo ok", + puzzle_id="p-101", + ) assert result["passed"] is True assert captured["capture_output"] is True @@ -76,17 +82,95 @@ def test_run_rejects_unsafe_and_oversized_inputs(): runtime.run({str(index): "x" for index in range(sandbox.MAX_FILES + 1)}, "echo ok") -def test_run_records_audit_event(monkeypatch): +def test_run_records_audit_event_fields_and_privacy(monkeypatch): events = [] monkeypatch.setattr(sandbox.sandbox_audit, "record_execution", events.append) - monkeypatch.setattr(sandbox.subprocess, "run", lambda *args, **kwargs: _CompletedProcess()) + monkeypatch.setattr( + sandbox.subprocess, "run", lambda *args, **kwargs: _CompletedProcess(returncode=0) + ) - result = sandbox.DockerSandbox().run({"answer.sh": "echo ok"}, "echo ok") + result = sandbox.DockerSandbox().run( + {"answer.sh": "secret_solution_code"}, + "secret_script", + puzzle_id="puzz-99", + ) assert result["passed"] is True assert len(events) == 1 - assert events[0]["passed"] is True - assert "source" not in events[0] + event = events[0] + assert event["job_id"] is not None + assert event["puzzle_id"] == "puzz-99" + assert event["passed"] is True + assert event["result"] == "success" + assert event["exit_code"] == 0 + assert event["timed_out"] is False + assert event["docker_failure"] is False + assert event["duration_ms"] >= 0 + # Ensure raw submitted source / content is never logged + assert "secret_solution_code" not in str(event) + assert "secret_script" not in str(event) + + +def test_run_records_distinct_timeout_and_docker_failure(monkeypatch): + events = [] + monkeypatch.setattr(sandbox.sandbox_audit, "record_execution", events.append) + + # 1. Timeout + def fake_timeout(*args, **kwargs): + raise subprocess.TimeoutExpired(cmd=["docker"], timeout=15) + + monkeypatch.setattr(sandbox.subprocess, "run", fake_timeout) + res_timeout = sandbox.DockerSandbox().run({"f.txt": "a"}, "echo 1", puzzle_id="p-time") + assert res_timeout["passed"] is False + assert res_timeout["timed_out"] is True + assert events[-1]["result"] == "timeout" + assert events[-1]["timed_out"] is True + assert events[-1]["docker_failure"] is False + + # 2. Docker daemon / process failure + def fake_docker_error(*args, **kwargs): + raise OSError("Docker daemon socket unavailable") + + monkeypatch.setattr(sandbox.subprocess, "run", fake_docker_error) + res_err = sandbox.DockerSandbox().run({"f.txt": "a"}, "echo 1", puzzle_id="p-dock") + assert res_err["passed"] is False + assert res_err["docker_failure"] is True + assert events[-1]["result"] == "docker_failure" + assert events[-1]["timed_out"] is False + assert events[-1]["docker_failure"] is True + + +def test_run_cleans_up_temp_directories(monkeypatch): + created_dirs = [] + original_mkdtemp = sandbox.tempfile.mkdtemp + + def tracked_mkdtemp(*args, **kwargs): + d = original_mkdtemp(*args, **kwargs) + created_dirs.append(d) + return d + + monkeypatch.setattr(sandbox.tempfile, "mkdtemp", tracked_mkdtemp) + monkeypatch.setattr( + sandbox.subprocess, "run", lambda *args, **kwargs: _CompletedProcess(returncode=0) + ) + + sandbox.DockerSandbox().run({"ok.txt": "data"}, "echo ok") + assert len(created_dirs) == 1 + assert not os.path.exists(created_dirs[0]) + + +def test_audit_write_failure_is_best_effort(monkeypatch): + def failing_audit(event): + raise OSError("disk full") + + monkeypatch.setattr(sandbox.sandbox_audit, "record_execution", failing_audit) + monkeypatch.setattr( + sandbox.subprocess, "run", lambda *args, **kwargs: _CompletedProcess(returncode=0) + ) + + # Should not raise exception + res = sandbox.DockerSandbox().run({"ok.txt": "data"}, "echo ok") + assert res["passed"] is True def test_audit_logger_writes_jsonl(tmp_path):