feat(sandbox): add privacy-conscious execution audit records and distinct failure states (closes #21)

sandbox.py

@@ -71,15 +71,25 @@ def _safe_target(workdir: str, filename: str) -> str:
             raise ValueError("filename must stay inside the sandbox")
         return str(target)
 
-    def run(self, files: Dict[str, str], script: str) -> Dict[str, Optional[object]]:
-        if not isinstance(files, dict) or len(files) > MAX_FILES:
-            raise ValueError(f"at most {MAX_FILES} files are allowed")
-        if not isinstance(script, str) or len(script.encode("utf-8")) > MAX_SCRIPT_BYTES:
-            raise ValueError("sandbox script is too large")
-        workdir = tempfile.mkdtemp(prefix="lux-sandbox-")
+    def run(
+        self,
+        files: Dict[str, str],
+        script: str,
+        puzzle_id: Optional[str] = None,
+    ) -> Dict[str, Optional[object]]:
+        workdir = None
         job_id = str(uuid.uuid4())
         started = time.monotonic()
+        runtime_name = os.getenv("LUX_DOCKER_RUNTIME", "runc")
+
         try:
+            if not isinstance(files, dict) or len(files) > MAX_FILES:
+                raise ValueError(f"at most {MAX_FILES} files are allowed")
+            if not isinstance(script, str) or len(script.encode("utf-8")) > MAX_SCRIPT_BYTES:
+                raise ValueError("sandbox script is too large")
+
+            workdir = tempfile.mkdtemp(prefix="lux-sandbox-")
+
             for filename, content in files.items():
                 target = self._safe_target(workdir, filename)
                 if not isinstance(content, str) or len(content.encode("utf-8")) > MAX_FILE_BYTES:
@@ -90,52 +100,109 @@ def run(self, files: Dict[str, str], script: str) -> Dict[str, Optional[object]]
             with open(os.path.join(workdir, "run.sh"), "w", encoding="utf-8") as handle:
                 handle.write(script)
             cmd = self.build_command(workdir)
+
             try:
                 completed = subprocess.run(cmd, capture_output=True, timeout=EXECUTION_TIMEOUT)
-                timed_out = False
             except subprocess.TimeoutExpired as error:
+                duration_ms = max(0, round((time.monotonic() - started) * 1000))
                 result = {
                     "passed": False,
                     "exit_code": None,
                     "stdout": (error.stdout or b"").decode("utf-8", errors="ignore"),
                     "stderr": "sandbox execution timed out",
                     "command": cmd,
                     "timed_out": True,
+                    "error": "timeout",
                 }
                 _record_audit(
                     {
                         "job_id": job_id,
-                        "runtime": os.getenv("LUX_DOCKER_RUNTIME", "runc"),
-                        "duration_ms": round((time.monotonic() - started) * 1000),
+                        "puzzle_id": puzzle_id,
+                        "runtime": runtime_name,
+                        "duration_ms": duration_ms,
                         "exit_code": None,
                         "timed_out": True,
+                        "docker_failure": False,
+                        "passed": False,
+                        "result": "timeout",
+                    }
+                )
+                return result
+            except (subprocess.SubprocessError, OSError) as error:
+                duration_ms = max(0, round((time.monotonic() - started) * 1000))
+                result = {
+                    "passed": False,
+                    "exit_code": None,
+                    "stdout": "",
+                    "stderr": f"docker execution failed: {error}",
+                    "command": cmd,
+                    "timed_out": False,
+                    "docker_failure": True,
+                    "error": "docker_failure",
+                }
+                _record_audit(
+                    {
+                        "job_id": job_id,
+                        "puzzle_id": puzzle_id,
+                        "runtime": runtime_name,
+                        "duration_ms": duration_ms,
+                        "exit_code": None,
+                        "timed_out": False,
+                        "docker_failure": True,
                         "passed": False,
+                        "result": "docker_failure",
                     }
                 )
                 return result
+
+            duration_ms = max(0, round((time.monotonic() - started) * 1000))
             stdout = completed.stdout.decode("utf-8", errors="ignore")
             stderr = completed.stderr.decode("utf-8", errors="ignore")
+            passed = completed.returncode == 0
+            result_status = "success" if passed else "nonzero_exit"
+
             result = {
-                "passed": completed.returncode == 0,
+                "passed": passed,
                 "exit_code": completed.returncode,
                 "stdout": stdout,
                 "stderr": stderr,
                 "command": cmd,
-                "timed_out": timed_out,
+                "timed_out": False,
+                "docker_failure": False,
             }
             _record_audit(
                 {
                     "job_id": job_id,
-                    "runtime": os.getenv("LUX_DOCKER_RUNTIME", "runc"),
-                    "duration_ms": round((time.monotonic() - started) * 1000),
+                    "puzzle_id": puzzle_id,
+                    "runtime": runtime_name,
+                    "duration_ms": duration_ms,
                     "exit_code": completed.returncode,
-                    "timed_out": timed_out,
-                    "passed": result["passed"],
+                    "timed_out": False,
+                    "docker_failure": False,
+                    "passed": passed,
+                    "result": result_status,
                 }
             )
             return result
+        except ValueError as err:
+            duration_ms = max(0, round((time.monotonic() - started) * 1000))
+            _record_audit(
+                {
+                    "job_id": job_id,
+                    "puzzle_id": puzzle_id,
+                    "runtime": runtime_name,
+                    "duration_ms": duration_ms,
+                    "exit_code": None,
+                    "timed_out": False,
+                    "docker_failure": False,
+                    "passed": False,
+                    "result": "validation_error",
+                }
+            )
+            raise err
         finally:
-            shutil.rmtree(workdir, ignore_errors=True)
+            if workdir and os.path.exists(workdir):
+                shutil.rmtree(workdir, ignore_errors=True)
 
 
 _RUNTIME: Optional[DockerSandbox] = None

server.py

@@ -810,8 +810,8 @@ def build_docker_command(source_dir):
     return DockerSandbox().build_command(source_dir)
 
 
-def run_docker(files, script):
-    return get_runtime().run(files, script)
+def run_docker(files, script, puzzle_id=None, **kwargs):
+    return get_runtime().run(files, script, puzzle_id=puzzle_id)
 
 
 def validate(puzzle, attempt, files):
@@ -827,7 +827,10 @@ def validate(puzzle, attempt, files):
 
     if v == "script":
         try:
-            result = run_docker(files, puzzle["test_script"])
+            try:
+                result = run_docker(files, puzzle["test_script"], puzzle_id=puzzle.get("id"))
+            except TypeError:
+                result = run_docker(files, puzzle["test_script"])
             return response(True, correct=result["passed"], output=result)
         except Exception as e:
             return response(False, error=str(e)), 500

tests/test_server_sandbox.py

@@ -1,4 +1,6 @@
 import json
+import os
+import subprocess
 
 import pytest
 
@@ -47,7 +49,11 @@ def fake_run(cmd, capture_output, timeout):
 
     monkeypatch.setattr(sandbox.subprocess, "run", fake_run)
 
-    result = server.run_docker({"answer.c": "int main(void){return 0;}"}, "echo ok")
+    result = server.run_docker(
+        {"answer.c": "int main(void){return 0;}"},
+        "echo ok",
+        puzzle_id="p-101",
+    )
 
     assert result["passed"] is True
     assert captured["capture_output"] is True
@@ -76,17 +82,95 @@ def test_run_rejects_unsafe_and_oversized_inputs():
         runtime.run({str(index): "x" for index in range(sandbox.MAX_FILES + 1)}, "echo ok")
 
 
-def test_run_records_audit_event(monkeypatch):
+def test_run_records_audit_event_fields_and_privacy(monkeypatch):
     events = []
     monkeypatch.setattr(sandbox.sandbox_audit, "record_execution", events.append)
-    monkeypatch.setattr(sandbox.subprocess, "run", lambda *args, **kwargs: _CompletedProcess())
+    monkeypatch.setattr(
+        sandbox.subprocess, "run", lambda *args, **kwargs: _CompletedProcess(returncode=0)
+    )
 
-    result = sandbox.DockerSandbox().run({"answer.sh": "echo ok"}, "echo ok")
+    result = sandbox.DockerSandbox().run(
+        {"answer.sh": "secret_solution_code"},
+        "secret_script",
+        puzzle_id="puzz-99",
+    )
 
     assert result["passed"] is True
     assert len(events) == 1
-    assert events[0]["passed"] is True
-    assert "source" not in events[0]
+    event = events[0]
+    assert event["job_id"] is not None
+    assert event["puzzle_id"] == "puzz-99"
+    assert event["passed"] is True
+    assert event["result"] == "success"
+    assert event["exit_code"] == 0
+    assert event["timed_out"] is False
+    assert event["docker_failure"] is False
+    assert event["duration_ms"] >= 0
+    # Ensure raw submitted source / content is never logged
+    assert "secret_solution_code" not in str(event)
+    assert "secret_script" not in str(event)
+
+
+def test_run_records_distinct_timeout_and_docker_failure(monkeypatch):
+    events = []
+    monkeypatch.setattr(sandbox.sandbox_audit, "record_execution", events.append)
+
+    # 1. Timeout
+    def fake_timeout(*args, **kwargs):
+        raise subprocess.TimeoutExpired(cmd=["docker"], timeout=15)
+
+    monkeypatch.setattr(sandbox.subprocess, "run", fake_timeout)
+    res_timeout = sandbox.DockerSandbox().run({"f.txt": "a"}, "echo 1", puzzle_id="p-time")
+    assert res_timeout["passed"] is False
+    assert res_timeout["timed_out"] is True
+    assert events[-1]["result"] == "timeout"
+    assert events[-1]["timed_out"] is True
+    assert events[-1]["docker_failure"] is False
+
+    # 2. Docker daemon / process failure
+    def fake_docker_error(*args, **kwargs):
+        raise OSError("Docker daemon socket unavailable")
+
+    monkeypatch.setattr(sandbox.subprocess, "run", fake_docker_error)
+    res_err = sandbox.DockerSandbox().run({"f.txt": "a"}, "echo 1", puzzle_id="p-dock")
+    assert res_err["passed"] is False
+    assert res_err["docker_failure"] is True
+    assert events[-1]["result"] == "docker_failure"
+    assert events[-1]["timed_out"] is False
+    assert events[-1]["docker_failure"] is True
+
+
+def test_run_cleans_up_temp_directories(monkeypatch):
+    created_dirs = []
+    original_mkdtemp = sandbox.tempfile.mkdtemp
+
+    def tracked_mkdtemp(*args, **kwargs):
+        d = original_mkdtemp(*args, **kwargs)
+        created_dirs.append(d)
+        return d
+
+    monkeypatch.setattr(sandbox.tempfile, "mkdtemp", tracked_mkdtemp)
+    monkeypatch.setattr(
+        sandbox.subprocess, "run", lambda *args, **kwargs: _CompletedProcess(returncode=0)
+    )
+
+    sandbox.DockerSandbox().run({"ok.txt": "data"}, "echo ok")
+    assert len(created_dirs) == 1
+    assert not os.path.exists(created_dirs[0])
+
+
+def test_audit_write_failure_is_best_effort(monkeypatch):
+    def failing_audit(event):
+        raise OSError("disk full")
+
+    monkeypatch.setattr(sandbox.sandbox_audit, "record_execution", failing_audit)
+    monkeypatch.setattr(
+        sandbox.subprocess, "run", lambda *args, **kwargs: _CompletedProcess(returncode=0)
+    )
+
+    # Should not raise exception
+    res = sandbox.DockerSandbox().run({"ok.txt": "data"}, "echo ok")
+    assert res["passed"] is True
 
 
 def test_audit_logger_writes_jsonl(tmp_path):