fix(rlm): fail closed on interpreter failures

dspy/predict/rlm.py

@@ -22,7 +22,7 @@
 import dspy
 from dspy.adapters.types.tool import Tool
 from dspy.adapters.utils import parse_value, translate_field_type
-from dspy.primitives.code_interpreter import SIMPLE_TYPES, CodeInterpreter, CodeInterpreterError, FinalOutput
+from dspy.primitives.code_interpreter import SIMPLE_TYPES, CodeExecutionError, CodeInterpreter, FinalOutput
 from dspy.primitives.module import Module
 from dspy.primitives.prediction import Prediction
 from dspy.primitives.python_interpreter import PythonInterpreter
@@ -583,7 +583,7 @@ def _execute_code(
         """Execute code in the interpreter, returning the result or an error string."""
         try:
             return repl.execute(code, variables=dict(input_args))
-        except (CodeInterpreterError, SyntaxError) as e:
+        except (CodeExecutionError, SyntaxError) as e:
             return f"[Error] {e}"
 
     def _execute_iteration(
@@ -632,6 +632,7 @@ def forward(self, **input_args) -> Prediction:
 
         Raises:
             ValueError: If required input fields are missing
+            CodeInterpreterError: If the interpreter process or protocol fails
         """
         self._validate_inputs(input_args)
 
@@ -717,6 +718,7 @@ async def aforward(self, **input_args) -> Prediction:
 
         Raises:
             ValueError: If required input fields are missing
+            CodeInterpreterError: If the interpreter process or protocol fails
         """
         self._validate_inputs(input_args)
 

dspy/primitives/__init__.py

@@ -1,5 +1,5 @@
 from dspy.primitives.base_module import BaseModule
-from dspy.primitives.code_interpreter import CodeInterpreter, CodeInterpreterError, FinalOutput
+from dspy.primitives.code_interpreter import CodeExecutionError, CodeInterpreter, CodeInterpreterError, FinalOutput
 from dspy.primitives.example import Example
 from dspy.primitives.module import Module
 from dspy.primitives.prediction import Completions, Prediction
@@ -8,6 +8,7 @@
 
 __all__ = [
     "BaseModule",
+    "CodeExecutionError",
     "CodeInterpreter",
     "Completions",
     "Example",

dspy/primitives/code_interpreter.py

@@ -14,26 +14,15 @@
 
 
 class CodeInterpreterError(RuntimeError):
-    """Error raised during code interpretation.
+    """Base class for errors reported by a code interpreter.
 
-    This exception covers two distinct failure modes:
-
-    1. **Execution errors**: The sandbox ran user code that failed.
-       - NameError, TypeError, ValueError, etc.
-       - Tool call failures (unknown tool, tool raised exception)
-       - These are normal user code errors.
-
-    2. **Protocol errors**: Communication between host and sandbox failed.
-       - Malformed JSON from sandbox
-       - Sandbox process crashed or became unresponsive
-       - Invalid JSON-RPC message structure
-       - These may indicate a corrupted sandbox needing restart.
+    A bare instance indicates that the interpreter process or protocol failed.
+    Recoverable submitted-code failures use :class:`CodeExecutionError`.
+    """
 
-    The error message typically includes the original error type (e.g., "NameError: ...")
-    which can help distinguish the failure mode.
 
-    Note: SyntaxError is raised separately (not wrapped) for invalid Python syntax.
-    """
+class CodeExecutionError(CodeInterpreterError):
+    """Recoverable error raised by code running in a healthy interpreter."""
 
 
 class FinalOutput:
@@ -128,7 +117,8 @@ def execute(
             - None: If no output was produced
 
         Raises:
-            CodeInterpreterError: On runtime errors (undefined vars, tool failures, etc.)
+            CodeExecutionError: On runtime errors in the submitted code or a called tool.
+            CodeInterpreterError: If the interpreter process or communication protocol fails.
             SyntaxError: On invalid Python syntax
 
         Note:

dspy/primitives/python_interpreter.py

@@ -19,9 +19,9 @@
 from os import PathLike
 from typing import Any, Callable
 
-from dspy.primitives.code_interpreter import SIMPLE_TYPES, CodeInterpreterError, FinalOutput
+from dspy.primitives.code_interpreter import SIMPLE_TYPES, CodeExecutionError, CodeInterpreterError, FinalOutput
 
-__all__ = ["PythonInterpreter", "FinalOutput", "CodeInterpreterError"]
+__all__ = ["PythonInterpreter", "FinalOutput", "CodeExecutionError", "CodeInterpreterError"]
 
 logger = logging.getLogger(__name__)
 
@@ -587,8 +587,9 @@ def execute(
 
                 if error_code == JSONRPC_APP_ERRORS["SyntaxError"]:
                     raise SyntaxError(f"Invalid Python syntax. message: {error_message}")
-                else:
-                    raise CodeInterpreterError(f"{error_type}: {error_data.get('args') or error_message}")
+                if error_code in JSONRPC_APP_ERRORS.values():
+                    raise CodeExecutionError(f"{error_type}: {error_data.get('args') or error_message}")
+                raise CodeInterpreterError(f"{error_type}: {error_data.get('args') or error_message}")
 
             # Unexpected message format - neither a recognized method nor a response
             raise CodeInterpreterError(f"Unexpected message format from sandbox: {msg}")

tests/predict/test_rlm.py

@@ -13,7 +13,7 @@
 
 from dspy.adapters.types.tool import Tool
 from dspy.predict.rlm import RLM, _strip_code_fences
-from dspy.primitives.code_interpreter import CodeInterpreterError, FinalOutput
+from dspy.primitives.code_interpreter import CodeExecutionError, CodeInterpreterError, FinalOutput
 from dspy.primitives.prediction import Prediction
 from dspy.primitives.python_interpreter import PythonInterpreter
 from dspy.primitives.repl_types import REPLEntry, REPLHistory, REPLVariable
@@ -556,7 +556,7 @@ def failing_tool() -> str:
             raise RuntimeError("Tool failed!")
 
         mock = MockInterpreter(responses=[
-            CodeInterpreterError("RuntimeError: Tool failed!"),
+            CodeExecutionError("RuntimeError: Tool failed!"),
             FinalOutput({"answer": "recovered"}),
         ])
         rlm = RLM("query -> answer", max_iters=5, interpreter=mock, tools=[failing_tool])
@@ -571,7 +571,7 @@ def failing_tool() -> str:
     def test_runtime_error_history_uses_stripped_code(self):
         """Runtime execution failures should preserve stripped code in history."""
         mock = MockInterpreter(responses=[
-            CodeInterpreterError("NameError: name 'x' is not defined"),
+            CodeExecutionError("NameError: name 'x' is not defined"),
             FinalOutput({"answer": "recovered"}),
         ])
         rlm = RLM("query -> answer", max_iters=5, interpreter=mock)
@@ -616,6 +616,23 @@ def test_syntax_error_from_strip_code_fences_is_recoverable(self):
         assert result.answer == "recovered"
         assert result.trajectory[0]["output"].startswith("[Error]")
 
+    def test_interpreter_failure_propagates(self):
+        """Process and protocol failures must not fall through to LM extraction."""
+        def fail_generated_code(code, variables):
+            if code == "pass":
+                return ""
+            raise CodeInterpreterError("protocol corrupt")
+
+        mock = MockInterpreter(execute_fn=fail_generated_code)
+        rlm = RLM("query -> answer", max_iters=1, interpreter=mock)
+        rlm.generate_action = make_mock_predictor([
+            {"reasoning": "Try code", "code": "print('test')"},
+        ])
+        rlm.extract = make_mock_predictor([{"answer": "hallucinated"}])
+
+        with pytest.raises(CodeInterpreterError, match="protocol corrupt"):
+            rlm.forward(query="test")
+
 
 class TestRLMDynamicSignature:
     """Tests for the dynamically built RLM signatures."""
@@ -812,7 +829,7 @@ def test_syntax_error(self):
     def test_runtime_error(self):
         """Test runtime error handling."""
         with PythonInterpreter(tools={}) as interp:
-            with pytest.raises(CodeInterpreterError):
+            with pytest.raises(CodeExecutionError):
                 interp.execute("undefined_variable")
 
 

tests/primitives/test_python_interpreter.py

@@ -4,7 +4,7 @@
 
 import pytest
 
-from dspy.primitives.code_interpreter import CodeInterpreterError, FinalOutput
+from dspy.primitives.code_interpreter import CodeExecutionError, CodeInterpreterError, FinalOutput
 from dspy.primitives.python_interpreter import PythonInterpreter
 
 pytestmark = pytest.mark.deno
@@ -70,18 +70,27 @@ def test_failure_syntax_error():
 def test_failure_zero_division():
     with PythonInterpreter() as interpreter:
         code = "1+0/0"
-        with pytest.raises(CodeInterpreterError, match="ZeroDivisionError"):
+        with pytest.raises(CodeExecutionError, match="ZeroDivisionError"):
             interpreter.execute(code)
 
 
 def test_exception_args():
     with PythonInterpreter() as interpreter:
         token = random.randint(1, 10**9)
         code = f"raise ValueError({token})"
-        with pytest.raises(CodeInterpreterError, match=rf"ValueError: \[{token}\]"):
+        with pytest.raises(CodeExecutionError, match=rf"ValueError: \[{token}\]"):
             interpreter.execute(code)
 
 
+def test_generated_exception_name_cannot_spoof_interpreter_failure():
+    with PythonInterpreter() as interpreter:
+        with pytest.raises(CodeExecutionError, match="CodeInterpreterError"):
+            interpreter.execute(
+                "class CodeInterpreterError(Exception):\n    pass\nraise CodeInterpreterError('generated failure')"
+            )
+        assert interpreter.execute("2 + 2") == 4
+
+
 def test_submit_with_list():
     """Test SUBMIT() with a list argument returns FinalOutput with dict format."""