From f24ae1816ca427356236dfeee2c71f3318d9a7b5 Mon Sep 17 00:00:00 2001 From: isaacbmiller Date: Mon, 13 Jul 2026 13:50:56 -0400 Subject: [PATCH] fix(rlm): surface interpreter failures --- dspy/predict/rlm.py | 6 +++-- dspy/primitives/__init__.py | 3 ++- dspy/primitives/code_interpreter.py | 26 +++++++-------------- dspy/primitives/python_interpreter.py | 9 +++---- tests/predict/test_rlm.py | 25 ++++++++++++++++---- tests/primitives/test_python_interpreter.py | 15 +++++++++--- 6 files changed, 52 insertions(+), 32 deletions(-) diff --git a/dspy/predict/rlm.py b/dspy/predict/rlm.py index 8906d03a0e..6ec6fa6e7f 100644 --- a/dspy/predict/rlm.py +++ b/dspy/predict/rlm.py @@ -22,7 +22,7 @@ import dspy from dspy.adapters.types.tool import Tool from dspy.adapters.utils import parse_value, translate_field_type -from dspy.primitives.code_interpreter import SIMPLE_TYPES, CodeInterpreter, CodeInterpreterError, FinalOutput +from dspy.primitives.code_interpreter import SIMPLE_TYPES, CodeExecutionError, CodeInterpreter, FinalOutput from dspy.primitives.module import Module from dspy.primitives.prediction import Prediction from dspy.primitives.python_interpreter import PythonInterpreter @@ -583,7 +583,7 @@ def _execute_code( """Execute code in the interpreter, returning the result or an error string.""" try: return repl.execute(code, variables=dict(input_args)) - except (CodeInterpreterError, SyntaxError) as e: + except (CodeExecutionError, SyntaxError) as e: return f"[Error] {e}" def _execute_iteration( @@ -632,6 +632,7 @@ def forward(self, **input_args) -> Prediction: Raises: ValueError: If required input fields are missing + CodeInterpreterError: If the interpreter process or protocol fails """ self._validate_inputs(input_args) @@ -717,6 +718,7 @@ async def aforward(self, **input_args) -> Prediction: Raises: ValueError: If required input fields are missing + CodeInterpreterError: If the interpreter process or protocol fails """ self._validate_inputs(input_args) diff --git a/dspy/primitives/__init__.py b/dspy/primitives/__init__.py index c6f26fcb46..f1ad6bf323 100644 --- a/dspy/primitives/__init__.py +++ b/dspy/primitives/__init__.py @@ -1,5 +1,5 @@ from dspy.primitives.base_module import BaseModule -from dspy.primitives.code_interpreter import CodeInterpreter, CodeInterpreterError, FinalOutput +from dspy.primitives.code_interpreter import CodeExecutionError, CodeInterpreter, CodeInterpreterError, FinalOutput from dspy.primitives.example import Example from dspy.primitives.module import Module from dspy.primitives.prediction import Completions, Prediction @@ -8,6 +8,7 @@ __all__ = [ "BaseModule", + "CodeExecutionError", "CodeInterpreter", "Completions", "Example", diff --git a/dspy/primitives/code_interpreter.py b/dspy/primitives/code_interpreter.py index 9eccb79605..d938c8f496 100644 --- a/dspy/primitives/code_interpreter.py +++ b/dspy/primitives/code_interpreter.py @@ -14,26 +14,15 @@ class CodeInterpreterError(RuntimeError): - """Error raised during code interpretation. + """Base class for errors reported by a code interpreter. - This exception covers two distinct failure modes: - - 1. **Execution errors**: The sandbox ran user code that failed. - - NameError, TypeError, ValueError, etc. - - Tool call failures (unknown tool, tool raised exception) - - These are normal user code errors. - - 2. **Protocol errors**: Communication between host and sandbox failed. - - Malformed JSON from sandbox - - Sandbox process crashed or became unresponsive - - Invalid JSON-RPC message structure - - These may indicate a corrupted sandbox needing restart. + A bare instance indicates that the interpreter process or protocol failed. + Recoverable submitted-code failures use :class:`CodeExecutionError`. + """ - The error message typically includes the original error type (e.g., "NameError: ...") - which can help distinguish the failure mode. - Note: SyntaxError is raised separately (not wrapped) for invalid Python syntax. - """ +class CodeExecutionError(CodeInterpreterError): + """Recoverable error raised by code running in a healthy interpreter.""" class FinalOutput: @@ -128,7 +117,8 @@ def execute( - None: If no output was produced Raises: - CodeInterpreterError: On runtime errors (undefined vars, tool failures, etc.) + CodeExecutionError: On runtime errors in the submitted code or a called tool. + CodeInterpreterError: If the interpreter process or communication protocol fails. SyntaxError: On invalid Python syntax Note: diff --git a/dspy/primitives/python_interpreter.py b/dspy/primitives/python_interpreter.py index 066d4f4e3f..7a45bb5ec9 100644 --- a/dspy/primitives/python_interpreter.py +++ b/dspy/primitives/python_interpreter.py @@ -19,9 +19,9 @@ from os import PathLike from typing import Any, Callable -from dspy.primitives.code_interpreter import SIMPLE_TYPES, CodeInterpreterError, FinalOutput +from dspy.primitives.code_interpreter import SIMPLE_TYPES, CodeExecutionError, CodeInterpreterError, FinalOutput -__all__ = ["PythonInterpreter", "FinalOutput", "CodeInterpreterError"] +__all__ = ["PythonInterpreter", "FinalOutput", "CodeExecutionError", "CodeInterpreterError"] logger = logging.getLogger(__name__) @@ -587,8 +587,9 @@ def execute( if error_code == JSONRPC_APP_ERRORS["SyntaxError"]: raise SyntaxError(f"Invalid Python syntax. message: {error_message}") - else: - raise CodeInterpreterError(f"{error_type}: {error_data.get('args') or error_message}") + if error_code in JSONRPC_APP_ERRORS.values(): + raise CodeExecutionError(f"{error_type}: {error_data.get('args') or error_message}") + raise CodeInterpreterError(f"{error_type}: {error_data.get('args') or error_message}") # Unexpected message format - neither a recognized method nor a response raise CodeInterpreterError(f"Unexpected message format from sandbox: {msg}") diff --git a/tests/predict/test_rlm.py b/tests/predict/test_rlm.py index e5a25497d0..69350eb592 100644 --- a/tests/predict/test_rlm.py +++ b/tests/predict/test_rlm.py @@ -13,7 +13,7 @@ from dspy.adapters.types.tool import Tool from dspy.predict.rlm import RLM, _strip_code_fences -from dspy.primitives.code_interpreter import CodeInterpreterError, FinalOutput +from dspy.primitives.code_interpreter import CodeExecutionError, CodeInterpreterError, FinalOutput from dspy.primitives.prediction import Prediction from dspy.primitives.python_interpreter import PythonInterpreter from dspy.primitives.repl_types import REPLEntry, REPLHistory, REPLVariable @@ -556,7 +556,7 @@ def failing_tool() -> str: raise RuntimeError("Tool failed!") mock = MockInterpreter(responses=[ - CodeInterpreterError("RuntimeError: Tool failed!"), + CodeExecutionError("RuntimeError: Tool failed!"), FinalOutput({"answer": "recovered"}), ]) rlm = RLM("query -> answer", max_iters=5, interpreter=mock, tools=[failing_tool]) @@ -571,7 +571,7 @@ def failing_tool() -> str: def test_runtime_error_history_uses_stripped_code(self): """Runtime execution failures should preserve stripped code in history.""" mock = MockInterpreter(responses=[ - CodeInterpreterError("NameError: name 'x' is not defined"), + CodeExecutionError("NameError: name 'x' is not defined"), FinalOutput({"answer": "recovered"}), ]) rlm = RLM("query -> answer", max_iters=5, interpreter=mock) @@ -616,6 +616,23 @@ def test_syntax_error_from_strip_code_fences_is_recoverable(self): assert result.answer == "recovered" assert result.trajectory[0]["output"].startswith("[Error]") + def test_interpreter_failure_propagates(self): + """Process and protocol failures must not fall through to LM extraction.""" + def fail_generated_code(code, variables): + if code == "pass": + return "" + raise CodeInterpreterError("protocol corrupt") + + mock = MockInterpreter(execute_fn=fail_generated_code) + rlm = RLM("query -> answer", max_iters=1, interpreter=mock) + rlm.generate_action = make_mock_predictor([ + {"reasoning": "Try code", "code": "print('test')"}, + ]) + rlm.extract = make_mock_predictor([{"answer": "hallucinated"}]) + + with pytest.raises(CodeInterpreterError, match="protocol corrupt"): + rlm.forward(query="test") + class TestRLMDynamicSignature: """Tests for the dynamically built RLM signatures.""" @@ -812,7 +829,7 @@ def test_syntax_error(self): def test_runtime_error(self): """Test runtime error handling.""" with PythonInterpreter(tools={}) as interp: - with pytest.raises(CodeInterpreterError): + with pytest.raises(CodeExecutionError): interp.execute("undefined_variable") diff --git a/tests/primitives/test_python_interpreter.py b/tests/primitives/test_python_interpreter.py index de49d65bd9..5804e39f88 100644 --- a/tests/primitives/test_python_interpreter.py +++ b/tests/primitives/test_python_interpreter.py @@ -4,7 +4,7 @@ import pytest -from dspy.primitives.code_interpreter import CodeInterpreterError, FinalOutput +from dspy.primitives.code_interpreter import CodeExecutionError, CodeInterpreterError, FinalOutput from dspy.primitives.python_interpreter import PythonInterpreter pytestmark = pytest.mark.deno @@ -70,7 +70,7 @@ def test_failure_syntax_error(): def test_failure_zero_division(): with PythonInterpreter() as interpreter: code = "1+0/0" - with pytest.raises(CodeInterpreterError, match="ZeroDivisionError"): + with pytest.raises(CodeExecutionError, match="ZeroDivisionError"): interpreter.execute(code) @@ -78,10 +78,19 @@ def test_exception_args(): with PythonInterpreter() as interpreter: token = random.randint(1, 10**9) code = f"raise ValueError({token})" - with pytest.raises(CodeInterpreterError, match=rf"ValueError: \[{token}\]"): + with pytest.raises(CodeExecutionError, match=rf"ValueError: \[{token}\]"): interpreter.execute(code) +def test_generated_exception_name_cannot_spoof_interpreter_failure(): + with PythonInterpreter() as interpreter: + with pytest.raises(CodeExecutionError, match="CodeInterpreterError"): + interpreter.execute( + "class CodeInterpreterError(Exception):\n pass\nraise CodeInterpreterError('generated failure')" + ) + assert interpreter.execute("2 + 2") == 4 + + def test_submit_with_list(): """Test SUBMIT() with a list argument returns FinalOutput with dict format."""