diff --git a/dspy/teleprompt/bootstrap_finetune.py b/dspy/teleprompt/bootstrap_finetune.py index 3437c9a3d8..799f51208d 100644 --- a/dspy/teleprompt/bootstrap_finetune.py +++ b/dspy/teleprompt/bootstrap_finetune.py @@ -11,7 +11,7 @@ from dspy.predict.predict import Predict from dspy.primitives.example import Example from dspy.primitives.module import Module -from dspy.teleprompt.bootstrap_trace import bootstrap_trace_data +from dspy.teleprompt.bootstrap_trace import FailedPrediction, bootstrap_trace_data from dspy.teleprompt.teleprompt import Teleprompter logger = logging.getLogger(__name__) @@ -176,7 +176,13 @@ def _prepare_finetune_data(self, trace_data: list[dict[str, Any]], lm: LM, pred_ adapter = self.adapter[lm] or settings.adapter or ChatAdapter() data_format = infer_data_format(adapter) for item in trace_data: - for trace_pred_ind, _ in enumerate(item["trace"]): + for trace_pred_ind, trace_instance in enumerate(item["trace"]): + if isinstance(trace_instance[2], FailedPrediction): + logger.debug( + "Skipping a trace step with a FailedPrediction output " + "(unparseable LM response) while preparing finetune data." + ) + continue if pred_ind is None or trace_pred_ind == pred_ind: call_data = build_call_data_from_trace( trace=item["trace"], diff --git a/tests/teleprompt/test_bootstrap_finetune.py b/tests/teleprompt/test_bootstrap_finetune.py index 97ad8e1147..7dca1cb8a4 100644 --- a/tests/teleprompt/test_bootstrap_finetune.py +++ b/tests/teleprompt/test_bootstrap_finetune.py @@ -4,7 +4,9 @@ from dspy import Example from dspy.predict import Predict from dspy.teleprompt import BootstrapFinetune +from dspy.teleprompt.bootstrap_trace import FailedPrediction from dspy.utils.dummies import DummyLM +from dspy.utils.exceptions import AdapterParseError # Define a simple metric function for testing @@ -122,3 +124,106 @@ def test_prepare_finetune_data_includes_all_predictors_without_filter(): {"inputs": {"input": "predictor-0"}, "outputs": {"output": "zero"}}, {"inputs": {"input": "predictor-1"}, "outputs": {"output": "one"}}, ] + + +def make_trace_with_failed_prediction(): + """Step 0 is a valid prediction; step 1 is a FailedPrediction (unparseable LM response).""" + return [ + { + "trace": [ + (Predict("input -> output"), {"input": "predictor-0"}, {"output": "zero"}), + ( + Predict("input -> output"), + {"input": "predictor-1"}, + FailedPrediction(completion_text="garbage", format_reward=-1), + ), + ] + } + ] + + +def test_prepare_finetune_data_skips_failed_prediction(): + """A FailedPrediction trace step is skipped instead of being forwarded to format_finetune_data.""" + bootstrap = BootstrapFinetune(adapter=TraceIdentityAdapter(), exclude_demos=True) + + data, _ = bootstrap._prepare_finetune_data( + trace_data=make_trace_with_failed_prediction(), + lm=DummyLM([]), + pred_ind=None, + ) + + assert data == [{"inputs": {"input": "predictor-0"}, "outputs": {"output": "zero"}}] + + +def test_prepare_finetune_data_skips_failed_prediction_with_metric(): + """The metric score filter retains failed entries (truthy -1 score); the FailedPrediction + guard must still prevent them from reaching format_finetune_data.""" + trace = make_trace_with_failed_prediction() + trace[0]["score"] = -1 # truthy, so it survives the `if d["score"]` filter + bootstrap = BootstrapFinetune(metric=simple_metric, adapter=TraceIdentityAdapter(), exclude_demos=True) + + data, _ = bootstrap._prepare_finetune_data( + trace_data=trace, + lm=DummyLM([]), + pred_ind=None, + ) + + assert data == [{"inputs": {"input": "predictor-0"}, "outputs": {"output": "zero"}}] + + +def test_prepare_finetune_data_does_not_crash_with_default_chat_adapter(): + """With the default ChatAdapter (the crash site), a FailedPrediction must not reach + format_finetune_data, which would otherwise raise AttributeError by calling outputs.get(...).""" + bootstrap = BootstrapFinetune(adapter=dspy.ChatAdapter(), exclude_demos=True) + + data, _ = bootstrap._prepare_finetune_data( + trace_data=make_trace_with_failed_prediction(), + lm=DummyLM([]), + pred_ind=None, + ) + + # The valid step is formatted into chat messages; the FailedPrediction is skipped. + assert len(data) == 1 + assert "messages" in data[0] + + +def _make_module_raising_parse_error(signature, lm, bad_input): + """Build a SimpleModule whose forward raises AdapterParseError for `bad_input` + and otherwise delegates to its predictor (mirroring an unparseable LM response).""" + module = SimpleModule(signature) + module.set_lm(lm) + predictor = module.predictor + + def forward(**kwargs): + if kwargs.get("input") == bad_input: + raise AdapterParseError( + adapter_name="ChatAdapter", + signature=predictor.signature, + lm_response="this is total garbage that does not parse at all", + parsed_result=None, + ) + return predictor(**kwargs) + + module.forward = forward + return module + + +def test_compile_does_not_crash_when_all_bootstrapped_steps_are_failed_predictions(): + """BootstrapFinetune.compile must not crash when every bootstrapped trace step is a + FailedPrediction (the end-to-end reproduction of the reported bug).""" + lm = DummyLM([{"output": "blue"}]) + dspy.configure(lm=lm) + student = _make_module_raising_parse_error("input -> output", lm, examples[0]["input"]) + + bootstrap = BootstrapFinetune(metric=simple_metric) + + with patch.object(bootstrap, "finetune_lms") as mock_finetune: + mock_finetune.return_value = {(lm, None): lm} + compiled = bootstrap.compile(student, trainset=[examples[0]]) + + assert getattr(compiled, "_compiled", False) + mock_finetune.assert_called_once() + # All trace steps were FailedPredictions and were skipped -> 0 training points. + finetune_dict = mock_finetune.call_args[0][0] + train_data = next(iter(finetune_dict.values()))["train_data"] + assert train_data == []