fix(dspy): skip FailedPrediction steps in BootstrapFinetune finetune data prep

dspy/teleprompt/bootstrap_finetune.py

@@ -11,7 +11,7 @@
 from dspy.predict.predict import Predict
 from dspy.primitives.example import Example
 from dspy.primitives.module import Module
-from dspy.teleprompt.bootstrap_trace import bootstrap_trace_data
+from dspy.teleprompt.bootstrap_trace import FailedPrediction, bootstrap_trace_data
 from dspy.teleprompt.teleprompt import Teleprompter
 
 logger = logging.getLogger(__name__)
@@ -176,7 +176,13 @@ def _prepare_finetune_data(self, trace_data: list[dict[str, Any]], lm: LM, pred_
         adapter = self.adapter[lm] or settings.adapter or ChatAdapter()
         data_format = infer_data_format(adapter)
         for item in trace_data:
-            for trace_pred_ind, _ in enumerate(item["trace"]):
+            for trace_pred_ind, trace_instance in enumerate(item["trace"]):
+                if isinstance(trace_instance[2], FailedPrediction):
+                    logger.debug(
+                        "Skipping a trace step with a FailedPrediction output "
+                        "(unparseable LM response) while preparing finetune data."
+                    )
+                    continue
                 if pred_ind is None or trace_pred_ind == pred_ind:
                     call_data = build_call_data_from_trace(
                         trace=item["trace"],

tests/teleprompt/test_bootstrap_finetune.py

@@ -4,7 +4,9 @@
 from dspy import Example
 from dspy.predict import Predict
 from dspy.teleprompt import BootstrapFinetune
+from dspy.teleprompt.bootstrap_trace import FailedPrediction
 from dspy.utils.dummies import DummyLM
+from dspy.utils.exceptions import AdapterParseError
 
 
 # Define a simple metric function for testing
@@ -122,3 +124,106 @@ def test_prepare_finetune_data_includes_all_predictors_without_filter():
         {"inputs": {"input": "predictor-0"}, "outputs": {"output": "zero"}},
         {"inputs": {"input": "predictor-1"}, "outputs": {"output": "one"}},
     ]
+
+
+def make_trace_with_failed_prediction():
+    """Step 0 is a valid prediction; step 1 is a FailedPrediction (unparseable LM response)."""
+    return [
+        {
+            "trace": [
+                (Predict("input -> output"), {"input": "predictor-0"}, {"output": "zero"}),
+                (
+                    Predict("input -> output"),
+                    {"input": "predictor-1"},
+                    FailedPrediction(completion_text="garbage", format_reward=-1),
+                ),
+            ]
+        }
+    ]
+
+
+def test_prepare_finetune_data_skips_failed_prediction():
+    """A FailedPrediction trace step is skipped instead of being forwarded to format_finetune_data."""
+    bootstrap = BootstrapFinetune(adapter=TraceIdentityAdapter(), exclude_demos=True)
+
+    data, _ = bootstrap._prepare_finetune_data(
+        trace_data=make_trace_with_failed_prediction(),
+        lm=DummyLM([]),
+        pred_ind=None,
+    )
+
+    assert data == [{"inputs": {"input": "predictor-0"}, "outputs": {"output": "zero"}}]
+
+
+def test_prepare_finetune_data_skips_failed_prediction_with_metric():
+    """The metric score filter retains failed entries (truthy -1 score); the FailedPrediction
+    guard must still prevent them from reaching format_finetune_data."""
+    trace = make_trace_with_failed_prediction()
+    trace[0]["score"] = -1  # truthy, so it survives the `if d["score"]` filter
+    bootstrap = BootstrapFinetune(metric=simple_metric, adapter=TraceIdentityAdapter(), exclude_demos=True)
+
+    data, _ = bootstrap._prepare_finetune_data(
+        trace_data=trace,
+        lm=DummyLM([]),
+        pred_ind=None,
+    )
+
+    assert data == [{"inputs": {"input": "predictor-0"}, "outputs": {"output": "zero"}}]
+
+
+def test_prepare_finetune_data_does_not_crash_with_default_chat_adapter():
+    """With the default ChatAdapter (the crash site), a FailedPrediction must not reach
+    format_finetune_data, which would otherwise raise AttributeError by calling outputs.get(...)."""
+    bootstrap = BootstrapFinetune(adapter=dspy.ChatAdapter(), exclude_demos=True)
+
+    data, _ = bootstrap._prepare_finetune_data(
+        trace_data=make_trace_with_failed_prediction(),
+        lm=DummyLM([]),
+        pred_ind=None,
+    )
+
+    # The valid step is formatted into chat messages; the FailedPrediction is skipped.
+    assert len(data) == 1
+    assert "messages" in data[0]
+
+
+def _make_module_raising_parse_error(signature, lm, bad_input):
+    """Build a SimpleModule whose forward raises AdapterParseError for `bad_input`
+    and otherwise delegates to its predictor (mirroring an unparseable LM response)."""
+    module = SimpleModule(signature)
+    module.set_lm(lm)
+    predictor = module.predictor
+
+    def forward(**kwargs):
+        if kwargs.get("input") == bad_input:
+            raise AdapterParseError(
+                adapter_name="ChatAdapter",
+                signature=predictor.signature,
+                lm_response="this is total garbage that does not parse at all",
+                parsed_result=None,
+            )
+        return predictor(**kwargs)
+
+    module.forward = forward
+    return module
+
+
+def test_compile_does_not_crash_when_all_bootstrapped_steps_are_failed_predictions():
+    """BootstrapFinetune.compile must not crash when every bootstrapped trace step is a
+    FailedPrediction (the end-to-end reproduction of the reported bug)."""
+    lm = DummyLM([{"output": "blue"}])
+    dspy.configure(lm=lm)
+    student = _make_module_raising_parse_error("input -> output", lm, examples[0]["input"])
+
+    bootstrap = BootstrapFinetune(metric=simple_metric)
+
+    with patch.object(bootstrap, "finetune_lms") as mock_finetune:
+        mock_finetune.return_value = {(lm, None): lm}
+        compiled = bootstrap.compile(student, trainset=[examples[0]])
+
+    assert getattr(compiled, "_compiled", False)
+    mock_finetune.assert_called_once()
+    # All trace steps were FailedPredictions and were skipped -> 0 training points.
+    finetune_dict = mock_finetune.call_args[0][0]
+    train_data = next(iter(finetune_dict.values()))["train_data"]
+    assert train_data == []