feat(adapter): render typed history frames

dspy/adapters/base.py

@@ -88,15 +88,17 @@ def plan_fields(
         messages: list[LMMessage] = []
         user_parts: list[LMPart] = []
         tools: list[LMToolSpec] = []
+        history_has_open_episode = False
 
         for name, field in list(prompt_signature.input_fields.items()):
             if name not in inputs:
                 continue
             value = inputs[name]
             if field.annotation == History:
                 prompt_signature = prompt_signature.delete(name)
-                messages.extend(self._history_to_lm_messages(prompt_signature, value))
+                messages.extend(value.to_lm_messages(self, prompt_signature))
                 inputs.pop(name, None)
+                history_has_open_episode = value.has_open_episode()
             elif field.annotation == Image:
                 prompt_signature = prompt_signature.delete(name)
                 user_parts.extend([LMTextPart(text=f"\n\n[[ ## {name} ## ]]\n"), self._image_to_lm_part(value)])
@@ -128,6 +130,10 @@ def plan_fields(
                 prompt_signature = prompt_signature.delete(tool_call_output_field_name).delete(tool_call_input_field_name)
                 inputs.pop(tool_call_input_field_name, None)
 
+        if history_has_open_episode:
+            for input_name in list(prompt_signature.input_fields):
+                inputs.pop(input_name, None)
+
         for name, field in list(prompt_signature.output_fields.items()):
             if field.annotation == Reasoning and Reasoning in self.native_response_types:
                 reasoning_signature = self._plan_native_reasoning(prompt_signature, name, lm, lm_kwargs)
@@ -208,13 +214,7 @@ def _last_user_message_index(self, messages: list[LMMessage]) -> int | None:
         return None
 
     def _history_to_lm_messages(self, signature: type[Signature], history: History) -> list[LMMessage]:
-        messages: list[LMMessage] = []
-        for turn in history.messages:
-            messages.append(LMMessage(role="user", parts=[LMTextPart(text=self.format_user_message_content(signature, turn))]))
-            messages.append(
-                LMMessage(role="assistant", parts=[LMTextPart(text=self.format_assistant_message_content(signature, turn))])
-            )
-        return messages
+        return history.to_lm_messages(self, signature)
 
     def _image_to_lm_part(self, image: Image) -> LMImagePart:
         source = image.url
@@ -301,7 +301,7 @@ def parse_response(self, plan: dict[str, Any], response: LMResponse, lm: BaseLM)
                     if field_name not in value:
                         value[field_name] = None
             elif output.tool_calls and tool_call_output_field_name:
-                value = {field_name: None for field_name in original_signature.output_fields.keys()}
+                value = dict.fromkeys(original_signature.output_fields.keys())
             else:
                 raise AdapterParseError(
                     adapter_name=type(self).__name__,
@@ -426,14 +426,18 @@ def format(
         # If the signature and inputs have conversation history, we need to format the conversation history and
         # remove the history field from the signature.
         history_field_name = self._get_history_field_name(signature)
+        has_open_episode = False
         if history_field_name:
+            history_obj = inputs_copy.get(history_field_name)
+            has_open_episode = hasattr(history_obj, "has_open_episode") and history_obj.has_open_episode()
+
             # In order to format the conversation history, we need to remove the history field from the signature.
             signature_without_history = signature.delete(history_field_name)
-            conversation_history = self.format_conversation_history(
-                signature_without_history,
-                history_field_name,
-                inputs_copy,
-            )
+            conversation_history = history_obj.to_lm_messages(self, signature_without_history) if history_obj else []
+            inputs_copy.pop(history_field_name, None)
+            if has_open_episode:
+                for input_name in list(signature_without_history.input_fields):
+                    inputs_copy.pop(input_name, None)
 
         messages = []
         system_message = self.format_system_message(signature)
@@ -443,7 +447,8 @@ def format(
             # Conversation history and current input
             content = self.format_user_message_content(signature_without_history, inputs_copy, main_request=True)
             messages.extend(conversation_history)
-            messages.append({"role": "user", "content": content})
+            if content:
+                messages.append({"role": "user", "content": content})
         else:
             # Only current input
             content = self.format_user_message_content(signature, inputs_copy, main_request=True)

dspy/adapters/types/history.py

@@ -3,6 +3,8 @@
 import pydantic
 from pydantic import Field, model_validator
 
+from dspy.core.types import LMMessage
+
 
 class Observation(pydantic.BaseModel):
     """External result produced by executing or evaluating a history frame."""
@@ -115,6 +117,97 @@ def has_open_episode(self) -> bool:
                 last_boundary = "output"
         return last_boundary == "input"
 
+    def to_lm_messages(self, adapter: Any, signature: type[Any]) -> list[LMMessage]:
+        messages: list[LMMessage] = []
+        for entry in self.frames:
+            frame = self._entry_to_frame(signature, entry)
+            if frame.inputs:
+                content = adapter.format_user_message_content(signature, frame.inputs)
+                if self._has_content(content):
+                    messages.append(self._content_message("user", content))
+
+            if frame.outputs:
+                messages.append(self._content_message("assistant", self._format_outputs(adapter, signature, frame.outputs)))
+            if frame.observations:
+                messages.append(self._content_message("user", self._format_observations(frame.observations)))
+        return messages
+
+    @staticmethod
+    def _entry_to_frame(signature: type[Any], entry: HistoryEntry) -> HistoryFrame:
+        if isinstance(entry, HistoryFrame):
+            return entry
+
+        inputs = {key: value for key, value in entry.items() if key in signature.input_fields}
+        outputs = {key: value for key, value in entry.items() if key in signature.output_fields}
+        unknown = {key: value for key, value in entry.items() if key not in inputs and key not in outputs}
+        if unknown and not outputs:
+            outputs = unknown
+        elif unknown:
+            outputs = {**outputs, **unknown}
+        return HistoryFrame(inputs=inputs, outputs=outputs, complete=True)
+
+    def _format_outputs(self, adapter: Any, signature: type[Any], outputs: dict[str, Any]) -> str:
+        signature_outputs = {key: value for key, value in outputs.items() if key in signature.output_fields}
+        unknown_outputs = {key: value for key, value in outputs.items() if key not in signature.output_fields}
+        if signature_outputs and not unknown_outputs:
+            return adapter.format_assistant_message_content(
+                signature,
+                signature_outputs,
+                missing_field_message="Not supplied for this conversation history message. ",
+            )
+
+        sections = []
+        if signature_outputs:
+            sections.append(
+                adapter.format_assistant_message_content(
+                    signature,
+                    signature_outputs,
+                    missing_field_message="Not supplied for this conversation history message. ",
+                ).strip()
+            )
+        for key, value in unknown_outputs.items():
+            sections.append(f"[[ ## {key} ## ]]\n{self._format_observation_content(value)}")
+        sections.append("[[ ## completed ## ]]")
+        return "\n\n".join(section for section in sections if section)
+
+    def _format_observations(self, observations: list[Observation]) -> str:
+        rendered = []
+        for idx, observation in enumerate(observations):
+            label = "Error" if observation.is_error else "Observation"
+            content = self._format_observation_content(observation.value)
+            subject = self._observation_subject(observation, idx)
+            if "\n" in content:
+                rendered.append(f"{subject}:\n{label}:\n{content}")
+            else:
+                rendered.append(f"{subject}:\n{label}: {content}")
+        observations_text = "\n\n".join(rendered)
+        return f"[[ ## observations ## ]]\n{observations_text}"
+
+    @staticmethod
+    def _observation_subject(observation: Observation, idx: int) -> str:
+        if observation.call_id is not None or observation.name is not None or observation.source == "tool":
+            tool_name = observation.name or f"unknown_{idx + 1}"
+            return f"Tool call {idx + 1} (`{tool_name}`)"
+        if observation.source:
+            return f"{observation.source} observation {idx + 1}"
+        return f"Observation {idx + 1}"
+
+    @staticmethod
+    def _format_observation_content(content: Any) -> str:
+        if isinstance(content, list):
+            return "\n".join(str(item) for item in content)
+        return str(content)
+
+    @staticmethod
+    def _has_content(content: Any) -> bool:
+        if isinstance(content, str):
+            return bool(content.strip())
+        return bool(content)
+
+    @staticmethod
+    def _content_message(role: str, content: Any) -> LMMessage:
+        return LMMessage.model_validate({"role": role, "content": content})
+
 
 def truncate_oldest_actions(history: History, *, max_tokens: int = 200_000, keep_n: int = 3) -> None:
     if len(str(history.frames)) // 4 <= max_tokens:

tests/adapters/test_history_formatting.py

@@ -0,0 +1,65 @@
+import json
+
+import dspy
+from dspy.adapters.types.history import History, Observation
+
+
+def add(a: int, b: int) -> int:
+    """Add two numbers."""
+    return a + b
+
+
+class AgentTurn(dspy.Signature):
+    question: str = dspy.InputField()
+    history: dspy.History = dspy.InputField()
+    tools: list[dspy.Tool] = dspy.InputField()
+    next_thought: str = dspy.OutputField()
+    tool_calls: dspy.ToolCalls = dspy.OutputField()
+
+
+def _history_with_open_tool_turn() -> History:
+    history = History(frames=[])
+    history.append_inputs({"question": "What is 1+2?"})
+    history.append_outputs(
+        {
+            "next_thought": "I should add.",
+            "tool_calls": dspy.ToolCalls.from_dict_list([{"name": "add", "args": {"a": 1, "b": 2}}]),
+        },
+        observations=[Observation(value=3, source="tool", name="add")],
+    )
+    return history
+
+
+def test_chat_adapter_formats_open_history_without_duplicate_input():
+    messages = dspy.ChatAdapter().format(
+        AgentTurn,
+        [],
+        {"question": "What is 1+2?", "history": _history_with_open_tool_turn(), "tools": [dspy.Tool(add)]},
+    )
+
+    texts = [message.text or "" for message in messages]
+
+    assert [message.role for message in messages] == ["system", "user", "assistant", "user", "user"]
+    assert sum("What is 1+2?" in text for text in texts[1:]) == 1
+    assert "[[ ## next_thought ## ]]\nI should add." in texts[2]
+    assert "[[ ## tool_calls ## ]]" in texts[2]
+    assert "Tool call 1 (`add`):\nObservation: 3" in texts[3]
+    assert "Respond with the corresponding output fields" in texts[4]
+
+
+def test_json_adapter_formats_open_history_without_duplicate_input():
+    messages = dspy.JSONAdapter().format(
+        AgentTurn,
+        [],
+        {"question": "What is 1+2?", "history": _history_with_open_tool_turn(), "tools": [dspy.Tool(add)]},
+    )
+
+    texts = [message.text or "" for message in messages]
+    assistant = json.loads(texts[2])
+
+    assert [message.role for message in messages] == ["system", "user", "assistant", "user", "user"]
+    assert sum("What is 1+2?" in text for text in texts[1:]) == 1
+    assert assistant["next_thought"] == "I should add."
+    assert assistant["tool_calls"]["tool_calls"][0]["function"]["name"] == "add"
+    assert "Tool call 1 (`add`):\nObservation: 3" in texts[3]
+    assert "Respond with a JSON object" in texts[4]