diff --git a/dspy/__init__.py b/dspy/__init__.py index 76afdb805d..3c1d7a7e15 100644 --- a/dspy/__init__.py +++ b/dspy/__init__.py @@ -6,7 +6,23 @@ from dspy.evaluate import Evaluate # isort: skip from dspy.clients import * # isort: skip -from dspy.adapters import Adapter, ChatAdapter, JSONAdapter, XMLAdapter, TwoStepAdapter, Image, Audio, File, History, Type, Tool, ToolCalls, Code, Reasoning # isort: skip +from dspy.adapters import ( + Adapter, + ChatAdapter, + JSONAdapter, + XMLAdapter, + TwoStepAdapter, + Image, + Audio, + File, + History, + Type, + Tool, + ToolCalls, + ToolCallResults, + Code, + Reasoning, +) # isort: skip from dspy.primitives.sandbox_serializable import SandboxSerializable # isort: skip from dspy.utils.exceptions import ContextWindowExceededError from dspy.utils.logging_utils import configure_dspy_loggers, disable_logging, enable_logging diff --git a/dspy/adapters/__init__.py b/dspy/adapters/__init__.py index c217d7260e..74e199f180 100644 --- a/dspy/adapters/__init__.py +++ b/dspy/adapters/__init__.py @@ -2,7 +2,7 @@ from dspy.adapters.chat_adapter import ChatAdapter from dspy.adapters.json_adapter import JSONAdapter from dspy.adapters.two_step_adapter import TwoStepAdapter -from dspy.adapters.types import Audio, Code, File, History, Image, Reasoning, Tool, ToolCalls, Type +from dspy.adapters.types import Audio, Code, File, History, Image, Reasoning, Tool, ToolCallResults, ToolCalls, Type from dspy.adapters.xml_adapter import XMLAdapter __all__ = [ @@ -19,5 +19,6 @@ "TwoStepAdapter", "Tool", "ToolCalls", + "ToolCallResults", "Reasoning", ] diff --git a/dspy/adapters/base.py b/dspy/adapters/base.py index efab5c29b3..6b143fd00d 100644 --- a/dspy/adapters/base.py +++ b/dspy/adapters/base.py @@ -1,23 +1,24 @@ import json import logging -from typing import Any, get_origin +from dataclasses import dataclass +from dataclasses import field as dataclass_field +from typing import Any, get_args, get_origin import json_repair from dspy.adapters._legacy_type_markers import ( - _expand_legacy_custom_type_markers_in_chat_message, _expand_legacy_custom_type_markers_in_lm_message, ) from dspy.adapters.types import History, Type from dspy.adapters.types.reasoning import Reasoning -from dspy.adapters.types.tool import Tool, ToolCalls +from dspy.adapters.types.tool import Tool, ToolCallResults, ToolCalls from dspy.clients.base_lm import BaseLM from dspy.clients.openai_format import ( - legacy_outputs_from_lm_response, lm_response_from_legacy_outputs, + message_to_openai_chat, to_openai_chat_request, ) -from dspy.core.types import LMMessage, LMRequest, LMResponse +from dspy.core.types import LMMessage, LMPart, LMRequest, LMResponse, LMTextPart, LMToolSpec from dspy.experimental import Citations from dspy.signatures.signature import Signature from dspy.utils.callback import BaseCallback, with_callbacks @@ -28,6 +29,16 @@ _DEFAULT_NATIVE_RESPONSE_TYPES = [Citations, Reasoning] +@dataclass +class _AdapterRequestState: + source_signature: type[Signature] + render_signature: type[Signature] + inputs: dict[str, Any] + lm_kwargs: dict[str, Any] + tools: list[LMToolSpec] = dataclass_field(default_factory=list) + hidden_output_fields: tuple[str, ...] = () + + class Adapter: """Base Adapter class. @@ -176,6 +187,7 @@ def _call_postprocess( { "name": v["function"]["name"], "args": json_repair.loads(v["function"]["arguments"]), + **({"id": v["id"]} if v.get("id") is not None else {}), } for v in tool_calls ] @@ -205,21 +217,20 @@ def _call_postprocess( def _render_request( self, lm: BaseLM, - lm_kwargs: dict[str, Any], + state: _AdapterRequestState, messages: list[LMMessage | dict[str, Any]], ) -> LMRequest: - """Build the normalized LM request for the current adapter call path. - - TODO(adapters-plan): This currently receives already-rendered messages. - Once planning lands, this should render from `_AdapterPlan` and apply - planned message/part insertions before creating `LMRequest`. - """ + request_kwargs = self._prepare_request_kwargs(lm, state) return LMRequest.from_call( model=lm.model, messages=self._coerce_lm_messages(messages), - **lm_kwargs, + tools=state.tools, + **request_kwargs, ) + def _prepare_request_kwargs(self, lm: BaseLM, state: _AdapterRequestState) -> dict[str, Any]: + return dict(state.lm_kwargs) + def _call_lm(self, lm: BaseLM, request: LMRequest) -> LMResponse: """Call current `BaseLM` through the normalized request/response boundary. @@ -299,15 +310,184 @@ def _chat_dict_to_lm_message(self, message: dict[str, Any]) -> LMMessage: message["content"] = sanitized return LMMessage(**message) - def _normalize_legacy_outputs(self, outputs: list[dict[str, Any] | str | None], request: LMRequest) -> LMResponse: + def _normalize_legacy_outputs( + self, outputs: list[dict[str, Any] | str | None] | LMResponse, request: LMRequest + ) -> LMResponse: """Convert current `BaseLM` outputs into a normalized `LMResponse`. TODO(language-models): Current `BaseLM` returns `list[str | dict | None]`. Future LMs should return `LMResponse` directly, making this method a compatibility-only path for old/custom LMs. """ + if isinstance(outputs, LMResponse): + return outputs return lm_response_from_legacy_outputs(outputs, request) + def _prepare_request_state( + self, + lm: BaseLM, + lm_kwargs: dict[str, Any], + signature: type[Signature], + inputs: dict[str, Any], + ) -> _AdapterRequestState: + """Build the normalized state for one adapter request. + + Preprocessing may rewrite the signature, inputs, and LM kwargs, so this copies caller-owned data before + applying it and records both the source signature used for final outputs and the render signature used for + prompt formatting. + """ + copied_inputs = dict(inputs) + copied_lm_kwargs = dict(lm_kwargs) + render_signature = self._call_preprocess(lm, copied_lm_kwargs, signature, copied_inputs) + tools = self._extract_lm_tools(copied_lm_kwargs) + + hidden_output_fields = tuple( + field_name for field_name in signature.output_fields if field_name not in render_signature.output_fields + ) + return _AdapterRequestState( + source_signature=signature, + render_signature=render_signature, + inputs=copied_inputs, + lm_kwargs=copied_lm_kwargs, + tools=tools, + hidden_output_fields=hidden_output_fields, + ) + + def _extract_lm_tools(self, lm_kwargs: dict[str, Any]) -> list[LMToolSpec]: + """Remove native LM tool specs from LM kwargs and normalize them for request rendering.""" + raw_tools = lm_kwargs.pop("tools", None) + if not raw_tools: + return [] + tools = raw_tools if isinstance(raw_tools, list) else [raw_tools] + return [self._coerce_lm_tool_spec(tool) for tool in tools] + + @staticmethod + def _coerce_lm_tool_spec(tool: Any) -> LMToolSpec: + """Convert supported tool shapes into the internal LMToolSpec representation.""" + if isinstance(tool, LMToolSpec): + return tool + if hasattr(tool, "to_lm_tool_spec"): + return tool.to_lm_tool_spec() + if isinstance(tool, dict): + if "function" in tool: + function = tool["function"] + provider_data = {key: value for key, value in tool.items() if key not in {"type", "function"}} + return LMToolSpec( + name=function.get("name"), + description=function.get("description"), + parameters=function.get("parameters", {}), + provider_data=provider_data, + ) + return LMToolSpec(**tool) + raise TypeError(f"Cannot convert {type(tool)!r} to LMToolSpec.") + + def _prepare_native_tool_result_inputs( + self, + render_signature: type[Signature], + inputs: dict[str, Any], + ) -> tuple[type[Signature], list[LMMessage]]: + messages: list[LMMessage] = [] + for field_name, field_info in list(render_signature.input_fields.items()): + value = inputs.get(field_name) + tool_call_results = self._coerce_tool_call_results_value(value, field_info) + if tool_call_results is None: + if value is None and self._annotation_includes(getattr(field_info, "annotation", None), ToolCallResults): + inputs.pop(field_name, None) + render_signature = render_signature.delete(field_name) + continue + messages.extend(tool_call_results.to_lm_messages()) + inputs.pop(field_name, None) + render_signature = render_signature.delete(field_name) + return render_signature, messages + + def _parse_response(self, state: _AdapterRequestState, response: LMResponse) -> list[dict[str, Any]]: + """Parse normalized LM outputs against the original source signature. + + Text is parsed with the render signature, while native response fields and tool call outputs are restored onto + the source signature so request-time signature rewrites do not drop fields from the final value. + """ + values = [] + tool_call_output_field_name = self._get_tool_call_output_field_name(state.source_signature) + + for output in response.outputs: + if output.metadata.get("empty_legacy_outputs"): + continue + + value: dict[str, Any] = {} + parsed_any = False + + if output.text and state.render_signature.output_fields: + value.update(self.parse(state.render_signature, output.text)) + parsed_any = True + + if output.tool_calls and tool_call_output_field_name: + value[tool_call_output_field_name] = ToolCalls.from_dict_list( + [ + { + "name": tool_call.name, + "args": tool_call.args, + **({"id": tool_call.id} if tool_call.id is not None else {}), + } + for tool_call in output.tool_calls + ] + ) + parsed_any = True + + output_dict = output.to_output_dict() + legacy_output = output.provider_output if output.provider_output is not None else output_dict + for name, field_info in state.source_signature.output_fields.items(): + if ( + isinstance(field_info.annotation, type) + and field_info.annotation in self.native_response_types + and issubclass(field_info.annotation, Type) + ): + parsed_value = field_info.annotation.parse_lm_response(legacy_output) + if parsed_value is not None: + value[name] = parsed_value + parsed_any = True + + for field_name in state.source_signature.output_fields: + value.setdefault(field_name, None) + + if not parsed_any: + raise AdapterParseError( + adapter_name=type(self).__name__, + signature=state.source_signature, + lm_response=str(output_dict), + message="The LM returned an empty or null response.", + ) + + if output.logprobs is not None: + value["logprobs"] = output.logprobs + + values.append(value) + + return values + + @classmethod + def _coerce_tool_call_results_value(cls, field_value: Any, field_info: Any = None) -> ToolCallResults | None: + if field_value is None: + return None + if isinstance(field_value, ToolCallResults): + return field_value + + annotation = getattr(field_info, "annotation", None) + if cls._annotation_includes(annotation, ToolCallResults): + return ToolCallResults.model_validate(field_value) + return None + + @classmethod + def _coerce_tool_calls_value(cls, field_value: Any, field_info: Any = None) -> ToolCalls | None: + if field_value is None: + return None + if isinstance(field_value, ToolCalls): + return field_value + + annotation = getattr(field_info, "annotation", None) + if cls._annotation_includes(annotation, ToolCalls): + return ToolCalls.model_validate(field_value) + return None + def __call__( self, lm: BaseLM, @@ -332,16 +512,11 @@ def __call__( List of dictionaries representing parsed LM responses. Each dictionary contains keys matching the signature's output field names. For multiple generations (n > 1), returns multiple dictionaries. """ - processed_signature = self._call_preprocess(lm, lm_kwargs, signature, inputs) - messages = self.format(processed_signature, demos, inputs) - request = self._render_request(lm, lm_kwargs, messages) + state = self._prepare_request_state(lm, lm_kwargs, signature, inputs) + messages = self.format(state.render_signature, demos, state.inputs) + request = self._render_request(lm, state, messages) response = self._call_lm(lm, request) - # TODO(adapters-response): We normalize at the LM boundary, but still - # convert back to legacy postprocess dictionaries here to keep this PR - # behavior-preserving. Replace with direct `LMResponse` parsing once the - # explicit adapter plan exists. - outputs = legacy_outputs_from_lm_response(response) - return self._call_postprocess(processed_signature, signature, outputs, lm, lm_kwargs) + return self._parse_response(state, response) async def acall( self, @@ -351,14 +526,11 @@ async def acall( demos: list[dict[str, Any]], inputs: dict[str, Any], ) -> list[dict[str, Any]]: - processed_signature = self._call_preprocess(lm, lm_kwargs, signature, inputs) - messages = self.format(processed_signature, demos, inputs) - request = self._render_request(lm, lm_kwargs, messages) + state = self._prepare_request_state(lm, lm_kwargs, signature, inputs) + messages = self.format(state.render_signature, demos, state.inputs) + request = self._render_request(lm, state, messages) response = await self._acall_lm(lm, request) - # TODO(adapters-response): Keep in sync with `__call__()` until both use - # direct `LMResponse` parsing. - outputs = legacy_outputs_from_lm_response(response) - return self._call_postprocess(processed_signature, signature, outputs, lm, lm_kwargs) + return self._parse_response(state, response) def format( self, @@ -406,34 +578,38 @@ def format( A list of multiturn messages as expected by the LM. """ inputs_copy = dict(inputs) + conversation_history: list[LMMessage] = [] + render_signature = signature - # If the signature and inputs have conversation history, we need to format the conversation history and - # remove the history field from the signature. - history_field_name = self._get_history_field_name(signature) - if history_field_name: - # In order to format the conversation history, we need to remove the history field from the signature. - signature_without_history = signature.delete(history_field_name) - conversation_history = self.format_conversation_history( - signature_without_history, - history_field_name, + tool_result_messages: list[LMMessage] = [] + if self.use_native_function_calling: + render_signature, tool_result_messages = self._prepare_native_tool_result_inputs( + render_signature, inputs_copy, ) - messages = [] - system_message = self.format_system_message(signature) - messages.append({"role": "system", "content": system_message}) - messages.extend(self.format_demos(signature, demos)) + input_signature = render_signature + + history_field_name = self._get_history_field_name(render_signature) if history_field_name: - # Conversation history and current input - content = self.format_user_message_content(signature_without_history, inputs_copy, main_request=True) - messages.extend(conversation_history) - messages.append({"role": "user", "content": content}) - else: - # Only current input - content = self.format_user_message_content(signature, inputs_copy, main_request=True) - messages.append({"role": "user", "content": content}) + input_signature = render_signature.delete(history_field_name) + history_obj = inputs_copy.pop(history_field_name, None) + if history_obj is not None: + conversation_history = self.format_history( + history_obj, + input_signature, + use_native_tool_calls=self.use_native_function_calling, + ) + + messages: list[LMMessage | dict[str, Any]] = [ + {"role": "system", "content": self.format_system_message(render_signature)} + ] - return [_expand_legacy_custom_type_markers_in_chat_message(message) for message in messages] + messages.extend(self.format_demos(render_signature, demos)) + messages.extend(conversation_history) + messages.extend(tool_result_messages) + messages.extend(self._format_input_messages(input_signature, inputs_copy, main_request=True)) + return [message_to_openai_chat(message) for message in self._coerce_lm_messages(messages)] def format_system_message(self, signature: type[Signature]) -> str: """Format the system message for the LM call. @@ -546,6 +722,9 @@ def format_demos(self, signature: type[Signature], demos: list[dict[str, Any]]) Returns: A list of multiturn messages. """ + return [message_to_openai_chat(message) for message in self._format_demos_as_lm_messages(signature, demos)] + + def _format_demos_as_lm_messages(self, signature: type[Signature], demos: list[dict[str, Any]]) -> list[LMMessage]: complete_demos = [] incomplete_demos = [] @@ -563,45 +742,94 @@ def format_demos(self, signature: type[Signature], demos: list[dict[str, Any]]) # We only keep incomplete demos that have at least one input and one output field incomplete_demos.append(demo) - messages = [] + messages: list[LMMessage] = [] incomplete_demo_prefix = "This is an example of the task, though some input or output fields are not supplied." for demo in incomplete_demos: - messages.append( - { - "role": "user", - "content": self.format_user_message_content(signature, demo, prefix=incomplete_demo_prefix), - } + messages.extend( + self._format_input_messages( + signature, + demo, + main_request=False, + prefix=incomplete_demo_prefix, + ) ) - messages.append( - { - "role": "assistant", - "content": self.format_assistant_message_content( - signature, demo, missing_field_message="Not supplied for this particular example. " - ), - } + messages.extend( + self._format_output_messages( + signature, + demo, + missing_field_message="Not supplied for this particular example. ", + ) ) for demo in complete_demos: - messages.append({"role": "user", "content": self.format_user_message_content(signature, demo)}) - messages.append( - { - "role": "assistant", - "content": self.format_assistant_message_content( - signature, demo, missing_field_message="Not supplied for this conversation history message. " - ), - } + messages.extend(self._format_input_messages(signature, demo, main_request=False)) + messages.extend( + self._format_output_messages( + signature, + demo, + missing_field_message="Not supplied for this conversation history message. ", + ) ) return messages - def _get_history_field_name(self, signature: type[Signature]) -> bool: + def _format_input_messages( + self, + signature: type[Signature], + inputs: dict[str, Any], + *, + main_request: bool, + prefix: str = "", + suffix: str = "", + ) -> list[LMMessage]: + regular_inputs = self._drop_absent_optional_inputs(signature, dict(inputs)) + content = self.format_user_message_content( + signature, + regular_inputs, + prefix=prefix, + suffix=suffix, + main_request=main_request, + ) + return [LMMessage.model_validate({"role": "user", "content": content})] if self._has_content(content) else [] + + def _format_output_messages( + self, + signature: type[Signature], + outputs: dict[str, Any], + *, + missing_field_message: str | None, + ) -> list[LMMessage]: + content = self.format_assistant_message_content( + signature, + outputs, + missing_field_message=missing_field_message, + ) + return ( + [LMMessage.model_validate({"role": "assistant", "content": content})] if self._has_content(content) else [] + ) + + @staticmethod + def _drop_absent_optional_inputs(signature: type[Signature], inputs: dict[str, Any]) -> dict[str, Any]: + return { + key: value + for key, value in inputs.items() + if not (value is None and key in signature.input_fields and signature.input_fields[key].default is None) + } + + @staticmethod + def _has_content(content: Any) -> bool: + if isinstance(content, str): + return bool(content.strip()) + return bool(content) + + def _get_history_field_name(self, signature: type[Signature]) -> str | None: for name, field in signature.input_fields.items(): if field.annotation == History: return name return None - def _get_tool_call_input_field_name(self, signature: type[Signature]) -> bool: + def _get_tool_call_input_field_name(self, signature: type[Signature]) -> str | None: for name, field in signature.input_fields.items(): # Look for annotation `list[dspy.Tool]` or `dspy.Tool` origin = get_origin(field.annotation) @@ -611,12 +839,140 @@ def _get_tool_call_input_field_name(self, signature: type[Signature]) -> bool: return name return None - def _get_tool_call_output_field_name(self, signature: type[Signature]) -> bool: + def _get_tool_call_output_field_name(self, signature: type[Signature]) -> str | None: for name, field in signature.output_fields.items(): - if field.annotation == ToolCalls: + if self._annotation_includes(field.annotation, ToolCalls): return name return None + @classmethod + def _annotation_includes(cls, annotation: Any, target: type) -> bool: + if annotation is target: + return True + return any(cls._annotation_includes(arg, target) for arg in get_args(annotation)) + + def force_tool_call_config(self, tool_name: str) -> dict[str, Any]: + if not self.use_native_function_calling: + return {} + return {"tool_choice": {"mode": "required", "allowed": [tool_name]}} + + def format_history( + self, + history: History, + signature: type[Signature], + *, + use_native_tool_calls: bool = False, + ) -> list[LMMessage]: + history = history if isinstance(history, History) else History.model_validate(history) + messages: list[LMMessage] = [] + + for entry in history.messages: + if not isinstance(entry, dict): + continue + + input_values = {key: value for key, value in entry.items() if key in signature.input_fields} + output_values = {key: value for key, value in entry.items() if key in signature.output_fields} + known_keys = set(input_values) | set(output_values) + unknown_values = {key: value for key, value in entry.items() if key not in known_keys} + + native_tool_results: list[LMMessage] = [] + if use_native_tool_calls: + regular_inputs = {} + for key, value in input_values.items(): + tool_results = self._coerce_tool_call_results_value(value, signature.input_fields.get(key)) + if tool_results is None: + regular_inputs[key] = value + else: + native_tool_results.extend(tool_results.to_lm_messages()) + if "tool_call_results" in unknown_values: + tool_results = self._coerce_tool_call_results_history_value(unknown_values.pop("tool_call_results")) + if tool_results is not None: + native_tool_results.extend(tool_results.to_lm_messages()) + else: + regular_inputs = input_values + + if regular_inputs: + messages.extend(self._format_input_messages(signature, regular_inputs, main_request=False)) + + native_tool_call_parts: list[LMPart] = [] + regular_outputs = {} + for key, value in output_values.items(): + tool_calls = None + if use_native_tool_calls: + tool_calls = self._coerce_tool_calls_value(value, signature.output_fields.get(key)) + if tool_calls is not None: + native_tool_call_parts.extend(tool_calls.to_lm_parts()) + else: + regular_outputs[key] = value + + if use_native_tool_calls and "tool_calls" in unknown_values: + tool_calls = self._coerce_tool_calls_history_value(unknown_values.pop("tool_calls")) + if tool_calls is not None: + native_tool_call_parts.extend(tool_calls.to_lm_parts()) + + assistant_text = self._format_history_assistant_text(signature, regular_outputs, unknown_values) + if use_native_tool_calls and native_tool_call_parts: + parts: list[LMPart] = [] + if assistant_text: + parts.append(LMTextPart(text=assistant_text)) + parts.extend(native_tool_call_parts) + messages.append(LMMessage(role="assistant", parts=parts)) + elif assistant_text: + messages.append(LMMessage.model_validate({"role": "assistant", "content": assistant_text})) + + messages.extend(native_tool_results) + + return messages + + @staticmethod + def _coerce_tool_call_results_history_value(field_value: Any) -> ToolCallResults | None: + try: + return ToolCallResults.model_validate(field_value) + except Exception: + return None + + @staticmethod + def _coerce_tool_calls_history_value(field_value: Any) -> ToolCalls | None: + try: + return ToolCalls.model_validate(field_value) + except Exception: + return None + + def _format_history_assistant_text( + self, + signature: type[Signature], + outputs: dict[str, Any], + unknown_outputs: dict[str, Any], + ) -> str | None: + text_signature = signature + for field_name, field_info in signature.output_fields.items(): + if self._annotation_includes(field_info.annotation, ToolCalls): + text_signature = text_signature.delete(field_name) + + sections = [] + signature_outputs = {key: value for key, value in outputs.items() if key in text_signature.output_fields} + if signature_outputs: + sections.append( + self.format_assistant_message_content( + text_signature, + signature_outputs, + missing_field_message="Not supplied for this conversation history message. ", + ) + ) + + for key, value in unknown_outputs.items(): + formatted_value = "\n".join(str(item) for item in value) if isinstance(value, list) else str(value) + sections.append(f"[[ ## {key} ## ]]\n{formatted_value}") + + if unknown_outputs and not any(section.endswith("[[ ## completed ## ]]") for section in sections): + sections.append("[[ ## completed ## ]]") + + if signature_outputs and not unknown_outputs and len(sections) == 1: + content = sections[0] + else: + content = "\n\n".join(section.strip() for section in sections if section).strip() + return content or None + def format_conversation_history( self, signature: type[Signature], @@ -640,25 +996,10 @@ def format_conversation_history( if conversation_history is None: return [] - messages = [] - for message in conversation_history: - messages.append( - { - "role": "user", - "content": self.format_user_message_content(signature, message), - } - ) - messages.append( - { - "role": "assistant", - "content": self.format_assistant_message_content(signature, message), - } - ) - # Remove the history field from the inputs del inputs[history_field_name] - - return messages + history = History(messages=conversation_history) + return [message_to_openai_chat(message) for message in self.format_history(history, signature)] def parse(self, signature: type[Signature], completion: str) -> dict[str, Any]: """Parse the LM output into a dictionary of the output fields. diff --git a/dspy/adapters/chat_adapter.py b/dspy/adapters/chat_adapter.py index 016c885d63..c4344a4ba9 100644 --- a/dspy/adapters/chat_adapter.py +++ b/dspy/adapters/chat_adapter.py @@ -70,19 +70,23 @@ def __call__( ) -> list[dict[str, Any]]: try: return super().__call__(lm, lm_kwargs, signature, demos, inputs) - except Exception as e: + except Exception as err: # fallback to JSONAdapter from dspy.adapters.json_adapter import JSONAdapter if ( - isinstance(e, ContextWindowExceededError) + isinstance(err, ContextWindowExceededError) or isinstance(self, JSONAdapter) or not self.use_json_adapter_fallback ): # On context window exceeded error, already using JSONAdapter, or use_json_adapter_fallback is False # we don't want to retry with a different adapter. Raise the original error instead of the fallback error. - raise e - return JSONAdapter()(lm, lm_kwargs, signature, demos, inputs) + raise + return JSONAdapter( + callbacks=self.callbacks, + use_native_function_calling=self.use_native_function_calling, + native_response_types=self.native_response_types, + )(lm, lm_kwargs, signature, demos, inputs) async def acall( self, @@ -94,25 +98,29 @@ async def acall( ) -> list[dict[str, Any]]: try: return await super().acall(lm, lm_kwargs, signature, demos, inputs) - except Exception as e: + except Exception as err: # fallback to JSONAdapter from dspy.adapters.json_adapter import JSONAdapter if ( - isinstance(e, ContextWindowExceededError) + isinstance(err, ContextWindowExceededError) or isinstance(self, JSONAdapter) or not self.use_json_adapter_fallback ): # On context window exceeded error, already using JSONAdapter, or use_json_adapter_fallback is False # we don't want to retry with a different adapter. Raise the original error instead of the fallback error. - raise e - return await JSONAdapter().acall(lm, lm_kwargs, signature, demos, inputs) + raise + return await JSONAdapter( + callbacks=self.callbacks, + use_native_function_calling=self.use_native_function_calling, + native_response_types=self.native_response_types, + ).acall(lm, lm_kwargs, signature, demos, inputs) def format_field_description(self, signature: type[Signature]) -> str: - return ( - f"Your input fields are:\n{get_field_description_string(signature.input_fields)}\n" - f"Your output fields are:\n{get_field_description_string(signature.output_fields)}" - ) + description = f"Your input fields are:\n{get_field_description_string(signature.input_fields)}" + if signature.output_fields: + description += f"\nYour output fields are:\n{get_field_description_string(signature.output_fields)}" + return description def format_field_structure(self, signature: type[Signature]) -> str: """ @@ -132,8 +140,9 @@ def format_signature_fields_for_instructions(fields: dict[str, FieldInfo]): ) parts.append(format_signature_fields_for_instructions(signature.input_fields)) - parts.append(format_signature_fields_for_instructions(signature.output_fields)) - parts.append("[[ ## completed ## ]]\n") + if signature.output_fields: + parts.append(format_signature_fields_for_instructions(signature.output_fields)) + parts.append("[[ ## completed ## ]]\n") return "\n\n".join(parts).strip() def format_task_description(self, signature: type[Signature]) -> str: @@ -164,7 +173,7 @@ def format_user_message_content( messages.append(suffix) return "\n\n".join(messages).strip() - def user_message_output_requirements(self, signature: type[Signature]) -> str: + def user_message_output_requirements(self, signature: type[Signature]) -> str | None: """Returns a simplified format reminder for the language model. In chat-based interactions, language models may lose track of the required output format @@ -182,6 +191,9 @@ def user_message_output_requirements(self, signature: type[Signature]) -> str: for inline reminders within chat messages. """ + if not signature.output_fields: + return None + def type_info(v): if v.annotation is not str: return f" (must be formatted as a valid Python {get_annotation_name(v.annotation)})" diff --git a/dspy/adapters/json_adapter.py b/dspy/adapters/json_adapter.py index 59a3b6f563..4167a00b23 100644 --- a/dspy/adapters/json_adapter.py +++ b/dspy/adapters/json_adapter.py @@ -38,9 +38,18 @@ def _has_open_ended_mapping(signature: SignatureMeta) -> bool: class JSONAdapter(ChatAdapter): - def __init__(self, callbacks: list[BaseCallback] | None = None, use_native_function_calling: bool = True): + def __init__( + self, + callbacks: list[BaseCallback] | None = None, + use_native_function_calling: bool = True, + native_response_types: list[type[type]] | None = None, + ): # JSONAdapter uses native function calling by default. - super().__init__(callbacks=callbacks, use_native_function_calling=use_native_function_calling) + super().__init__( + callbacks=callbacks, + use_native_function_calling=use_native_function_calling, + native_response_types=native_response_types, + ) def _json_adapter_call_common(self, lm, lm_kwargs, signature, demos, inputs, call_fn): """Common call logic to be used for both sync and async calls.""" @@ -49,7 +58,11 @@ def _json_adapter_call_common(self, lm, lm_kwargs, signature, demos, inputs, cal has_tool_calls = any(field.annotation == ToolCalls for field in signature.output_fields.values()) - if _has_open_ended_mapping(signature) or (not self.use_native_function_calling and has_tool_calls) or not lm.supports_response_schema: + if ( + _has_open_ended_mapping(signature) + or (not self.use_native_function_calling and has_tool_calls) + or not lm.supports_response_schema + ): # We found that structured output mode doesn't work well with dspy.ToolCalls as output field. # So we fall back to json mode if native function calling is disabled and ToolCalls is present. lm_kwargs["response_format"] = {"type": "json_object"} @@ -68,10 +81,6 @@ def __call__( return result try: - structured_output_model = _get_structured_outputs_response_format( - signature, self.use_native_function_calling - ) - lm_kwargs["response_format"] = structured_output_model return super().__call__(lm, lm_kwargs, signature, demos, inputs) except Exception: logger.warning("Failed to use structured output format, falling back to JSON mode.") @@ -91,16 +100,37 @@ async def acall( return await result try: - structured_output_model = _get_structured_outputs_response_format( - signature, self.use_native_function_calling - ) - lm_kwargs["response_format"] = structured_output_model return await super().acall(lm, lm_kwargs, signature, demos, inputs) except Exception: logger.warning("Failed to use structured output format, falling back to JSON mode.") lm_kwargs["response_format"] = {"type": "json_object"} return await super().acall(lm, lm_kwargs, signature, demos, inputs) + def _prepare_request_kwargs(self, lm: BaseLM, state) -> dict[str, Any]: + request_kwargs = dict(state.lm_kwargs) + if "response_format" in request_kwargs or "response_format" not in lm.supported_params: + return request_kwargs + if not state.render_signature.output_fields: + return request_kwargs + + has_tool_calls = any( + self._annotation_includes(field.annotation, ToolCalls) + for field in state.source_signature.output_fields.values() + ) + if ( + _has_open_ended_mapping(state.render_signature) + or (not self.use_native_function_calling and has_tool_calls) + or not lm.supports_response_schema + ): + request_kwargs["response_format"] = {"type": "json_object"} + return request_kwargs + + request_kwargs["response_format"] = _get_structured_outputs_response_format( + state.render_signature, + self.use_native_function_calling, + ) + return request_kwargs + def format_field_structure(self, signature: type[Signature]) -> str: parts = [] parts.append("All interactions will be structured in the following way, with the appropriate values filled in.") @@ -116,11 +146,15 @@ def format_signature_fields_for_instructions(fields: dict[str, FieldInfo], role: parts.append("Inputs will have the following structure:") parts.append(format_signature_fields_for_instructions(signature.input_fields, role="user")) - parts.append("Outputs will be a JSON object with the following fields.") - parts.append(format_signature_fields_for_instructions(signature.output_fields, role="assistant")) + if signature.output_fields: + parts.append("Outputs will be a JSON object with the following fields.") + parts.append(format_signature_fields_for_instructions(signature.output_fields, role="assistant")) return "\n\n".join(parts).strip() - def user_message_output_requirements(self, signature: type[Signature]) -> str: + def user_message_output_requirements(self, signature: type[Signature]) -> str | None: + if not signature.output_fields: + return None + def type_info(v): return ( f" (must be formatted as a valid Python {get_annotation_name(v.annotation)})" diff --git a/dspy/adapters/two_step_adapter.py b/dspy/adapters/two_step_adapter.py index 0e427dea25..cb13b64f40 100644 --- a/dspy/adapters/two_step_adapter.py +++ b/dspy/adapters/two_step_adapter.py @@ -180,6 +180,7 @@ def format_user_message_content( inputs: dict[str, Any], prefix: str = "", suffix: str = "", + main_request: bool = False, ) -> str: parts = [prefix] diff --git a/dspy/adapters/types/__init__.py b/dspy/adapters/types/__init__.py index 5ec8043021..36403e74d8 100644 --- a/dspy/adapters/types/__init__.py +++ b/dspy/adapters/types/__init__.py @@ -5,6 +5,6 @@ from dspy.adapters.types.history import History from dspy.adapters.types.image import Image from dspy.adapters.types.reasoning import Reasoning -from dspy.adapters.types.tool import Tool, ToolCalls +from dspy.adapters.types.tool import Tool, ToolCallResults, ToolCalls -__all__ = ["History", "Image", "Audio", "File", "Type", "Tool", "ToolCalls", "Code", "Reasoning"] +__all__ = ["History", "Image", "Audio", "File", "Type", "Tool", "ToolCalls", "ToolCallResults", "Code", "Reasoning"] diff --git a/dspy/adapters/types/history.py b/dspy/adapters/types/history.py index 6dda4f9b7c..148ae4cdad 100644 --- a/dspy/adapters/types/history.py +++ b/dspy/adapters/types/history.py @@ -1,6 +1,10 @@ -from typing import Any +from typing import Any, Callable import pydantic +from pydantic import Field + +from dspy.adapters.types.tool import ToolCallResults, ToolCalls +from dspy.adapters.utils import serialize_for_json class History(pydantic.BaseModel): @@ -58,11 +62,51 @@ class MySignature(dspy.Signature): ``` """ - messages: list[dict[str, Any]] + messages: list[dict[str, Any]] = Field(default_factory=list) model_config = pydantic.ConfigDict( - frozen=True, str_strip_whitespace=True, validate_assignment=True, extra="forbid", ) + + def __init__(self, *args: Any, compact_fn: Callable[["History"], None] | None = None, **kwargs: Any): + super().__init__(*args, **kwargs) + object.__setattr__(self, "_compact_fn", compact_fn) + + @pydantic.model_serializer() + def serialize_model(self) -> dict[str, Any]: + return {"messages": [self._serialize_message(message) for message in self.messages]} + + def compact_if_needed(self) -> None: + compact_fn = getattr(self, "_compact_fn", None) + if compact_fn is not None: + compact_fn(self) + + def append(self, message: dict[str, Any]) -> dict[str, Any]: + message = dict(message) + self.messages.append(message) + return message + + @staticmethod + def _serialize_message(message: dict[str, Any]) -> dict[str, Any]: + serialized = {} + for key, value in message.items(): + if isinstance(value, ToolCalls): + serialized[key] = { + "tool_calls": [ + { + "name": tool_call.name, + "args": serialize_for_json(tool_call.args), + **({"id": tool_call.id} if tool_call.id is not None else {}), + } + for tool_call in value.tool_calls + ] + } + elif isinstance(value, ToolCallResults): + serialized[key] = value.format() + elif hasattr(value, "model_dump"): + serialized[key] = value.model_dump() + else: + serialized[key] = value + return serialized diff --git a/dspy/adapters/types/tool.py b/dspy/adapters/types/tool.py index e6deb9b7c2..1dc61e0510 100644 --- a/dspy/adapters/types/tool.py +++ b/dspy/adapters/types/tool.py @@ -1,12 +1,16 @@ import asyncio import inspect +import json from typing import TYPE_CHECKING, Any, Callable, get_origin, get_type_hints +import json_repair import pydantic from jsonschema import ValidationError, validate from pydantic import BaseModel, TypeAdapter, create_model from dspy.adapters.types.base_type import Type +from dspy.adapters.utils import serialize_for_json +from dspy.core.types import LMMessage, LMTextPart, LMToolCallPart, LMToolResultPart, LMToolSpec from dspy.dsp.utils.settings import settings from dspy.utils.callback import with_callbacks @@ -162,6 +166,14 @@ def format_as_litellm_function_call(self): }, } + def to_lm_tool_spec(self) -> LMToolSpec: + args = self.args or {} + return LMToolSpec( + name=self.name or "", + description=self.desc, + parameters={"type": "object", "properties": args, "required": list(args.keys())}, + ) + def _run_async_in_sync(self, coroutine): try: loop = asyncio.get_running_loop() @@ -263,6 +275,7 @@ class ToolCalls(Type): class ToolCall(Type): name: str args: dict[str, Any] + id: str | None = None def format(self): return { @@ -273,6 +286,20 @@ def format(self): }, } + @classmethod + def __get_pydantic_json_schema__(cls, core_schema, handler): + schema = handler(core_schema) + properties = schema.get("properties") + if isinstance(properties, dict): + properties.pop("id", None) + required = schema.get("required") + if isinstance(required, list): + schema["required"] = [field for field in required if field != "id"] + return schema + + def to_lm_part(self, tool_call_id: str | None = None) -> LMToolCallPart: + return LMToolCallPart(id=tool_call_id or self.id, name=self.name, args=self.args) + def execute(self, functions: dict[str, Any] | list[Tool] | None = None) -> Any: """Execute this individual tool call and return its result. @@ -310,7 +337,9 @@ def execute(self, functions: dict[str, Any] | list[Tool] | None = None) -> Any: break if func is None: - raise ValueError(f"Tool function '{self.name}' not found. Please pass the tool functions to the `execute` method.") + raise ValueError( + f"Tool function '{self.name}' not found. Please pass the tool functions to the `execute` method." + ) try: args = self.args or {} @@ -340,8 +369,7 @@ def from_dict_list(cls, tool_calls_dicts: list[dict[str, Any]]) -> "ToolCalls": tool_calls = ToolCalls.from_dict_list(tool_calls_dict) ``` """ - tool_calls = [cls.ToolCall(**item) for item in tool_calls_dicts] - return cls(tool_calls=tool_calls) + return cls.model_validate(tool_calls_dicts) @classmethod def description(cls) -> str: @@ -350,41 +378,182 @@ def description(cls) -> str: "Arguments must be provided in JSON format." ) - def format(self) -> list[dict[str, Any]]: - # The tool_call field is compatible with OpenAI's tool calls schema. + def format(self) -> dict[str, Any]: return { "tool_calls": [tool_call.format() for tool_call in self.tool_calls], } + @classmethod + def parse_lm_response(cls, response: str | dict[str, Any]) -> "ToolCalls | None": + if not isinstance(response, dict): + return None + tool_calls = response.get("tool_calls") + if not tool_calls: + return None + return cls.model_validate(tool_calls) + + def to_lm_parts(self, id_prefix: str | None = None) -> list[LMToolCallPart]: + return [ + tool_call.to_lm_part(f"{id_prefix}_{idx}" if id_prefix is not None and tool_call.id is None else None) + for idx, tool_call in enumerate(self.tool_calls) + ] + + def with_call_ids(self, id_prefix: str) -> "ToolCalls": + tool_calls = [ + tool_call if tool_call.id is not None else tool_call.model_copy(update={"id": f"{id_prefix}_{idx}"}) + for idx, tool_call in enumerate(self.tool_calls) + ] + return self.model_copy(update={"tool_calls": tool_calls}) + + @classmethod + def _canonical_tool_call(cls, item: Any) -> Any: + if isinstance(item, cls.ToolCall): + normalized = {"name": item.name, "args": item.args} + if item.id is not None: + normalized["id"] = item.id + return normalized + + if isinstance(item, dict): + if "name" in item and "args" in item: + normalized = {"name": item["name"], "args": item["args"]} + call_id = item.get("id") or item.get("call_id") + if call_id is not None: + normalized["id"] = call_id + return normalized + if "function" in item and isinstance(item["function"], dict): + function = item["function"] + arguments = function.get("arguments", {}) + normalized = { + "name": function.get("name"), + "args": json_repair.loads(arguments) if isinstance(arguments, str) else arguments, + } + call_id = item.get("id") or item.get("call_id") + if call_id is not None: + normalized["id"] = call_id + return normalized + return item + + if hasattr(item, "name") and hasattr(item, "args"): + normalized = {"name": item.name, "args": item.args} + tool_call_id = getattr(item, "id", None) or getattr(item, "call_id", None) + if tool_call_id is not None: + normalized["id"] = tool_call_id + return normalized + + return item + @pydantic.model_validator(mode="before") @classmethod def validate_input(cls, data: Any): if isinstance(data, cls): return data - # Handle case where data is a list of dicts with "name" and "args" keys - if isinstance(data, list) and all( - isinstance(item, dict) and "name" in item and "args" in item for item in data - ): - return {"tool_calls": [cls.ToolCall(**item) for item in data]} - # Handle case where data is a dict + tool_calls_data = None + if isinstance(data, list): + tool_calls_data = data elif isinstance(data, dict): if "tool_calls" in data: - # Handle case where data is a dict with "tool_calls" key tool_calls_data = data["tool_calls"] - if isinstance(tool_calls_data, list): - return { - "tool_calls": [ - cls.ToolCall(**item) if isinstance(item, dict) else item for item in tool_calls_data - ] - } elif "name" in data and "args" in data: - # Handle case where data is a dict with "name" and "args" keys - return {"tool_calls": [cls.ToolCall(**data)]} + return {"tool_calls": [cls._canonical_tool_call(data)]} + + if isinstance(tool_calls_data, list): + normalized = [cls._canonical_tool_call(item) for item in tool_calls_data] + if all(isinstance(item, dict) and "name" in item and "args" in item for item in normalized): + return {"tool_calls": normalized} raise ValueError(f"Received invalid value for `dspy.ToolCalls`: {data}") +class _ToolCallResult(BaseModel): + call_id: str | None = None + name: str | None = None + value: Any + is_error: bool = False + + def to_lm_message(self) -> LMMessage: + jsonable_value = serialize_for_json(self.value) + text = ( + json.dumps(jsonable_value, ensure_ascii=False) + if isinstance(jsonable_value, (dict, list)) + else str(jsonable_value) + ) + return LMMessage( + role="tool", + parts=[ + LMToolResultPart( + call_id=self.call_id, + name=self.name, + content=[LMTextPart(text=text)], + is_error=self.is_error, + ) + ], + ) + + def format(self) -> dict[str, Any]: + formatted = {"value": serialize_for_json(self.value), "is_error": self.is_error} + if self.call_id is not None: + formatted["call_id"] = self.call_id + if self.name is not None: + formatted["name"] = self.name + return formatted + + +class ToolCallResults(Type): + tool_call_results: list[_ToolCallResult] + + @classmethod + def from_dict_list(cls, tool_call_result_dicts: list[dict[str, Any]]) -> "ToolCallResults": + return cls.model_validate(tool_call_result_dicts) + + @classmethod + def from_tool_calls_and_values( + cls, + tool_calls: list[ToolCalls.ToolCall], + values: list[Any], + is_errors: list[bool] | None = None, + ) -> "ToolCallResults": + if is_errors is None: + is_errors = [False] * len(values) + return cls( + tool_call_results=[ + _ToolCallResult( + call_id=tool_call.id, + name=tool_call.name, + value=value, + is_error=is_error, + ) + for tool_call, value, is_error in zip(tool_calls, values, is_errors, strict=True) + ] + ) + + @classmethod + def description(cls) -> str: + return ( + "Results returned by previous tool calls, including the tool call id, tool name, returned value, " + "and whether the tool execution failed." + ) + + def format(self) -> dict[str, Any]: + return {"tool_call_results": [result.format() for result in self.tool_call_results]} + + def to_lm_messages(self) -> list[LMMessage]: + return [result.to_lm_message() for result in self.tool_call_results] + + @pydantic.model_validator(mode="before") + @classmethod + def validate_input(cls, data: Any): + if isinstance(data, cls): + return data + if isinstance(data, list): + return {"tool_call_results": data} + if isinstance(data, dict) and "tool_call_results" in data: + return data + if isinstance(data, dict) and "value" in data: + return {"tool_call_results": [data]} + raise ValueError(f"Received invalid value for `dspy.ToolCallResults`: {data}") + + def _resolve_json_schema_reference(schema: dict) -> dict: """Recursively resolve json model schema, expanding all references.""" diff --git a/dspy/adapters/xml_adapter.py b/dspy/adapters/xml_adapter.py index 662b1d2868..63437fb7a9 100644 --- a/dspy/adapters/xml_adapter.py +++ b/dspy/adapters/xml_adapter.py @@ -38,7 +38,8 @@ def format_signature_fields_for_instructions(fields: dict[str, FieldInfo]): ) parts.append(format_signature_fields_for_instructions(signature.input_fields)) - parts.append(format_signature_fields_for_instructions(signature.output_fields)) + if signature.output_fields: + parts.append(format_signature_fields_for_instructions(signature.output_fields)) return "\n\n".join(parts).strip() def format_user_message_content( @@ -51,12 +52,15 @@ def format_user_message_content( ) -> str: messages = [prefix] - messages.append(self.format_field_with_value( - { - FieldInfoWithName(name=k, info=v): inputs.get(k) - for k, v in signature.input_fields.items() if k in inputs - }, - )) + messages.append( + self.format_field_with_value( + { + FieldInfoWithName(name=k, info=v): inputs.get(k) + for k, v in signature.input_fields.items() + if k in inputs + }, + ) + ) if main_request: output_requirements = self.user_message_output_requirements(signature) @@ -79,7 +83,10 @@ def format_assistant_message_content( }, ) - def user_message_output_requirements(self, signature: type[Signature]) -> str: + def user_message_output_requirements(self, signature: type[Signature]) -> str | None: + if not signature.output_fields: + return None + message = "Respond with the corresponding output fields wrapped in XML tags " message += ", then ".join(f"`<{f}>`" for f in signature.output_fields) message += "." diff --git a/dspy/clients/openai_format.py b/dspy/clients/openai_format.py index 98d72d5a2d..4a83bee9b4 100644 --- a/dspy/clients/openai_format.py +++ b/dspy/clients/openai_format.py @@ -27,6 +27,7 @@ import os from typing import Any +import json_repair import pydantic from dspy.core.types import ( @@ -649,7 +650,7 @@ def provider_tool_call_to_part(tool_call: Any) -> LMToolCallPart: arguments = get_value(function, "arguments", get_value(tool_call, "arguments", "{}")) provider_data = model_dump(tool_call) try: - args = json.loads(arguments) if isinstance(arguments, str) else dict(arguments) + args = json_repair.loads(arguments) if isinstance(arguments, str) else dict(arguments) except Exception as error: args = {} provider_data["raw_arguments"] = arguments @@ -664,7 +665,7 @@ def responses_function_call_to_part(output_item: Any) -> LMToolCallPart: provider_data = model_dump(output_item) if isinstance(args, str): try: - args = json.loads(args) + args = json_repair.loads(args) except Exception as error: provider_data["raw_arguments"] = args provider_data["arguments_parse_error"] = str(error) diff --git a/dspy/predict/__init__.py b/dspy/predict/__init__.py index 906ef90ae9..e3ad562a6a 100644 --- a/dspy/predict/__init__.py +++ b/dspy/predict/__init__.py @@ -8,6 +8,7 @@ from dspy.predict.predict import Predict from dspy.predict.program_of_thought import ProgramOfThought from dspy.predict.react import ReAct, Tool +from dspy.predict.reactv2 import ReActV2 from dspy.predict.refine import Refine from dspy.predict.rlm import RLM @@ -21,6 +22,7 @@ "Predict", "ProgramOfThought", "ReAct", + "ReActV2", "Refine", "RLM", "Tool", diff --git a/dspy/predict/reactv2.py b/dspy/predict/reactv2.py new file mode 100644 index 0000000000..d3a21e4b72 --- /dev/null +++ b/dspy/predict/reactv2.py @@ -0,0 +1,280 @@ +import logging +import traceback +from copy import deepcopy +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Callable + +from pydantic import TypeAdapter + +import dspy +from dspy.adapters.types.history import History +from dspy.adapters.types.tool import Tool, ToolCallResults, ToolCalls +from dspy.primitives.module import Module +from dspy.signatures.signature import ensure_signature +from dspy.utils.exceptions import AdapterParseError, ContextWindowExceededError + +logger = logging.getLogger(__name__) + +if TYPE_CHECKING: + from dspy.signatures.signature import Signature + +_RECOVERABLE_FORCED_SUBMIT_ERRORS = (AdapterParseError, ContextWindowExceededError, ValueError) + + +@dataclass(frozen=True) +class ToolExecutionResult: + value: object + is_error: bool = False + + +def _build_submit_tool(signature: type["Signature"]) -> Tool: + outputs = ", ".join([f"`{key}`" for key in signature.output_fields]) + output_args = {} + output_arg_types = {} + for name, field in signature.output_fields.items(): + annotation = getattr(field, "annotation", str) + output_args[name] = _json_schema_for_annotation(annotation) + output_arg_types[name] = annotation + + return Tool( + func=lambda **kwargs: kwargs, + name="submit", + desc=f"Call this tool to end the task and return your final answer. Takes: {outputs}.", + args=output_args, + arg_types=output_arg_types, + ) + + +def _json_schema_for_annotation(annotation: Any) -> dict[str, Any]: + try: + schema = TypeAdapter(annotation).json_schema() + except Exception: + return {"type": "string"} + return _inline_json_schema_refs(schema) + + +def _inline_json_schema_refs(schema: dict[str, Any]) -> dict[str, Any]: + definitions = schema.get("$defs") or schema.get("definitions") + if not definitions: + return schema + + def resolve(value: Any) -> Any: + if isinstance(value, list): + return [resolve(item) for item in value] + if not isinstance(value, dict): + return value + + ref = value.get("$ref") + if isinstance(ref, str) and ref.startswith(("#/$defs/", "#/definitions/")): + ref_name = ref.rsplit("/", 1)[-1] + return resolve(definitions[ref_name]) + + return {key: resolve(item) for key, item in value.items() if key not in {"$defs", "definitions"}} + + return resolve(schema) + + +class ReActV2(Module): + def __init__(self, signature: type["Signature"] | str, tools: list[Callable], max_iters: int = 20): + super().__init__() + self.signature = signature = ensure_signature(signature) + self.max_iters = max_iters + + tools = [tool if isinstance(tool, Tool) else Tool(tool) for tool in tools] + self.tools = {tool.name: tool for tool in tools} + self.tools["submit"] = _build_submit_tool(signature) + + optional_input_fields = deepcopy(signature.input_fields) + for field in optional_input_fields.values(): + field.default = None + + react_signature = ( + dspy.Signature(optional_input_fields, self._build_instructions()) + .append("history", dspy.InputField(), type_=dspy.History) + .append("tools", dspy.InputField(), type_=list[dspy.Tool]) + .append("tool_call_results", dspy.InputField(default=None), type_=dspy.ToolCallResults) + .append("next_thought", dspy.OutputField(), type_=dspy.Reasoning) + .append("tool_calls", dspy.OutputField(), type_=dspy.ToolCalls) + ) + extract_signature = dspy.Signature( + {**signature.input_fields, **signature.output_fields}, + signature.instructions, + ).append( + "trajectory", + dspy.InputField(desc="The agent's history of thoughts, actions, and tool call results"), + type_=str, + ) + self.react = dspy.Predict(react_signature) + self.extract = dspy.ChainOfThought(extract_signature) + + def _build_instructions(self) -> str: + inputs = ", ".join([f"`{key}`" for key in self.signature.input_fields]) + outputs = ", ".join([f"`{key}`" for key in self.signature.output_fields]) + instructions = [f"{self.signature.instructions}\n"] if self.signature.instructions else [] + instructions.extend( + [ + f"You are an Agent. Given {inputs}, use tools to produce {outputs}.", + "Each turn: think, then call one or more tools. After each tool call you receive a tool result.", + "When you have enough information to answer, call `submit` to finish.", + "\nAvailable tools:\n", + ] + ) + instructions.extend(f"({idx + 1}) {tool}" for idx, tool in enumerate(self.tools.values())) + return "\n".join(instructions) + + def forward(self, **input_args): + history_arg = input_args.pop("history", None) + if history_arg is None: + history = History() + elif isinstance(history_arg, History): + history = history_arg + else: + history = History.model_validate(history_arg) + max_iters = input_args.pop("max_iters", self.max_iters) + tool_list = list(self.tools.values()) + pending_inputs = dict(input_args) + + break_reason = None + for idx in range(max_iters): + try: + pred = self.react(history=history, tools=tool_list, **pending_inputs) + except ContextWindowExceededError: + history.compact_if_needed() + try: + pred = self.react(history=history, tools=tool_list, **pending_inputs) + except ContextWindowExceededError: + logger.warning("Context window exceeded after compaction, ending loop.") + break_reason = "context_overflow" + break + except (AdapterParseError, ValueError) as err: + logger.warning(f"Agent iteration {idx} failed: {_fmt_exc(err)}") + break_reason = "parse_error" + break + + if pred.tool_calls is None or not pred.tool_calls.tool_calls: + logger.warning("Agent returned no tool calls, ending loop.") + break_reason = "no_tool_calls" + break + + tool_calls = pred.tool_calls.with_call_ids(f"call_{len(history.messages)}") + tool_results = [self._execute_tool_call(tool_call) for tool_call in tool_calls.tool_calls] + self._append_tool_turn( + history, + inputs=pending_inputs, + next_thought=pred.next_thought, + tool_calls=tool_calls, + tool_results=tool_results, + ) + pending_inputs = {} + + for tool_call, tool_result in zip(tool_calls.tool_calls, tool_results, strict=True): + if tool_call.name == "submit" and not tool_result.is_error: + history.messages[-1].update(tool_result.value) + return dspy.Prediction(history=history, termination_reason="submit", **tool_result.value) + + return self._forced_submit(history, pending_inputs, break_reason=break_reason) + + def _forced_submit(self, history: History, input_args: dict[str, object], break_reason: str | None = None): + tool_list = list(self.tools.values()) + adapter = dspy.settings.adapter or dspy.ChatAdapter() + call_config = adapter.force_tool_call_config("submit") if hasattr(adapter, "force_tool_call_config") else {} + + try: + pred = self.react(history=history, tools=tool_list, config=call_config, **input_args) + except _RECOVERABLE_FORCED_SUBMIT_ERRORS as err: + logger.debug(f"Forced submit tier 1 (react) failed: {_fmt_exc(err)}") + pred = None + + if pred and pred.tool_calls and pred.tool_calls.tool_calls: + for tool_call in pred.tool_calls.tool_calls: + if tool_call.name != "submit": + continue + try: + result = self.tools["submit"](**tool_call.args) + except ValueError as err: + logger.debug(f"Forced submit tool execution failed: {_fmt_exc(err)}") + continue + + tool_calls = ToolCalls(tool_calls=[tool_call]).with_call_ids(f"call_{len(history.messages)}") + self._append_tool_turn( + history, + inputs=input_args, + next_thought=pred.next_thought, + tool_calls=tool_calls, + tool_results=[ToolExecutionResult(value=result, is_error=False)], + ) + try: + history.messages[-1].update(result) + return dspy.Prediction(history=history, termination_reason="forced_submit", **result) + except TypeError as err: + logger.debug(f"Forced submit result was not a valid output mapping: {_fmt_exc(err)}") + + try: + trajectory_text = self._render_history_as_text(history) + extract = self.extract(trajectory=trajectory_text, **input_args) + result = {key: getattr(extract, key) for key in self.signature.output_fields if hasattr(extract, key)} + if any(value is not None for value in result.values()): + history.append(result) + return dspy.Prediction(history=history, termination_reason="extract", **result) + except _RECOVERABLE_FORCED_SUBMIT_ERRORS as err: + logger.debug(f"Forced submit tier 2 (extract) failed: {_fmt_exc(err)}") + + return dspy.Prediction(history=history, termination_reason=break_reason or "failed") + + def _execute_tool_call(self, tool_call: ToolCalls.ToolCall) -> ToolExecutionResult: + tool = self.tools.get(tool_call.name) + if tool is None: + return ToolExecutionResult(value=f"Unknown tool: {tool_call.name}", is_error=True) + try: + return ToolExecutionResult(value=tool(**tool_call.args), is_error=False) + except Exception as err: + return ToolExecutionResult(value=f"Execution error in {tool_call.name}: {_fmt_exc(err)}", is_error=True) + + @staticmethod + def _append_tool_turn( + history: History, + *, + inputs: dict[str, object] | None = None, + next_thought, + tool_calls: ToolCalls, + tool_results: list[ToolExecutionResult], + ) -> None: + history.append( + { + **(inputs or {}), + "next_thought": next_thought, + "tool_calls": tool_calls, + "tool_call_results": ToolCallResults.from_tool_calls_and_values( + tool_calls.tool_calls, + [tool_result.value for tool_result in tool_results], + [tool_result.is_error for tool_result in tool_results], + ), + } + ) + + def _render_history_as_text(self, history: History) -> str: + lines = [] + for event in history.messages: + thought = event.get("next_thought") + tool_calls = event.get("tool_calls") + tool_call_results = event.get("tool_call_results") + if thought: + lines.append(f"[Thought] {thought}") + if isinstance(tool_calls, ToolCalls): + for tool_call in tool_calls.tool_calls: + args = ", ".join(f"{key}={value!r}" for key, value in (tool_call.args or {}).items()) + lines.append(f"[Action] {tool_call.name}({args})") + if isinstance(tool_call_results, ToolCallResults): + for result in tool_call_results.tool_call_results: + prefix = "[Error]" if result.is_error else "[Observation]" + lines.append(f"{prefix} {result.name}: {result.value}") + for key, value in event.items(): + if key in {"next_thought", "tool_calls", "tool_call_results"}: + continue + label = "[Input]" if key in self.signature.input_fields else "[Output]" + lines.append(f"{label} {key}: {value}") + return "\n".join(lines) + + +def _fmt_exc(err: BaseException, *, limit: int = 5) -> str: + return "\n" + "".join(traceback.format_exception(type(err), err, err.__traceback__, limit=limit)).strip() diff --git a/tests/adapters/test_adapter_base.py b/tests/adapters/test_adapter_base.py new file mode 100644 index 0000000000..33793afdac --- /dev/null +++ b/tests/adapters/test_adapter_base.py @@ -0,0 +1,348 @@ +from typing import ClassVar + +import dspy +from dspy.core.types import LMOutput, LMResponse, LMTextPart +from dspy.utils.callback import BaseCallback + + +def search(query: str) -> str: + return query + + +def add(a: int, b: int) -> int: + return a + b + + +class NativeToolLM: + model = "openai/gpt-5-nano" + model_type = "chat" + kwargs: ClassVar[dict] = {} + supported_params = frozenset() + supports_function_calling = True + supports_reasoning = False + supports_response_schema = False + + def __init__(self, output=None): + self.output = output + self.messages = None + self.kwargs = None + + def __call__(self, messages, **kwargs): + self.messages = messages + self.kwargs = kwargs + return [self.output] + + +def test_prepare_request_state_copies_kwargs_and_extracts_tools(): + class ToolSignature(dspy.Signature): + question: str = dspy.InputField() + tools: list[dspy.Tool] = dspy.InputField() + tool_calls: dspy.ToolCalls = dspy.OutputField() + + adapter = dspy.Adapter(use_native_function_calling=True) + lm_kwargs = {"temperature": 0.2} + inputs = {"question": "Q?", "tools": [dspy.Tool(search)]} + + state = adapter._prepare_request_state(NativeToolLM(), lm_kwargs, ToolSignature, inputs) + + assert lm_kwargs == {"temperature": 0.2} + assert inputs["tools"][0].name == "search" + assert "tools" not in state.lm_kwargs + assert state.tools[0].name == "search" + assert "tools" not in state.render_signature.input_fields + assert "tool_calls" not in state.render_signature.output_fields + assert state.hidden_output_fields == ("tool_calls",) + + +def test_public_format_omits_none_native_tool_results_field(): + class ToolSignature(dspy.Signature): + question: str = dspy.InputField() + tools: list[dspy.Tool] = dspy.InputField() + tool_call_results: dspy.ToolCallResults = dspy.InputField(default=None) + tool_calls: dspy.ToolCalls = dspy.OutputField() + + adapter = dspy.ChatAdapter(use_native_function_calling=True) + inputs = {"question": "What is 1+2?", "tools": [dspy.Tool(add)], "tool_call_results": None} + + state = adapter._prepare_request_state(NativeToolLM(), {}, ToolSignature, inputs) + messages = adapter.format(state.render_signature, [], state.inputs) + + assert all(message["role"] != "tool" for message in messages) + assert "[[ ## tool_call_results ## ]]" not in messages[-1]["content"] + assert "None" not in messages[-1]["content"] + + +def test_prepare_request_state_preserves_normal_signature_and_copies_data(): + class QASignature(dspy.Signature): + question: str = dspy.InputField() + answer: str = dspy.OutputField() + + adapter = dspy.Adapter() + lm_kwargs = {"temperature": 0.2, "metadata": {"trace_id": "abc"}} + inputs = {"question": "Q?"} + + state = adapter._prepare_request_state(NativeToolLM(), lm_kwargs, QASignature, inputs) + + assert state.source_signature is QASignature + assert state.render_signature is QASignature + assert state.inputs == inputs + assert state.inputs is not inputs + assert state.lm_kwargs == lm_kwargs + assert state.lm_kwargs is not lm_kwargs + assert state.tools == [] + assert state.hidden_output_fields == () + assert lm_kwargs == {"temperature": 0.2, "metadata": {"trace_id": "abc"}} + assert inputs == {"question": "Q?"} + + +def test_render_request_normal_state_preserves_messages_and_kwargs(): + class QASignature(dspy.Signature): + question: str = dspy.InputField() + answer: str = dspy.OutputField() + + adapter = dspy.ChatAdapter() + lm = NativeToolLM() + state = adapter._prepare_request_state( + lm, + {"temperature": 0.7, "n": 2, "custom_option": "value"}, + QASignature, + {"question": "Q2?"}, + ) + + messages = adapter.format(state.render_signature, [{"question": "Q1?", "answer": "A1"}], state.inputs) + request = adapter._render_request(lm, state, messages) + + assert request.model == lm.model + assert request.tools == [] + assert request.config.temperature == 0.7 + assert request.config.n == 2 + assert request.config.extensions == {"custom_option": "value"} + assert [message.role for message in request.messages] == ["system", "user", "assistant", "user"] + assert "Q1?" in request.messages[1].text + assert "A1" in request.messages[2].text + assert "Q2?" in request.messages[3].text + + +def test_render_request_normal_state_expands_history_without_mutating_inputs(): + class HistorySignature(dspy.Signature): + history: dspy.History = dspy.InputField() + question: str = dspy.InputField() + answer: str = dspy.OutputField() + + adapter = dspy.ChatAdapter() + lm = NativeToolLM() + history = dspy.History(messages=[{"question": "What is the capital of France?", "answer": "Paris"}]) + inputs = {"history": history, "question": "What country is it in?"} + + state = adapter._prepare_request_state(lm, {}, HistorySignature, inputs) + messages = adapter.format(state.render_signature, [], state.inputs) + request = adapter._render_request(lm, state, messages) + + assert inputs == {"history": history, "question": "What country is it in?"} + assert [message.role for message in request.messages] == ["system", "user", "assistant", "user"] + assert "What is the capital of France?" in request.messages[1].text + assert "Paris" in request.messages[2].text + assert "What country is it in?" in request.messages[3].text + assert "[[ ## history ## ]]" not in request.messages[3].text + assert "Paris" not in request.messages[3].text + + +def test_adapter_format_callbacks_receive_public_arguments(): + class QASignature(dspy.Signature): + question: str = dspy.InputField() + answer: str = dspy.OutputField() + + class CaptureFormatCallback(BaseCallback): + def __init__(self): + self.inputs = None + + def on_adapter_format_start(self, call_id, instance, inputs): + self.inputs = inputs + + callback = CaptureFormatCallback() + lm = NativeToolLM("[[ ## answer ## ]]\nA\n\n[[ ## completed ## ]]") + + dspy.ChatAdapter(callbacks=[callback], use_json_adapter_fallback=False)( + lm, + {}, + QASignature, + [{"question": "demo question", "answer": "demo answer"}], + {"question": "Q?"}, + ) + + assert set(callback.inputs) == {"signature", "demos", "inputs"} + assert callback.inputs["signature"] is QASignature + assert callback.inputs["demos"] == [{"question": "demo question", "answer": "demo answer"}] + assert callback.inputs["inputs"] == {"question": "Q?"} + + +def test_call_path_uses_public_format_override(): + class QASignature(dspy.Signature): + question: str = dspy.InputField() + answer: str = dspy.OutputField() + + class FormatOverrideAdapter(dspy.Adapter): + def format(self, signature, demos, inputs): + return [ + {"role": "system", "content": "custom system"}, + {"role": "user", "content": f"custom user: {inputs['question']}"}, + ] + + def parse(self, signature, completion): + return {"answer": completion} + + lm = NativeToolLM("custom answer") + + result = FormatOverrideAdapter()( + lm, + {}, + QASignature, + [{"question": "demo question", "answer": "demo answer"}], + {"question": "Q?"}, + ) + + assert result == [{"answer": "custom answer"}] + assert lm.messages == [ + {"role": "system", "content": "custom system"}, + {"role": "user", "content": "custom user: Q?"}, + ] + + +def test_call_path_uses_public_format_demos_override(): + class QASignature(dspy.Signature): + question: str = dspy.InputField() + answer: str = dspy.OutputField() + + class DemoOverrideAdapter(dspy.ChatAdapter): + def format_demos(self, signature, demos): + return [{"role": "user", "content": "custom demo message"}] + + lm = NativeToolLM("[[ ## answer ## ]]\nA\n\n[[ ## completed ## ]]") + + result = DemoOverrideAdapter(use_json_adapter_fallback=False)( + lm, + {}, + QASignature, + [{"question": "demo question", "answer": "demo answer"}], + {"question": "Q?"}, + ) + + assert result == [{"answer": "A"}] + assert any(message["content"] == "custom demo message" for message in lm.messages) + assert not any("demo question" in message["content"] for message in lm.messages) + + +def test_parse_response_normal_state_parses_text_output_and_logprobs(): + class QASignature(dspy.Signature): + question: str = dspy.InputField() + answer: str = dspy.OutputField() + + adapter = dspy.ChatAdapter() + state = adapter._prepare_request_state(NativeToolLM(), {}, QASignature, {"question": "Q?"}) + response = LMResponse( + model="dummy", + outputs=[ + LMOutput( + parts=[LMTextPart(text="[[ ## answer ## ]]\nA\n\n[[ ## completed ## ]]")], + logprobs={"tokens": ["A"]}, + ) + ], + ) + + assert adapter._parse_response(state, response) == [{"answer": "A", "logprobs": {"tokens": ["A"]}}] + + +def test_native_tool_response_can_combine_visible_text_and_tool_calls(): + class ToolSignature(dspy.Signature): + question: str = dspy.InputField() + tools: list[dspy.Tool] = dspy.InputField() + answer: str = dspy.OutputField() + tool_calls: dspy.ToolCalls = dspy.OutputField() + + lm = NativeToolLM( + { + "text": "[[ ## answer ## ]]\nworking\n\n[[ ## completed ## ]]", + "tool_calls": [ + { + "id": "call_add", + "type": "function", + "function": {"name": "add", "arguments": '{"a": 1, "b": 2}'}, + } + ], + } + ) + + result = dspy.ChatAdapter(use_native_function_calling=True)( + lm, + {}, + ToolSignature, + [], + {"question": "What is 1+2?", "tools": [dspy.Tool(add)]}, + )[0] + + assert result["answer"] == "working" + assert result["tool_calls"].tool_calls[0].name == "add" + assert result["tool_calls"].tool_calls[0].args == {"a": 1, "b": 2} + assert "tools" in lm.kwargs + assert lm.kwargs["tools"][0]["function"]["name"] == "add" + + +def test_native_tool_response_repairs_nonstandard_json_arguments(): + class ToolSignature(dspy.Signature): + question: str = dspy.InputField() + tools: list[dspy.Tool] = dspy.InputField() + tool_calls: dspy.ToolCalls = dspy.OutputField() + + lm = NativeToolLM( + { + "tool_calls": [ + { + "id": "call_add", + "type": "function", + "function": {"name": "add", "arguments": "{'a': 1, 'b': 2}"}, + } + ], + } + ) + + result = dspy.ChatAdapter(use_native_function_calling=True)( + lm, + {}, + ToolSignature, + [], + {"question": "What is 1+2?", "tools": [dspy.Tool(add)]}, + )[0] + + assert result["tool_calls"].tool_calls[0].args == {"a": 1, "b": 2} + + +def test_native_tool_result_input_replays_as_tool_message(): + class ToolResultSignature(dspy.Signature): + question: str = dspy.InputField() + tools: list[dspy.Tool] = dspy.InputField() + tool_call_results: dspy.ToolCallResults = dspy.InputField() + tool_calls: dspy.ToolCalls = dspy.OutputField() + + result = dspy.ToolCallResults.from_dict_list([{"call_id": "call_1", "name": "add", "value": "3"}]) + lm = NativeToolLM( + { + "tool_calls": [ + { + "id": "call_submit", + "type": "function", + "function": {"name": "submit", "arguments": '{"answer": "3"}'}, + } + ], + } + ) + + dspy.ChatAdapter(use_native_function_calling=True)( + lm, + {}, + ToolResultSignature, + [], + {"question": "What is 1+2?", "tools": [dspy.Tool(add)], "tool_call_results": result}, + ) + + assert any(message["role"] == "tool" and message["tool_call_id"] == "call_1" for message in lm.messages) + assert "tool_call_results" not in lm.messages[-1]["content"] diff --git a/tests/adapters/test_chat_adapter.py b/tests/adapters/test_chat_adapter.py index 17c1603766..9c7952ff07 100644 --- a/tests/adapters/test_chat_adapter.py +++ b/tests/adapters/test_chat_adapter.py @@ -1080,25 +1080,16 @@ class NativeToolSignature(dspy.Signature): expected_messages = [{"role": "system", "content": "Your input fields are:\n" "1. `question` (str):\n" - "Your output fields are:\n" - "\n" "All interactions will be structured in the following way, with the appropriate " "values filled in.\n" "\n" "[[ ## question ## ]]\n" "{question}\n" - "\n" - "\n" - "\n" - "[[ ## completed ## ]]\n" "In adhering to this structure, your objective is: \n" " Given the fields `question`, `tools`, produce the fields `tool_calls`."}, {"role": "user", "content": "[[ ## question ## ]]\n" - "Q?\n" - "\n" - "Respond with the corresponding output fields, starting with the field , and then " - "ending with the marker for `[[ ## completed ## ]]`."}] + "Q?"}] assert messages == expected_messages expected_lm_kwargs = {"tools": [{"type": "function", "function": {"name": "search", @@ -1109,6 +1100,87 @@ class NativeToolSignature(dspy.Signature): "required": ["query", "k"]}}}]} assert lm_kwargs == expected_lm_kwargs + +def test_chat_adapter_format_exact_messages_with_native_tool_call_history_result(): + class FunctionCallingLM(dspy.utils.DummyLM): + @property + def supports_function_calling(self): + return True + + def add(a: int, b: int) -> int: + """Add two numbers.""" + return a + b + + class ToolHistorySignature(dspy.Signature): + question: str = dspy.InputField() + history: dspy.History = dspy.InputField() + tools: list[dspy.Tool] = dspy.InputField() + tool_call_results: dspy.ToolCallResults = dspy.InputField(default=None) + answer: str = dspy.OutputField() + tool_calls: dspy.ToolCalls = dspy.OutputField() + + history = dspy.History( + messages=[ + { + "question": "What is 1+2?", + "tool_calls": [{"id": "call_add", "name": "add", "args": {"a": 1, "b": 2}}], + "tool_call_results": [{"call_id": "call_add", "name": "add", "value": {"sum": 3}}], + } + ] + ) + + messages, lm_kwargs = format_messages_and_lm_kwargs( + dspy.ChatAdapter(use_native_function_calling=True), + ToolHistorySignature, + [], + {"question": "Use the prior tool result.", "history": history, "tools": [dspy.Tool(add)]}, + lm=FunctionCallingLM([{}]), + ) + + expected_messages = [{"role": "system", + "content": "Your input fields are:\n" + "1. `question` (str): \n" + "2. `history` (History):\n" + "Your output fields are:\n" + "1. `answer` (str):\n" + "All interactions will be structured in the following way, with the appropriate " + "values filled in.\n" + "\n" + "[[ ## question ## ]]\n" + "{question}\n" + "\n" + "[[ ## history ## ]]\n" + "{history}\n" + "\n" + "[[ ## answer ## ]]\n" + "{answer}\n" + "\n" + "[[ ## completed ## ]]\n" + "In adhering to this structure, your objective is: \n" + " Given the fields `question`, `history`, `tools`, `tool_call_results`, produce the fields " + "`answer`, `tool_calls`."}, + {"role": "user", "content": "[[ ## question ## ]]\nWhat is 1+2?"}, + {"role": "assistant", + "content": [], + "tool_calls": [{"type": "function", + "function": {"name": "add", "arguments": '{"a": 1, "b": 2}'}, + "id": "call_add"}]}, + {"role": "tool", "content": '{"sum": 3}', "tool_call_id": "call_add", "name": "add"}, + {"role": "user", + "content": "[[ ## question ## ]]\n" + "Use the prior tool result.\n" + "\n" + "Respond with the corresponding output fields, starting with the field `[[ ## " + "answer ## ]]`, and then ending with the marker for `[[ ## completed ## ]]`."}] + assert messages == expected_messages + expected_lm_kwargs = {"tools": [{"type": "function", + "function": {"name": "add", + "description": "Add two numbers.", + "parameters": {"type": "object", + "properties": {"a": {"type": "integer"}, "b": {"type": "integer"}}, + "required": ["a", "b"]}}}]} + assert lm_kwargs == expected_lm_kwargs + def test_chat_adapter_format_exact_messages_with_tool_input(): def search(query: str, k: int = 3) -> str: """Search for documents.""" @@ -1926,7 +1998,13 @@ def get_weather(city: str) -> str: ) assert result[0]["tool_calls"] == dspy.ToolCalls( - tool_calls=[dspy.ToolCalls.ToolCall(name="get_weather", args={"city": "Paris"})] + tool_calls=[ + dspy.ToolCalls.ToolCall( + name="get_weather", + args={"city": "Paris"}, + id="call_pQm8ajtSMxgA0nrzK2ivFmxG", + ) + ] ) # `answer` is not present, so we set it to None assert result[0]["answer"] is None diff --git a/tests/adapters/test_json_adapter.py b/tests/adapters/test_json_adapter.py index c6dd1f679d..bd89f3b70d 100644 --- a/tests/adapters/test_json_adapter.py +++ b/tests/adapters/test_json_adapter.py @@ -577,8 +577,6 @@ class NativeToolSignature(dspy.Signature): expected_messages = [{"role": "system", "content": "Your input fields are:\n" "1. `question` (str):\n" - "Your output fields are:\n" - "\n" "All interactions will be structured in the following way, with the appropriate " "values filled in.\n" "\n" @@ -586,17 +584,11 @@ class NativeToolSignature(dspy.Signature): "\n" "[[ ## question ## ]]\n" "{question}\n" - "\n" - "Outputs will be a JSON object with the following fields.\n" - "\n" - "{}\n" "In adhering to this structure, your objective is: \n" " Given the fields `question`, `tools`, produce the fields `tool_calls`."}, {"role": "user", "content": "[[ ## question ## ]]\n" - "Q?\n" - "\n" - "Respond with a JSON object in the following order of fields: ."}] + "Q?"}] assert messages == expected_messages expected_lm_kwargs = {"tools": [{"type": "function", "function": {"name": "search", @@ -1500,7 +1492,13 @@ def get_weather(city: str) -> str: ) assert result[0]["tool_calls"] == dspy.ToolCalls( - tool_calls=[dspy.ToolCalls.ToolCall(name="get_weather", args={"city": "Paris"})] + tool_calls=[ + dspy.ToolCalls.ToolCall( + name="get_weather", + args={"city": "Paris"}, + id="call_pQm8ajtSMxgA0nrzK2ivFmxG", + ) + ] ) # `answer` is not present, so we set it to None assert result[0]["answer"] is None diff --git a/tests/adapters/test_tool.py b/tests/adapters/test_tool.py index cfcffe0947..9d68080421 100644 --- a/tests/adapters/test_tool.py +++ b/tests/adapters/test_tool.py @@ -1,4 +1,5 @@ import asyncio +import json from typing import Any import pytest @@ -397,9 +398,7 @@ def test_async_tool_call_in_sync_mode(): ([], {"tool_calls": []}), ( [{"name": "search", "args": {"query": "hello"}}], - { - "tool_calls": [{"type": "function", "function": {"name": "search", "arguments": {"query": "hello"}}}], - }, + {"tool_calls": [{"type": "function", "function": {"name": "search", "arguments": {"query": "hello"}}}]}, ), ( [ @@ -409,18 +408,13 @@ def test_async_tool_call_in_sync_mode(): { "tool_calls": [ {"type": "function", "function": {"name": "search", "arguments": {"query": "hello"}}}, - { - "type": "function", - "function": {"name": "translate", "arguments": {"text": "world", "lang": "fr"}}, - }, + {"type": "function", "function": {"name": "translate", "arguments": {"text": "world", "lang": "fr"}}}, ], }, ), ( [{"name": "get_time", "args": {}}], - { - "tool_calls": [{"type": "function", "function": {"name": "get_time", "arguments": {}}}], - }, + {"tool_calls": [{"type": "function", "function": {"name": "get_time", "arguments": {}}}]}, ), ] @@ -450,6 +444,86 @@ def test_tool_calls_format_from_dict_list(): assert result["tool_calls"][1]["function"]["name"] == "translate" +def test_tool_calls_preserve_call_ids_and_fill_missing_ids(): + tool_calls = ToolCalls.from_dict_list( + [ + {"name": "search", "args": {"query": "hello"}, "id": "call_search"}, + {"name": "lookup", "args": {"key": "world"}}, + ] + ).with_call_ids("call") + + assert [tool_call.id for tool_call in tool_calls.tool_calls] == ["call_search", "call_1"] + assert [part.id for part in tool_calls.to_lm_parts()] == ["call_search", "call_1"] + assert "id" not in str(tool_calls.format()) + + +def test_tool_calls_schema_does_not_prompt_for_call_ids(): + schema = ToolCalls.model_json_schema() + assert "id" not in schema["$defs"]["ToolCall"]["properties"] + + +def test_tool_calls_parse_provider_shape_with_json_arguments_and_call_id(): + tool_calls = ToolCalls.model_validate( + [ + { + "type": "function", + "call_id": "call_search", + "function": {"name": "search", "arguments": '{"query": "hello"}'}, + } + ] + ) + + assert tool_calls.tool_calls[0].id == "call_search" + assert tool_calls.tool_calls[0].name == "search" + assert tool_calls.tool_calls[0].args == {"query": "hello"} + + +def test_tool_call_results_roundtrip_and_lm_messages(): + tool_calls = ToolCalls.from_dict_list([{"name": "search", "args": {"query": "hello"}, "id": "call_search"}]) + results = dspy.ToolCallResults.from_tool_calls_and_values(tool_calls.tool_calls, ["world"]) + + assert "previous tool calls" in dspy.ToolCallResults.description() + assert results.format() == { + "tool_call_results": [{"call_id": "call_search", "name": "search", "value": "world", "is_error": False}] + } + message = results.to_lm_messages()[0] + assert message.role == "tool" + assert message.parts[0].call_id == "call_search" + assert message.parts[0].name == "search" + + +def test_tool_call_results_serialize_values_like_adapter_fields(): + class ResultModel(BaseModel): + answer: str + score: float + + tool_calls = ToolCalls.from_dict_list([{"name": "search", "args": {}, "id": "call_search"}]) + results = dspy.ToolCallResults.from_tool_calls_and_values( + tool_calls.tool_calls, + [ResultModel(answer="café", score=0.9)], + ) + + assert results.format()["tool_call_results"][0]["value"] == {"answer": "café", "score": 0.9} + assert json.dumps(results.format()) + + message = results.to_lm_messages()[0] + assert message.parts[0].content[0].text == '{"answer": "café", "score": 0.9}' + + +def test_history_serialization_preserves_tool_call_ids(): + tool_calls = dspy.ToolCalls.from_dict_list([{"name": "search", "args": {"query": "hello"}, "id": "call_search"}]) + tool_results = dspy.ToolCallResults.from_tool_calls_and_values(tool_calls.tool_calls, ["world"]) + history = dspy.History(messages=[{"tool_calls": tool_calls, "tool_call_results": tool_results}]) + + serialized = history.model_dump() + restored = dspy.History.model_validate(serialized) + + serialized_tool_call = serialized["messages"][0]["tool_calls"]["tool_calls"][0] + assert serialized_tool_call == {"name": "search", "args": {"query": "hello"}, "id": "call_search"} + assert restored.messages[0]["tool_calls"]["tool_calls"][0]["id"] == "call_search" + assert restored.messages[0]["tool_call_results"]["tool_call_results"][0]["call_id"] == "call_search" + + def test_toolcalls_vague_match(): """ Test that ToolCalls can parse the data with slightly off format: @@ -551,10 +625,7 @@ def get_weather(city: str) -> str: def add_numbers(a: int, b: int) -> int: return a + b - tools = [ - dspy.Tool(get_weather), - dspy.Tool(add_numbers) - ] + tools = [dspy.Tool(get_weather), dspy.Tool(add_numbers)] tool_call = dspy.ToolCalls.ToolCall(name="get_weather", args={"city": "Berlin"}) result = tool_call.execute(functions=tools) @@ -575,11 +646,8 @@ def get_pi(): # Test error case tool_call4 = dspy.ToolCalls.ToolCall(name="nonexistent", args={}) - try: + with pytest.raises(ValueError, match="not found"): tool_call4.execute(functions=tools) - assert False, "Should have raised ValueError" - except ValueError as e: - assert "not found" in str(e) def test_tool_call_execute_with_local_functions(): diff --git a/tests/predict/test_reactv2.py b/tests/predict/test_reactv2.py new file mode 100644 index 0000000000..3bc8d35d11 --- /dev/null +++ b/tests/predict/test_reactv2.py @@ -0,0 +1,240 @@ +import pydantic + +import dspy +from dspy.predict.reactv2 import ReActV2, ToolExecutionResult, _build_submit_tool +from dspy.utils.dummies import DummyLM + + +def add(a: int, b: int) -> int: + """Add two numbers.""" + return a + b + + +def test_submit_tool_returns_dict(): + signature = dspy.Signature("question -> answer") + submit = _build_submit_tool(signature) + + assert submit(answer="42") == {"answer": "42"} + + +def test_submit_tool_accepts_structured_output(): + class Answer(pydantic.BaseModel): + text: str + score: float + + class StructuredSignature(dspy.Signature): + question: str = dspy.InputField() + answer: Answer = dspy.OutputField() + + submit = _build_submit_tool(StructuredSignature) + + assert submit(answer={"text": "done", "score": 0.9}) == {"answer": Answer(text="done", score=0.9)} + + +def test_forward_with_structured_submit_output(): + class Answer(pydantic.BaseModel): + text: str + score: float + + class StructuredSignature(dspy.Signature): + question: str = dspy.InputField() + answer: Answer = dspy.OutputField() + + lm = DummyLM( + [ + { + "next_thought": "I can answer.", + "tool_calls": [{"name": "submit", "args": {"answer": {"text": "done", "score": 0.9}}}], + }, + ] + ) + dspy.configure(lm=lm) + + result = ReActV2(StructuredSignature, tools=[add])(question="finish") + + assert result.answer == Answer(text="done", score=0.9) + assert result.history.messages[-1]["answer"] == Answer(text="done", score=0.9) + + +def test_basic_forward_with_submit_records_history_messages(): + lm = DummyLM( + [ + {"next_thought": "I should add.", "tool_calls": [{"name": "add", "args": {"a": 1, "b": 2}}]}, + {"next_thought": "I have the answer.", "tool_calls": [{"name": "submit", "args": {"answer": "3"}}]}, + ] + ) + dspy.configure(lm=lm) + react = ReActV2("question -> answer", tools=[add]) + + result = react(question="What is 1+2?") + + assert result.answer == "3" + assert len(result.history.messages) == 2 + assert result.history.messages[0]["question"] == "What is 1+2?" + assert result.history.messages[0]["next_thought"] == "I should add." + assert result.history.messages[0]["tool_call_results"].tool_call_results[0].value == 3 + assert result.history.messages[-1]["answer"] == "3" + second_call_user_messages = [ + message["content"] for message in lm.history[1]["messages"] if message["role"] == "user" + ] + assert any("tool_call_results" in content for content in second_call_user_messages) + assert all("None" not in content for content in second_call_user_messages) + assert "What is 1+2?" in second_call_user_messages[0] + assert "What is 1+2?" not in second_call_user_messages[-1] + + +def test_forward_with_native_tool_calling_renders_tool_results_as_tool_messages(): + class NativeToolLoopLM(dspy.BaseLM): + def __init__(self): + super().__init__(model="openai/gpt-5-nano", cache=False) + self.calls = [] + + @property + def supported_params(self): + return frozenset() + + @property + def supports_function_calling(self): + return True + + @property + def supports_reasoning(self): + return False + + @property + def supports_response_schema(self): + return False + + def __call__(self, messages, **kwargs): + self.calls.append({"messages": messages, "kwargs": kwargs}) + if len(self.calls) == 1: + return [ + { + "text": "[[ ## next_thought ## ]]\nI should add.\n\n[[ ## completed ## ]]", + "tool_calls": [ + { + "function": {"name": "add", "arguments": '{"a": 1, "b": 2}'}, + "id": "call_add", + "type": "function", + } + ], + } + ] + return [ + { + "text": "[[ ## next_thought ## ]]\nI can answer.\n\n[[ ## completed ## ]]", + "tool_calls": [ + { + "function": {"name": "submit", "arguments": '{"answer": "3"}'}, + "id": "call_submit", + "type": "function", + } + ], + } + ] + + lm = NativeToolLoopLM() + with dspy.context(lm=lm, adapter=dspy.ChatAdapter(use_native_function_calling=True)): + result = ReActV2("question -> answer", tools=[add])(question="What is 1+2?") + + assert result.answer == "3" + assert result.history.messages[0]["tool_call_results"].tool_call_results[0].call_id == "call_add" + assert any( + message["role"] == "tool" and message["tool_call_id"] == "call_add" for message in lm.calls[1]["messages"] + ) + assert lm.calls[0]["kwargs"]["tools"][0]["function"]["name"] == "add" + + +def test_react_signature_defaults_tool_call_results_to_none(): + react = ReActV2("question -> answer", tools=[add]) + + assert react.react.signature.input_fields["question"].default is None + field = react.react.signature.input_fields["tool_call_results"] + assert field.default is None + assert field.annotation == dspy.ToolCallResults + + +def test_forward_with_existing_history_does_not_append_empty_input_event(): + lm = DummyLM( + [ + {"next_thought": "I have the answer.", "tool_calls": [{"name": "submit", "args": {"answer": "3"}}]}, + ] + ) + dspy.configure(lm=lm) + react = ReActV2("question -> answer", tools=[add]) + history = dspy.History(messages=[{"question": "What is 1+2?"}]) + + result = react(history=history) + + assert result.answer == "3" + assert {} not in result.history.messages + assert result.history.messages[0] == {"question": "What is 1+2?"} + + +def test_forward_accepts_serialized_history(): + lm = DummyLM( + [ + {"next_thought": "I have the answer.", "tool_calls": [{"name": "submit", "args": {"answer": "3"}}]}, + ] + ) + dspy.configure(lm=lm) + react = ReActV2("question -> answer", tools=[add]) + + result = react(history={"messages": [{"question": "What is 1+2?"}]}) + + assert result.answer == "3" + assert isinstance(result.history, dspy.History) + assert result.history.messages[0] == {"question": "What is 1+2?"} + + +def test_unknown_tool_returns_error_observation(): + lm = DummyLM( + [ + {"next_thought": "Call fake.", "tool_calls": [{"name": "nonexistent", "args": {}}]}, + {"next_thought": "Now submit.", "tool_calls": [{"name": "submit", "args": {"answer": "ok"}}]}, + ] + ) + dspy.configure(lm=lm) + react = ReActV2("question -> answer", tools=[add]) + + result = react(question="test") + + assert result.answer == "ok" + tool_results = [ + result + for message in result.history.messages + for result in getattr(message.get("tool_call_results"), "tool_call_results", []) + ] + assert any(result.is_error and "Unknown tool" in str(result.value) for result in tool_results) + + +def test_append_tool_turn_records_observation_ids(): + history = dspy.History(messages=[]) + tool_calls = dspy.ToolCalls.from_dict_list([{"name": "add", "args": {"a": 1, "b": 2}, "id": "call_add"}]) + + ReActV2._append_tool_turn( + history, + next_thought="add", + tool_calls=tool_calls, + tool_results=[ToolExecutionResult(value=3)], + ) + + tool_result = history.messages[0]["tool_call_results"].tool_call_results[0] + assert tool_result.call_id == "call_add" + assert tool_result.name == "add" + + +def test_forced_submit_runs_when_loop_returns_no_tool_calls(): + lm = DummyLM( + [ + {"next_thought": "No call.", "tool_calls": []}, + {"next_thought": "Force submit.", "tool_calls": [{"name": "submit", "args": {"answer": "done"}}]}, + ] + ) + dspy.configure(lm=lm) + react = ReActV2("question -> answer", tools=[add]) + + result = react(question="test") + + assert result.answer == "done" + assert result.termination_reason == "forced_submit"