diff --git a/CHANGELOG.md b/CHANGELOG.md index e5b5428..2a44091 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ Notable additions, fixes, or breaking changes to the Freeplay SDK. ### Added +- **`gemini_api_chat` flavor**: New flavor for the Gemini API (`google-generativeai` SDK). Returns plain-dict tool schemas compatible with `google.genai`, while `gemini_chat` continues to return `vertexai.generative_models.Tool` objects for Vertex AI users. +- **Gemini message parts passthrough**: History messages already in Gemini format (with `parts`, e.g., function calls and function responses) are now passed through without re-wrapping. Role `"assistant"` is automatically translated to `"model"`. + - Interactive REPL for development and testing: - `make repl` - Production mode (connects to app.freeplay.ai with SSL verification enabled) - `make repl-local` - Local development mode (connects to localhost:8000 with SSL verification disabled) diff --git a/pyproject.toml b/pyproject.toml index ad9acc5..b767285 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "freeplay" -version = "0.5.6" +version = "0.5.7" description = "" authors = [ {name = "Freeplay Engineering", email = "support@freeplay.ai"}, diff --git a/scripts/type-baseline/pyright-baseline.json b/scripts/type-baseline/pyright-baseline.json index 15ae870..ac7079c 100644 --- a/scripts/type-baseline/pyright-baseline.json +++ b/scripts/type-baseline/pyright-baseline.json @@ -72,46 +72,6 @@ "rule": "reportUnnecessaryIsInstance", "severity": "error" }, - { - "file": "src/freeplay/resources/adapters.py", - "line": 205, - "character": 16, - "message": "Type of \"append\" is partially unknown\n\u00a0\u00a0Type of \"append\" is \"(object: Unknown, /) -> None\"", - "rule": "reportUnknownMemberType", - "severity": "error" - }, - { - "file": "src/freeplay/resources/adapters.py", - "line": 215, - "character": 16, - "message": "Type of \"append\" is partially unknown\n\u00a0\u00a0Type of \"append\" is \"(object: Unknown, /) -> None\"", - "rule": "reportUnknownMemberType", - "severity": "error" - }, - { - "file": "src/freeplay/resources/adapters.py", - "line": 222, - "character": 16, - "message": "Type of \"append\" is partially unknown\n\u00a0\u00a0Type of \"append\" is \"(object: Unknown, /) -> None\"", - "rule": "reportUnknownMemberType", - "severity": "error" - }, - { - "file": "src/freeplay/resources/adapters.py", - "line": 224, - "character": 15, - "message": "Return type, \"list[Unknown]\", is partially unknown", - "rule": "reportUnknownVariableType", - "severity": "error" - }, - { - "file": "src/freeplay/resources/adapters.py", - "line": 239, - "character": 13, - "message": "Unnecessary isinstance call; \"MediaContentUrl\" is always an instance of \"MediaContentUrl\"", - "rule": "reportUnnecessaryIsInstance", - "severity": "error" - }, { "file": "tests/__init__.py", "line": 10, diff --git a/src/freeplay/resources/adapters.py b/src/freeplay/resources/adapters.py index 5047c7e..8b15fce 100644 --- a/src/freeplay/resources/adapters.py +++ b/src/freeplay/resources/adapters.py @@ -196,13 +196,20 @@ def to_llm_syntax( if len(messages) < 1: raise ValueError("Must have at least one message to format") - gemini_messages = [] + gemini_messages: List[Dict[str, Any]] = [] for message in messages: if message["role"] == "system": continue - if "has_media" in message and message["has_media"]: + # Already in Gemini format (e.g., history from previous turns + # with function calls, function responses, or multi-part content) + if "parts" in message: + msg_copy: Dict[str, Any] = copy.deepcopy(message) + if msg_copy.get("role") == "assistant": + msg_copy["role"] = "model" + gemini_messages.append(msg_copy) + elif "has_media" in message and message["has_media"]: gemini_messages.append( { "role": self.__translate_role(message["role"]), @@ -220,7 +227,8 @@ def to_llm_syntax( } ) else: - gemini_messages.append(copy.deepcopy(message)) + fallback: Dict[str, Any] = copy.deepcopy(message) + gemini_messages.append(fallback) return gemini_messages @@ -237,12 +245,11 @@ def __map_content( "mime_type": content.content_type, } } - elif isinstance(content, MediaContentUrl): + else: + # MediaContentUrl -- Gemini does not support image URLs raise ValueError( "Message contains an image URL, but image URLs are not supported by Gemini" ) - else: - raise ValueError(f"Unexpected content type {type(content)}") @staticmethod def __translate_role(role: str) -> str: @@ -336,7 +343,7 @@ def adaptor_for_flavor(flavor_name: str) -> LLMAdapter: return AnthropicAdapter() elif flavor_name == "llama_3_chat": return Llama3Adapter() - elif flavor_name == "gemini_chat": + elif flavor_name in ["gemini_chat", "gemini_api_chat"]: return GeminiAdapter() elif flavor_name == "amazon_bedrock_converse": return BedrockConverseAdapter() diff --git a/src/freeplay/resources/prompts.py b/src/freeplay/resources/prompts.py index e79aba8..8923be9 100644 --- a/src/freeplay/resources/prompts.py +++ b/src/freeplay/resources/prompts.py @@ -236,6 +236,16 @@ def __format_tool_schema(flavor_name: str, tool_schema: List[ToolSchema]) -> Any return [Tool(function_declarations=function_declarations)] except ImportError: raise VertexAIToolSchemaError() + elif flavor_name == "gemini_api_chat": + function_declarations = [ + { + "name": schema.name, + "description": schema.description, + "parameters": schema.parameters, + } + for schema in tool_schema + ] + return [{"functionDeclarations": function_declarations}] raise UnsupportedToolSchemaError() @@ -600,6 +610,7 @@ def __flavor_to_provider(flavor: str) -> str: "anthropic_chat": "anthropic", "openai_chat": "openai", "gemini_chat": "vertex", + "gemini_api_chat": "gemini", } provider = flavor_provider.get(flavor) if not provider: diff --git a/tests/test_adapters.py b/tests/test_adapters.py index fafa9b1..bcb53a4 100644 --- a/tests/test_adapters.py +++ b/tests/test_adapters.py @@ -6,9 +6,11 @@ AnthropicAdapter, GeminiAdapter, BedrockConverseAdapter, + MissingFlavorError, TextContent, MediaContentUrl, MediaContentBase64, + adaptor_for_flavor, ) @@ -338,3 +340,193 @@ def test_bedrock_converse(self) -> None: } }, ) + + # ------------------------------------------------------------------ + # Gemini parts passthrough (history with function calls / responses) + # ------------------------------------------------------------------ + + def test_gemini_parts_passthrough(self) -> None: + """Messages with 'parts' key are passed through without re-wrapping.""" + messages: List[Dict[str, Any]] = [ + { + "role": "user", + "parts": [{"text": "What is the weather?"}], + }, + { + "role": "model", + "parts": [ + { + "functionCall": { + "name": "get_weather", + "args": {"location": "Seattle"}, + } + } + ], + }, + { + "role": "user", + "parts": [ + { + "functionResponse": { + "name": "get_weather", + "response": {"temperature": "72°F"}, + } + } + ], + }, + { + "role": "model", + "parts": [{"text": "It's 72°F in Seattle."}], + }, + ] + + result = GeminiAdapter().to_llm_syntax(messages) + assert isinstance(result, list) + + self.assertEqual(len(result), 4) + # All messages should be preserved as-is + self.assertEqual(result[0]["role"], "user") + self.assertEqual(result[0]["parts"], [{"text": "What is the weather?"}]) + + self.assertEqual(result[1]["role"], "model") + self.assertIn("functionCall", result[1]["parts"][0]) + + self.assertEqual(result[2]["role"], "user") + self.assertIn("functionResponse", result[2]["parts"][0]) + + self.assertEqual(result[3]["role"], "model") + self.assertEqual(result[3]["parts"], [{"text": "It's 72°F in Seattle."}]) + + def test_gemini_parts_passthrough_translates_assistant_to_model(self) -> None: + """Parts messages with role 'assistant' are translated to 'model'.""" + messages: List[Dict[str, Any]] = [ + { + "role": "assistant", + "parts": [ + { + "functionCall": { + "name": "search", + "args": {"query": "test"}, + } + } + ], + }, + ] + + result = GeminiAdapter().to_llm_syntax(messages) + assert isinstance(result, list) + + self.assertEqual(len(result), 1) + self.assertEqual(result[0]["role"], "model") + self.assertIn("functionCall", result[0]["parts"][0]) + + def test_gemini_parts_passthrough_does_not_mutate_original(self) -> None: + """Parts passthrough deep-copies, so original messages are not mutated.""" + original: Dict[str, Any] = { + "role": "assistant", + "parts": [{"text": "original"}], + } + messages: List[Dict[str, Any]] = [original] + + result = GeminiAdapter().to_llm_syntax(messages) + assert isinstance(result, list) + + self.assertEqual(result[0]["role"], "model") + self.assertEqual(original["role"], "assistant") + + def test_gemini_mixed_content_and_parts(self) -> None: + """Handles a mix of standard content and pre-formatted parts messages.""" + messages: List[Dict[str, Any]] = [ + {"role": "system", "content": "System instructions"}, + {"role": "user", "content": "Hello"}, + { + "role": "model", + "parts": [{"functionCall": {"name": "greet", "args": {}}}], + }, + { + "role": "user", + "parts": [ + { + "functionResponse": { + "name": "greet", + "response": {"greeting": "Hi!"}, + } + } + ], + }, + {"role": "assistant", "content": "Done"}, + ] + + result = GeminiAdapter().to_llm_syntax(messages) + assert isinstance(result, list) + + self.assertEqual(len(result), 4) # system is skipped + self.assertEqual(result[0], {"role": "user", "parts": [{"text": "Hello"}]}) + self.assertEqual(result[1]["role"], "model") + self.assertIn("functionCall", result[1]["parts"][0]) + self.assertEqual(result[2]["role"], "user") + self.assertIn("functionResponse", result[2]["parts"][0]) + self.assertEqual(result[3], {"role": "model", "parts": [{"text": "Done"}]}) + + def test_gemini_media_in_history_parts(self) -> None: + """History messages with inlineData parts are passed through correctly.""" + messages: List[Dict[str, Any]] = [ + { + "role": "user", + "parts": [ + {"text": "What's in this image?"}, + { + "inlineData": { + "mimeType": "image/png", + "data": "iVBORw0KGgoAAAANSUhEUg==", + } + }, + ], + }, + {"role": "model", "parts": [{"text": "I see a cat."}]}, + {"role": "user", "content": "Can you describe it in more detail?"}, + ] + + result = GeminiAdapter().to_llm_syntax(messages) + assert isinstance(result, list) + + self.assertEqual(len(result), 3) + + # First message: media parts passed through unchanged + self.assertEqual(result[0]["role"], "user") + self.assertEqual(len(result[0]["parts"]), 2) + self.assertEqual(result[0]["parts"][0], {"text": "What's in this image?"}) + self.assertIn("inlineData", result[0]["parts"][1]) + self.assertEqual(result[0]["parts"][1]["inlineData"]["mimeType"], "image/png") + self.assertEqual( + result[0]["parts"][1]["inlineData"]["data"], "iVBORw0KGgoAAAANSUhEUg==" + ) + + # Second message: text parts passed through + self.assertEqual(result[1]["role"], "model") + self.assertEqual(result[1]["parts"], [{"text": "I see a cat."}]) + + # Third message: standard content converted to Gemini format + self.assertEqual( + result[2], + { + "role": "user", + "parts": [{"text": "Can you describe it in more detail?"}], + }, + ) + + # ------------------------------------------------------------------ + # adaptor_for_flavor() registry tests + # ------------------------------------------------------------------ + + def test_adaptor_for_gemini_chat(self) -> None: + adapter = adaptor_for_flavor("gemini_chat") + self.assertIsInstance(adapter, GeminiAdapter) + + def test_adaptor_for_gemini_api_chat(self) -> None: + adapter = adaptor_for_flavor("gemini_api_chat") + self.assertIsInstance(adapter, GeminiAdapter) + + def test_adaptor_for_unknown_flavor_raises(self) -> None: + with self.assertRaises(MissingFlavorError): + adaptor_for_flavor("nonexistent_flavor") diff --git a/tests/test_freeplay.py b/tests/test_freeplay.py index d90e61f..2076e6c 100644 --- a/tests/test_freeplay.py +++ b/tests/test_freeplay.py @@ -1261,6 +1261,160 @@ def test_prompt_format__history_llama(self) -> None: formatted_prompt.llm_prompt_text, ) + def test_prompt_format__history_gemini(self) -> None: + """Full bind→format flow with Gemini-format history (function call cycle).""" + messages: List[TemplateMessage] = [ + TemplateChatMessage(role="system", content="System message"), + HistoryTemplateMessage(kind="history"), + TemplateChatMessage(role="user", content="User message {{number}}"), + ] + + gemini_api_prompt_info = PromptInfo( + prompt_template_id=str(uuid.uuid4()), + prompt_template_version_id=str(uuid.uuid4()), + template_name="template-name", + environment="environment", + model_parameters=LLMParameters({}), + provider_info=None, + provider="gemini", + model="gemini-2.0-flash", + flavor_name="gemini_api_chat", + ) + + template_prompt = TemplatePrompt(gemini_api_prompt_info, messages=messages) + + # History in Gemini format -- function call/response cycle from a previous turn + history = [ + {"role": "user", "parts": [{"text": "What is the weather?"}]}, + { + "role": "model", + "parts": [ + { + "functionCall": { + "name": "get_weather", + "args": {"location": "Seattle"}, + } + } + ], + }, + { + "role": "user", + "parts": [ + { + "functionResponse": { + "name": "get_weather", + "response": {"temperature": "72°F"}, + } + } + ], + }, + {"role": "model", "parts": [{"text": "It's 72°F in Seattle."}]}, + ] + + bound_prompt = template_prompt.bind({"number": 2}, history=history) + formatted_prompt = bound_prompt.format() + + # System message extracted to system_content + self.assertEqual(formatted_prompt.system_content, "System message") + + # llm_prompt: history messages pass through unchanged, + # template user message converted to Gemini format + self.assertEqual( + formatted_prompt.llm_prompt, + [ + {"role": "user", "parts": [{"text": "What is the weather?"}]}, + { + "role": "model", + "parts": [ + { + "functionCall": { + "name": "get_weather", + "args": {"location": "Seattle"}, + } + } + ], + }, + { + "role": "user", + "parts": [ + { + "functionResponse": { + "name": "get_weather", + "response": {"temperature": "72°F"}, + } + } + ], + }, + {"role": "model", "parts": [{"text": "It's 72°F in Seattle."}]}, + {"role": "user", "parts": [{"text": "User message 2"}]}, + ], + ) + + def test_prompt_format__history_gemini_with_media(self) -> None: + """Full bind→format flow with Gemini-format history containing inline media.""" + messages: List[TemplateMessage] = [ + TemplateChatMessage(role="system", content="System message"), + HistoryTemplateMessage(kind="history"), + TemplateChatMessage(role="user", content="User message {{number}}"), + ] + + gemini_api_prompt_info = PromptInfo( + prompt_template_id=str(uuid.uuid4()), + prompt_template_version_id=str(uuid.uuid4()), + template_name="template-name", + environment="environment", + model_parameters=LLMParameters({}), + provider_info=None, + provider="gemini", + model="gemini-2.0-flash", + flavor_name="gemini_api_chat", + ) + + template_prompt = TemplatePrompt(gemini_api_prompt_info, messages=messages) + + # History with inline media from a previous turn + history = [ + { + "role": "user", + "parts": [ + {"text": "What's in this image?"}, + { + "inlineData": { + "mimeType": "image/png", + "data": "iVBORw0KGgoAAAANSUhEUg==", + } + }, + ], + }, + {"role": "model", "parts": [{"text": "I see a cat in the image."}]}, + ] + + bound_prompt = template_prompt.bind({"number": 2}, history=history) + formatted_prompt = bound_prompt.format() + + self.assertEqual(formatted_prompt.system_content, "System message") + + # History with inlineData parts passes through unchanged + self.assertEqual( + formatted_prompt.llm_prompt, + [ + { + "role": "user", + "parts": [ + {"text": "What's in this image?"}, + { + "inlineData": { + "mimeType": "image/png", + "data": "iVBORw0KGgoAAAANSUhEUg==", + } + }, + ], + }, + {"role": "model", "parts": [{"text": "I see a cat in the image."}]}, + {"role": "user", "parts": [{"text": "User message 2"}]}, + ], + ) + def test_prompt_format__bad_history(self) -> None: # send pass history to prompt that doesn't support it messages: List[TemplateMessage] = [ @@ -1458,6 +1612,112 @@ def test_prompt_format_with_tool_schema_gemini(self) -> None: except ImportError: self.skipTest("Vertex AI SDK not installed") + def test_prompt_format_with_tool_schema_gemini_api_chat(self) -> None: + """gemini_api_chat returns plain dicts (no Vertex AI SDK dependency).""" + messages: List[TemplateMessage] = [ + TemplateChatMessage(role="system", content="System message"), + TemplateChatMessage(role="user", content="User message {{number}}"), + ] + tool_schema = [ + ToolSchema( + name="get_weather", + description="Get weather information", + parameters={ + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state", + }, + }, + "required": ["location"], + }, + ) + ] + + gemini_api_prompt_info = PromptInfo( + prompt_template_id=str(uuid.uuid4()), + prompt_template_version_id=str(uuid.uuid4()), + template_name="template-name", + environment="environment", + model_parameters=LLMParameters({}), + provider_info=None, + provider="gemini", + model="gemini-2.0-flash", + flavor_name="gemini_api_chat", + ) + + template_prompt = TemplatePrompt( + gemini_api_prompt_info, messages=messages, tool_schema=tool_schema + ) + + bound_prompt = template_prompt.bind({"number": 1}) + formatted_prompt = bound_prompt.format() + + # Verify plain dicts are returned (not vertexai.generative_models.Tool) + self.assertIsInstance(formatted_prompt.tool_schema, list) + self.assertEqual(len(formatted_prompt.tool_schema), 1) + self.assertIsInstance(formatted_prompt.tool_schema[0], dict) + + # Verify structure: single Tool with functionDeclarations + tool = formatted_prompt.tool_schema[0] + self.assertIn("functionDeclarations", tool) + self.assertEqual(len(tool["functionDeclarations"]), 1) + + fd = tool["functionDeclarations"][0] + self.assertEqual(fd["name"], "get_weather") + self.assertEqual(fd["description"], "Get weather information") + self.assertEqual(fd["parameters"]["required"], ["location"]) + + def test_prompt_format_with_tool_schema_gemini_api_chat_multiple_tools( + self, + ) -> None: + """Multiple tools are grouped into a single functionDeclarations array.""" + messages: List[TemplateMessage] = [ + TemplateChatMessage(role="user", content="User message {{number}}"), + ] + tool_schema = [ + ToolSchema( + name="get_weather", + description="Get weather", + parameters={"type": "object", "properties": {}}, + ), + ToolSchema( + name="get_time", + description="Get time", + parameters={"type": "object", "properties": {}}, + ), + ] + + prompt_info = PromptInfo( + prompt_template_id=str(uuid.uuid4()), + prompt_template_version_id=str(uuid.uuid4()), + template_name="template-name", + environment="environment", + model_parameters=LLMParameters({}), + provider_info=None, + provider="gemini", + model="gemini-2.0-flash", + flavor_name="gemini_api_chat", + ) + + template_prompt = TemplatePrompt( + prompt_info, messages=messages, tool_schema=tool_schema + ) + + bound_prompt = template_prompt.bind({"number": 1}) + formatted_prompt = bound_prompt.format() + + # Single Tool with two functionDeclarations + self.assertEqual(len(formatted_prompt.tool_schema), 1) + self.assertEqual( + len(formatted_prompt.tool_schema[0]["functionDeclarations"]), 2 + ) + names = [ + fd["name"] for fd in formatted_prompt.tool_schema[0]["functionDeclarations"] + ] + self.assertEqual(names, ["get_weather", "get_time"]) + def test_prompt_format_with_output_schema_openai(self) -> None: messages: List[TemplateMessage] = [ TemplateChatMessage(role="system", content="System message"),