Merge pull request #21 from community-of-python/images

Add image support for OpenAI client

Author: insani7y

README.md

@@ -24,7 +24,7 @@ import any_llm_client
 
 
 config = any_llm_client.OpenAIConfig(
-    url="http://127.0.0.1:11434/v1/chat/completions", 
+    url="http://127.0.0.1:11434/v1/chat/completions",
     model_name="qwen2.5-coder:1.5b",
     request_extra={"best_of": 3}
 )
@@ -57,7 +57,7 @@ import any_llm_client
 
 
 config = any_llm_client.OpenAIConfig(
-    url="http://127.0.0.1:11434/v1/chat/completions", 
+    url="http://127.0.0.1:11434/v1/chat/completions",
     model_name="qwen2.5-coder:1.5b",
     request_extra={"best_of": 3}
 )
@@ -164,7 +164,9 @@ async with any_llm_client.OpenAIClient(config, ...) as client:
 
 #### Errors
 
-`any_llm_client.LLMClient.request_llm_message()` and `any_llm_client.LLMClient.stream_llm_message_chunks()` will raise `any_llm_client.LLMError` or `any_llm_client.OutOfTokensOrSymbolsError` when the LLM API responds with a failed HTTP status.
+`any_llm_client.LLMClient.request_llm_message()` and `any_llm_client.LLMClient.stream_llm_message_chunks()` will raise:
+- `any_llm_client.LLMError` or `any_llm_client.OutOfTokensOrSymbolsError` when the LLM API responds with a failed HTTP status,
+- `any_llm_client.LLMRequestValidationError` when images are passed to YandexGPT client.
 
 #### Timeouts, proxy & other HTTP settings
 
@@ -203,3 +205,30 @@ await client.request_llm_message("Кек, чо как вообще на нара
 ```
 
 The `extra` parameter is united with `request_extra` in OpenAIConfig
+
+
+#### Passing images
+
+You can pass images to OpenAI client (YandexGPT doesn't support images yet):
+
+```python
+await client.request_llm_message(
+    messages=[
+        any_llm_client.TextContentItem("What's on the image?"),
+        any_llm_client.ImageContentItem("https://upload.wikimedia.org/wikipedia/commons/a/a9/Example.jpg"),
+    ]
+)
+```
+
+You can also pass a data url with base64-encoded image:
+
+```python
+await client.request_llm_message(
+    messages=[
+        any_llm_client.TextContentItem("What's on the image?"),
+        any_llm_client.ImageContentItem(
+            f"data:image/jpeg;base64,{base64.b64encode(image_content_bytes).decode('utf-8')}"
+        ),
+    ]
+)
+```

any_llm_client/__init__.py

@@ -2,26 +2,35 @@
 from any_llm_client.clients.openai import OpenAIClient, OpenAIConfig
 from any_llm_client.clients.yandexgpt import YandexGPTClient, YandexGPTConfig
 from any_llm_client.core import (
+    AnyContentItem,
     AssistantMessage,
+    ContentItemList,
+    ImageContentItem,
     LLMClient,
     LLMConfig,
     LLMError,
+    LLMRequestValidationError,
     Message,
     MessageRole,
     OutOfTokensOrSymbolsError,
     SystemMessage,
+    TextContentItem,
     UserMessage,
 )
 from any_llm_client.main import AnyLLMConfig, get_client
 from any_llm_client.retry import RequestRetryConfig
 
 
 __all__ = [
+    "AnyContentItem",
     "AnyLLMConfig",
     "AssistantMessage",
+    "ContentItemList",
+    "ImageContentItem",
     "LLMClient",
     "LLMConfig",
     "LLMError",
+    "LLMRequestValidationError",
     "Message",
     "MessageRole",
     "MockLLMClient",
@@ -31,6 +40,7 @@
     "OutOfTokensOrSymbolsError",
     "RequestRetryConfig",
     "SystemMessage",
+    "TextContentItem",
     "UserMessage",
     "YandexGPTClient",
     "YandexGPTConfig",

any_llm_client/clients/openai.py

@@ -19,6 +19,7 @@
     Message,
     MessageRole,
     OutOfTokensOrSymbolsError,
+    TextContentItem,
     UserMessage,
 )
 from any_llm_client.http import get_http_client_from_kwargs, make_http_request, make_streaming_http_request
@@ -41,16 +42,34 @@ class OpenAIConfig(LLMConfig):
     api_type: typing.Literal["openai"] = "openai"
 
 
-class ChatCompletionsMessage(pydantic.BaseModel):
+class ChatCompletionsTextContentItem(pydantic.BaseModel):
+    type: typing.Literal["text"] = "text"
+    text: str
+
+
+class ChatCompletionsContentUrl(pydantic.BaseModel):
+    url: str
+
+
+class ChatCompletionsImageContentItem(pydantic.BaseModel):
+    type: typing.Literal["image_url"] = "image_url"
+    image_url: ChatCompletionsContentUrl
+
+
+ChatCompletionsAnyContentItem = ChatCompletionsImageContentItem | ChatCompletionsTextContentItem
+ChatCompletionsContentItemList = typing.Annotated[list[ChatCompletionsAnyContentItem], annotated_types.MinLen(1)]
+
+
+class ChatCompletionsInputMessage(pydantic.BaseModel):
     role: MessageRole
-    content: str
+    content: str | ChatCompletionsContentItemList
 
 
 class ChatCompletionsRequest(pydantic.BaseModel):
     model_config = pydantic.ConfigDict(extra="allow")
     stream: bool
     model: str
-    messages: list[ChatCompletionsMessage]
+    messages: list[ChatCompletionsInputMessage]
     temperature: float
 
 
@@ -67,22 +86,54 @@ class ChatCompletionsStreamingEvent(pydantic.BaseModel):
     choices: typing.Annotated[list[OneStreamingChoice], annotated_types.MinLen(1)]
 
 
+class OneNotStreamingChoiceMessage(pydantic.BaseModel):
+    role: MessageRole
+    content: str
+
+
 class OneNotStreamingChoice(pydantic.BaseModel):
-    message: ChatCompletionsMessage
+    message: OneNotStreamingChoiceMessage
 
 
 class ChatCompletionsNotStreamingResponse(pydantic.BaseModel):
     choices: typing.Annotated[list[OneNotStreamingChoice], annotated_types.MinLen(1)]
 
 
+def _prepare_one_message(one_message: Message) -> ChatCompletionsInputMessage:
+    if isinstance(one_message.content, str):
+        return ChatCompletionsInputMessage(role=one_message.role, content=one_message.content)
+    content_items: typing.Final = [
+        ChatCompletionsTextContentItem(text=one_content_item.text)
+        if isinstance(one_content_item, TextContentItem)
+        else ChatCompletionsImageContentItem(image_url=ChatCompletionsContentUrl(url=one_content_item.image_url))
+        for one_content_item in one_message.content
+    ]
+    return ChatCompletionsInputMessage(role=one_message.role, content=content_items)
+
+
+def _merge_content_chunks(
+    content_chunks: list[str | ChatCompletionsContentItemList],
+) -> str | ChatCompletionsContentItemList:
+    if all(isinstance(one_content_chunk, str) for one_content_chunk in content_chunks):
+        return "\n\n".join(typing.cast("list[str]", content_chunks))
+
+    new_content_items: ChatCompletionsContentItemList = []
+    for one_content_chunk in content_chunks:
+        if isinstance(one_content_chunk, str):
+            new_content_items.append(ChatCompletionsTextContentItem(text=one_content_chunk))
+        else:
+            new_content_items += one_content_chunk
+    return new_content_items
+
+
 def _make_user_assistant_alternate_messages(
-    messages: typing.Iterable[ChatCompletionsMessage],
-) -> typing.Iterable[ChatCompletionsMessage]:
+    messages: typing.Iterable[ChatCompletionsInputMessage],
+) -> typing.Iterable[ChatCompletionsInputMessage]:
     current_message_role = MessageRole.user
     current_message_content_chunks = []
 
     for one_message in messages:
-        if not one_message.content.strip():
+        if isinstance(one_message.content, str) and not one_message.content.strip():
             continue
 
         if (
@@ -91,14 +142,16 @@ def _make_user_assistant_alternate_messages(
             current_message_content_chunks.append(one_message.content)
         else:
             if current_message_content_chunks:
-                yield ChatCompletionsMessage(
-                    role=current_message_role, content="\n\n".join(current_message_content_chunks)
+                yield ChatCompletionsInputMessage(
+                    role=current_message_role, content=_merge_content_chunks(current_message_content_chunks)
                 )
             current_message_content_chunks = [one_message.content]
             current_message_role = one_message.role
 
     if current_message_content_chunks:
-        yield ChatCompletionsMessage(role=current_message_role, content="\n\n".join(current_message_content_chunks))
+        yield ChatCompletionsInputMessage(
+            role=current_message_role, content=_merge_content_chunks(current_message_content_chunks)
+        )
 
 
 def _handle_status_error(*, status_code: int, content: bytes) -> typing.NoReturn:
@@ -132,11 +185,9 @@ def _build_request(self, payload: dict[str, typing.Any]) -> httpx.Request:
             headers={"Authorization": f"Bearer {self.config.auth_token}"} if self.config.auth_token else None,
         )
 
-    def _prepare_messages(self, messages: str | list[Message]) -> list[ChatCompletionsMessage]:
+    def _prepare_messages(self, messages: str | list[Message]) -> list[ChatCompletionsInputMessage]:
         messages = [UserMessage(messages)] if isinstance(messages, str) else messages
-        initial_messages: typing.Final = (
-            ChatCompletionsMessage(role=one_message.role, content=one_message.text) for one_message in messages
-        )
+        initial_messages: typing.Final = (_prepare_one_message(one_message) for one_message in messages)
         return (
             list(_make_user_assistant_alternate_messages(initial_messages))
             if self.config.force_user_assistant_message_alternation

any_llm_client/clients/yandexgpt.py

@@ -11,13 +11,15 @@
 import typing_extensions
 
 from any_llm_client.core import (
+    ImageContentItem,
     LLMClient,
     LLMConfig,
     LLMConfigValue,
     LLMError,
+    LLMRequestValidationError,
     Message,
+    MessageRole,
     OutOfTokensOrSymbolsError,
-    UserMessage,
 )
 from any_llm_client.http import get_http_client_from_kwargs, make_http_request, make_streaming_http_request
 from any_llm_client.retry import RequestRetryConfig
@@ -50,15 +52,20 @@ class YandexGPTCompletionOptions(pydantic.BaseModel):
     max_tokens: int = pydantic.Field(gt=0, alias="maxTokens")
 
 
+class YandexGPTMessage(pydantic.BaseModel):
+    role: MessageRole
+    text: str
+
+
 class YandexGPTRequest(pydantic.BaseModel):
     model_config = pydantic.ConfigDict(protected_namespaces=(), extra="allow")
     model_uri: str = pydantic.Field(alias="modelUri")
     completion_options: YandexGPTCompletionOptions = pydantic.Field(alias="completionOptions")
-    messages: list[Message]
+    messages: list[YandexGPTMessage]
 
 
 class YandexGPTAlternative(pydantic.BaseModel):
-    message: Message
+    message: YandexGPTMessage
 
 
 class YandexGPTResult(pydantic.BaseModel):
@@ -111,15 +118,32 @@ def _prepare_payload(
         stream: bool,
         extra: dict[str, typing.Any] | None,
     ) -> dict[str, typing.Any]:
-        messages = [UserMessage(messages)] if isinstance(messages, str) else messages
+        if isinstance(messages, str):
+            prepared_messages = [YandexGPTMessage(role=MessageRole.user, text=messages)]
+        else:
+            prepared_messages = []
+            for one_message in messages:
+                if isinstance(one_message.content, list):
+                    if len(one_message.content) != 1:
+                        raise LLMRequestValidationError(
+                            "YandexGPTClient does not support multiple content items per message"
+                        )
+                    message_content = one_message.content[0]
+                    if isinstance(message_content, ImageContentItem):
+                        raise LLMRequestValidationError("YandexGPTClient does not support image content items")
+                    message_text = message_content.text
+                else:
+                    message_text = one_message.content
+                prepared_messages.append(YandexGPTMessage(role=one_message.role, text=message_text))
+
         return YandexGPTRequest(
             modelUri=f"gpt://{self.config.folder_id}/{self.config.model_name}/{self.config.model_version}",
             completionOptions=YandexGPTCompletionOptions(
                 stream=stream,
                 temperature=self.config._resolve_request_temperature(temperature),  # noqa: SLF001
                 maxTokens=self.config.max_tokens,
             ),
-            messages=messages,
+            messages=prepared_messages,
             **self.config.request_extra | (extra or {}),
         ).model_dump(mode="json", by_alias=True)