diff --git a/cookbook_ru/gigachat_vision/cat.jpg b/cookbook_ru/gigachat_vision/cat.jpg new file mode 100644 index 0000000000000..d84aec4976e6c Binary files /dev/null and b/cookbook_ru/gigachat_vision/cat.jpg differ diff --git a/cookbook_ru/gigachat_vision/gigachat_vision.ipynb b/cookbook_ru/gigachat_vision/gigachat_vision.ipynb new file mode 100644 index 0000000000000..f3a804da50bf1 --- /dev/null +++ b/cookbook_ru/gigachat_vision/gigachat_vision.ipynb @@ -0,0 +1,192 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "collapsed": false + }, + "source": [ + "# Работа с GigaChat Vision\n", + "В этом ноутбуке мы рассмотрим работу с GigaChat Vision\n", + "\n", + "Наши действия:\n", + "1. Загружаем фото на S3 хранилище GigaChat\n", + "2. Просим GigaChat проанализировать фотографии, выделить сущности на них и написать описание к фото\n", + "3. Выдать ответ в виде JSON, после чего преобразуем этот JSON в Pydantic модель\n", + "\n", + "Тестировать будем на этих фото\n", + "![фото 1](cat.jpg)\n", + "![фото 2](sea.jpg)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "# !pip install -U gigachain-community" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "is_executing": true + } + } + }, + { + "cell_type": "code", + "execution_count": 1, + "outputs": [], + "source": [ + "import os\n", + "import getpass\n", + "\n", + "if \"GIGACHAT_CREDENTIALS\" not in os.environ:\n", + " os.environ[\"GIGACHAT_CREDENTIALS\"] = getpass.getpass(\"Введите credentials gigachat: \")" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "markdown", + "source": [ + "Пока что Vision модель доступна в качестве preview на урле для новых моделей.\n", + "После релиза, можно будет использовать стандартный прод base_url и модель `GigaChat-Pro`" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "from langchain_community.chat_models.gigachat import GigaChat\n", + "\n", + "llm = GigaChat(\n", + " verify_ssl_certs=False,\n", + " timeout=6000,\n", + " base_url=\"https://gigachat-preview.devices.sberbank.ru/api/v1\",\n", + " model=\"GigaChat-Pro-preview\",\n", + " temperature=0.01,\n", + " flags=[]\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from typing import List, Optional\n", + "\n", + "from langchain.output_parsers import PydanticOutputParser\n", + "from langchain_core.messages import HumanMessage\n", + "from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder\n", + "from langchain_core.pydantic_v1 import BaseModel, Field, validator\n", + "from langchain_core.runnables import RunnableLambda, RunnableParallel\n", + "\n", + "\n", + "class Photo(BaseModel):\n", + " \"\"\"Информация о фото\"\"\"\n", + "\n", + " content: str = Field(..., description=\"Что изображено на фото? 1-3 слова\")\n", + " description: str = Field(..., description=\"Опиши детальнее фото\")\n", + "\n", + "\n", + "# Set up a parser\n", + "parser = PydanticOutputParser(pydantic_object=Photo)\n", + "\n", + "\n", + "def _get_messages_from_url(url: str):\n", + " return {\n", + " \"history\": [\n", + " HumanMessage(content=\"\", additional_kwargs={\"attachments\": [url]}),\n", + " ]\n", + " }\n", + "\n", + "\n", + "# Prompt\n", + "prompt = ChatPromptTemplate.from_messages(\n", + " [\n", + " (\n", + " \"system\",\n", + " \"Определи объект на фото. Ответь на запрос пользователя в формате JSON. Schema Information: \\n{format_instructions}\",\n", + " ),\n", + " MessagesPlaceholder(\"history\"),\n", + " ]\n", + ").partial(format_instructions=parser.get_format_instructions())\n", + "\n", + "chain = RunnableLambda(_get_messages_from_url) | prompt | llm | parser" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": false + }, + "source": [ + "В итоге мы создали LCEL цепочку, в которую мы можем передавать id загруженных файлов и в ответ получать Pydantic модели" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": "[Photo(content='котенок', description='милый котенок с черно-белой шерстью и большими глазами'),\n Photo(content='пляж', description='Пляж с каменистым берегом и волнами. Вдалеке видны здания и маяк.')]" + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Пример работы\n", + "file = llm.upload_file(open(\"cat.jpg\", \"rb\"))\n", + "file2 = llm.upload_file(open(\"sea.jpg\", \"rb\"))\n", + "chain.batch([file.id_, file2.id_])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": false + }, + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/cookbook_ru/gigachat_vision/sea.jpg b/cookbook_ru/gigachat_vision/sea.jpg new file mode 100644 index 0000000000000..78c1de03b93fb Binary files /dev/null and b/cookbook_ru/gigachat_vision/sea.jpg differ