From efa97b4fec7e97173f5fbfccce83a07a4f532ab8 Mon Sep 17 00:00:00 2001 From: Robert Haase Date: Fri, 20 Sep 2024 11:20:44 +0200 Subject: [PATCH] add github models marketplace endpoint --- docs/00_setup/environment-cpu.yml | 1 + docs/00_setup/environment-gpu.yml | 1 + .../15_endpoint_apis/05_azure_endpoints.ipynb | 185 ++++++++++++++++++ docs/_toc.yml | 1 + 4 files changed, 188 insertions(+) create mode 100644 docs/15_endpoint_apis/05_azure_endpoints.ipynb diff --git a/docs/00_setup/environment-cpu.yml b/docs/00_setup/environment-cpu.yml index a716c38..7fabc14 100644 --- a/docs/00_setup/environment-cpu.yml +++ b/docs/00_setup/environment-cpu.yml @@ -33,4 +33,5 @@ dependencies: - pygithub - opencv-python - pooch + - azure-ai-inference prefix: C:\Users\haase\miniconda3\envs\genai-cpu diff --git a/docs/00_setup/environment-gpu.yml b/docs/00_setup/environment-gpu.yml index 9eeb228..f05f2da 100644 --- a/docs/00_setup/environment-gpu.yml +++ b/docs/00_setup/environment-gpu.yml @@ -38,4 +38,5 @@ dependencies: - trl - opencv-python - pooch + - azure-ai-inference prefix: C:\Users\haase\miniconda3\envs\genai-gpu diff --git a/docs/15_endpoint_apis/05_azure_endpoints.ipynb b/docs/15_endpoint_apis/05_azure_endpoints.ipynb new file mode 100644 index 0000000..826bdbc --- /dev/null +++ b/docs/15_endpoint_apis/05_azure_endpoints.ipynb @@ -0,0 +1,185 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "300eb7a0-35ca-47bf-a76b-a90799af24ef", + "metadata": {}, + "source": [ + "# Github Models Marketplace\n", + "If you have signed up to [Github Marketplace Models](https://github.com/marketplace/models/), you can use their infrastructure for prompting LLMs. Depending on which LLM you are using, you need to approach them differently. They can be used using the same API key though." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "705270fd-13fc-4576-a1cf-cbd86b9523e8", + "metadata": {}, + "outputs": [], + "source": [ + "def prompt_azure_openai(message:str, model=\"gpt-4o\"):\n", + " \"\"\"A prompt helper function that sends a message to ollama and returns only the text response.\"\"\"\n", + " import os\n", + " import openai\n", + " if isinstance(message, str):\n", + " message = [{\"role\": \"user\", \"content\": message}]\n", + " \n", + " # setup connection to the LLM\n", + " client = openai.OpenAI(\n", + " base_url = \"https://models.inference.ai.azure.com/\",\n", + " api_key = os.environ.get(\"GH_MODELS_API_KEY\") # potentially enter a different key from your environment\n", + " )\n", + " response = client.chat.completions.create(\n", + " model=model,\n", + " messages=message\n", + " )\n", + " \n", + " # extract answer\n", + " return response.choices[0].message.content" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "0e525565-69ba-44c1-aee9-a55be03799fe", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'The capital of France is Paris.'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "prompt_azure_openai(\"What is the capital of france?\")" + ] + }, + { + "cell_type": "markdown", + "id": "b1fac02a-9b2d-4c6f-97b0-f3228f2b9217", + "metadata": {}, + "source": [ + "## Meta LLama 3.1" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "e8db1b13-687b-471f-9c04-8e88479f1e75", + "metadata": {}, + "outputs": [], + "source": [ + "def prompt_azure(message:str, model=\"meta-llama-3.1-70b-instruct\"):\n", + " import os\n", + " from azure.ai.inference import ChatCompletionsClient\n", + " from azure.ai.inference.models import SystemMessage, UserMessage\n", + " from azure.core.credentials import AzureKeyCredential\n", + " \n", + " client = ChatCompletionsClient(\n", + " endpoint=\"https://models.inference.ai.azure.com\",\n", + " credential=AzureKeyCredential(os.environ.get(\"GH_MODELS_API_KEY\")),\n", + " )\n", + " \n", + " response = client.complete(\n", + " messages=[\n", + " SystemMessage(content=\"You are a helpful assistant.\"),\n", + " UserMessage(content=\"What is the capital of France?\"),\n", + " ],\n", + " temperature=1.0,\n", + " top_p=1.0,\n", + " max_tokens=1000,\n", + " model=model\n", + " )\n", + " \n", + " print(response.choices[0].message.content)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "8d636dcd-74aa-45b4-b3ee-213148db7efa", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The capital of France is Paris.\n" + ] + } + ], + "source": [ + "prompt_azure(\"What is the capital of france?\")" + ] + }, + { + "cell_type": "markdown", + "id": "69fc048e-cb22-4943-a824-01dd773b7fea", + "metadata": {}, + "source": [ + "## Phi 3.5\n", + "The same function also works with other models" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "670c96f7-423b-49d1-a3f2-3ee1936d5a96", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " The capital of France is Paris. It is not only the largest city in France but also a global center for art, fashion, gastronomy, and culture. Paris is known for its landmarks such as the Eiffel Tower, Notre-Dame Cathedral, and the Louvre Museum, which is the world's largest art museum. The city is also one of the major economic and political hubs in Europe.\n" + ] + } + ], + "source": [ + "prompt_azure(\"What is the capital of france?\", model=\"Phi-3.5-mini-instruct\")" + ] + }, + { + "cell_type": "markdown", + "id": "de348c4f-44a3-4151-bab8-9ff2fb65153e", + "metadata": {}, + "source": [ + "## Exercise\n", + "Check out different models on the [Github Models Marketplace](https://github.com/marketplace/models/) and invoke another one, e.g. a Mistral model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c1cd22ef-2178-4551-89ca-000dd982eb3a", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/docs/_toc.yml b/docs/_toc.yml index 6c106ad..ff282c5 100644 --- a/docs/_toc.yml +++ b/docs/_toc.yml @@ -25,6 +25,7 @@ parts: - file: 15_endpoint_apis/02_ollama_endpoint.ipynb - file: 15_endpoint_apis/04_scadsai_llm_endpoint.ipynb - file: 15_endpoint_apis/03_blablador_endpoint.ipynb + - file: 15_endpoint_apis/05_azure_endpoints.ipynb - file: 15_endpoint_apis/10_anthropic_api.ipynb - file: 15_endpoint_apis/20_google_gemini.ipynb - file: 15_endpoint_apis/30_huggingface.ipynb