serve_LLM_Using_vLLM_framework.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4",
      "machine_shape": "hm"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "Author: [Zoumana KEITA](https://www.linkedin.com/in/zoumana-keita/)"
      ],
      "metadata": {
        "id": "O1zmah9ZeHfS"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "_UUZ-4OB_Auw"
      },
      "outputs": [],
      "source": [
        "!pip -q install vllm"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from vllm import LLM, SamplingParams"
      ],
      "metadata": {
        "id": "J8ddGSm_CdNM"
      },
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "prompts = [\"Abidjan is located in\",\n",
        "           \"A Data Scientist is a person who\",\n",
        "           \"The future of agriculture in Africa is\"]  # Sample prompts.\n",
        "\n",
        "sampling_params = SamplingParams(temperature=0.8, top_p=0.95, max_tokens=50)"
      ],
      "metadata": {
        "id": "MYgyk6DdCfqv"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "llm = LLM(model=\"facebook/opt-125m\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jjsGXdpWVwFB",
        "outputId": "01ae7935-389e-4c13-e300-2d3dfaaebeb4"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO 08-17 02:04:45 llm_engine.py:70] Initializing an LLM engine with config: model='facebook/opt-125m', tokenizer='facebook/opt-125m', tokenizer_mode=auto, trust_remote_code=False, dtype=torch.float16, use_dummy_weights=False, download_dir=None, use_np_weights=False, tensor_parallel_size=1, seed=0)\n",
            "INFO 08-17 02:04:50 llm_engine.py:196] # GPU blocks: 22217, # CPU blocks: 7281\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "outputs = llm.generate(prompts, sampling_params)  # Generate texts from the prompts."
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "idcChyvWYYGJ",
        "outputId": "626c55c7-6fe2-4e26-ca91-1a836dbbeadb"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Processed prompts: 100%|██████████| 3/3 [00:00<00:00,  4.92it/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import textwrap\n",
        "wrapper = textwrap.TextWrapper(width=60)"
      ],
      "metadata": {
        "id": "uagCI4FqfytN"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Print the outputs.\n",
        "for output in outputs:\n",
        "    prompt = output.prompt\n",
        "    generated_text = output.outputs[0].text\n",
        "    print(f\"Prompt: {prompt}\\nGenerated text: { wrapper.fill(generated_text)}\")\n",
        "    print('---'*5)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nmeEmILHX_Ad",
        "outputId": "21bd3496-bee0-48ee-d760-acfe782a9ac3"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Prompt: Abidjan is located in\n",
            "Generated text:  the north of the country on the banks of the Karaba river.\n",
            "It is a UNESCO World Heritage site and is the most visited\n",
            "tourist destination in the north of the country.  History\n",
            "Abidjan’s origins  The\n",
            "---------------\n",
            "Prompt: A Data Scientist is a person who\n",
            "Generated text:  is passionate about data science. I am a Data Scientist for\n",
            "6 years now and in between working in data science and\n",
            "programming, I am a Data Scientist.  It is the truth.  I am\n",
            "in my first year in Data Science\n",
            "---------------\n",
            "Prompt: The future of agriculture in Africa is\n",
            "Generated text:  changing, according to a new report.  A recent report by a\n",
            "ZN World Affairs Council based on data from the African\n",
            "Development Bank (ADB) shows that one in four producers of\n",
            "agriculture globally will be changing their local and\n",
            "international production patterns\n",
            "---------------\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# To use vLLM for online serving, you can start an OpenAI API-compatible server via:\n",
        "\n"
      ],
      "metadata": {
        "id": "QupNp9f7EibZ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!python -m vllm.entrypoints.openai.api_server --host 127.0.0.1 --port 8000 --model facebook/opt-125m & npx localtunnel --port 8000"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "d02pCcghnVH8",
        "outputId": "4bc696f1-a3b5-432b-e4af-101c00d1a3e1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[K\u001b[?25hnpx: installed 22 in 2.107s\n",
            "your url is: https://tricky-taxis-smash.loca.lt\n",
            "INFO 08-17 03:59:51 llm_engine.py:70] Initializing an LLM engine with config: model='facebook/opt-125m', tokenizer='facebook/opt-125m', tokenizer_mode=auto, trust_remote_code=False, dtype=torch.float16, use_dummy_weights=False, download_dir=None, use_np_weights=False, tensor_parallel_size=1, seed=0)\n",
            "INFO 08-17 03:59:58 llm_engine.py:196] # GPU blocks: 22217, # CPU blocks: 7281\n",
            "\u001b[32mINFO\u001b[0m:     Started server process [\u001b[36m103011\u001b[0m]\n",
            "\u001b[32mINFO\u001b[0m:     Waiting for application startup.\n",
            "\u001b[32mINFO\u001b[0m:     Application startup complete.\n",
            "\u001b[32mINFO\u001b[0m:     Uvicorn running on \u001b[1mhttp://127.0.0.1:8000\u001b[0m (Press CTRL+C to quit)\n",
            "INFO 08-17 04:00:11 async_llm_engine.py:117] Received request cmpl-fae223cf2fb54f63adfd194fc074a3d8: prompt: 'Abidjan is located in', sampling params: SamplingParams(n=1, best_of=1, presence_penalty=0.0, frequency_penalty=0.0, temperature=0.8, top_p=0.95, top_k=-1, use_beam_search=False, stop=[], ignore_eos=False, max_tokens=50, logprobs=None), prompt token ids: None.\n",
            "INFO 08-17 04:00:11 llm_engine.py:394] Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 1 reqs, Swapped: 0 reqs, Pending: 0 reqs, GPU KV cache usage: 0.0%, CPU KV cache usage: 0.0%\n",
            "INFO 08-17 04:00:11 async_llm_engine.py:171] Finished request cmpl-fae223cf2fb54f63adfd194fc074a3d8.\n",
            "\u001b[32mINFO\u001b[0m:     50.27.36.141:0 - \"\u001b[1mPOST /v1/completions HTTP/1.1\u001b[0m\" \u001b[32m200 OK\u001b[0m\n",
            "INFO 08-17 04:00:38 async_llm_engine.py:117] Received request cmpl-dbdfe8371989450dbeaac2dcaa25697c: prompt: 'Abidjan is located in', sampling params: SamplingParams(n=1, best_of=1, presence_penalty=0.0, frequency_penalty=0.0, temperature=0.8, top_p=0.95, top_k=-1, use_beam_search=False, stop=[], ignore_eos=False, max_tokens=50, logprobs=None), prompt token ids: None.\n",
            "INFO 08-17 04:00:38 llm_engine.py:394] Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 1 reqs, Swapped: 0 reqs, Pending: 0 reqs, GPU KV cache usage: 0.0%, CPU KV cache usage: 0.0%\n",
            "INFO 08-17 04:00:39 async_llm_engine.py:171] Finished request cmpl-dbdfe8371989450dbeaac2dcaa25697c.\n",
            "\u001b[32mINFO\u001b[0m:     50.27.36.141:0 - \"\u001b[1mPOST /v1/completions HTTP/1.1\u001b[0m\" \u001b[32m200 OK\u001b[0m\n",
            "INFO 08-17 04:02:26 async_llm_engine.py:117] Received request cmpl-860c87c1776c4224b6ca6b2cc7e7f4b7: prompt: 'Abidjan is located in', sampling params: SamplingParams(n=1, best_of=1, presence_penalty=0.0, frequency_penalty=0.0, temperature=0.8, top_p=1.0, top_k=-1, use_beam_search=False, stop=[], ignore_eos=False, max_tokens=50, logprobs=None), prompt token ids: None.\n",
            "INFO 08-17 04:02:26 llm_engine.py:394] Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 1 reqs, Swapped: 0 reqs, Pending: 0 reqs, GPU KV cache usage: 0.0%, CPU KV cache usage: 0.0%\n",
            "INFO 08-17 04:02:26 async_llm_engine.py:171] Finished request cmpl-860c87c1776c4224b6ca6b2cc7e7f4b7.\n",
            "\u001b[32mINFO\u001b[0m:     50.27.36.141:0 - \"\u001b[1mPOST /v1/completions HTTP/1.1\u001b[0m\" \u001b[32m200 OK\u001b[0m\n",
            "INFO 08-17 04:18:26 async_llm_engine.py:117] Received request cmpl-b6dce3007dd84e3daeafac7ce310d0ce: prompt: 'Abidjan is located in', sampling params: SamplingParams(n=1, best_of=1, presence_penalty=0.0, frequency_penalty=0.0, temperature=0.8, top_p=1.0, top_k=-1, use_beam_search=False, stop=[], ignore_eos=False, max_tokens=50, logprobs=None), prompt token ids: None.\n",
            "INFO 08-17 04:18:26 llm_engine.py:394] Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 1 reqs, Swapped: 0 reqs, Pending: 0 reqs, GPU KV cache usage: 0.0%, CPU KV cache usage: 0.0%\n",
            "INFO 08-17 04:18:27 async_llm_engine.py:171] Finished request cmpl-b6dce3007dd84e3daeafac7ce310d0ce.\n",
            "\u001b[32mINFO\u001b[0m:     50.27.36.141:0 - \"\u001b[1mPOST /v1/completions HTTP/1.1\u001b[0m\" \u001b[32m200 OK\u001b[0m\n",
            "INFO 08-17 04:19:20 async_llm_engine.py:117] Received request cmpl-aaf3c765be1041c4beeb85ab5540e725: prompt: 'Abidjan is located in', sampling params: SamplingParams(n=1, best_of=1, presence_penalty=0.0, frequency_penalty=0.0, temperature=0.8, top_p=1.0, top_k=-1, use_beam_search=False, stop=[], ignore_eos=False, max_tokens=50, logprobs=None), prompt token ids: None.\n",
            "INFO 08-17 04:19:20 llm_engine.py:394] Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 1 reqs, Swapped: 0 reqs, Pending: 0 reqs, GPU KV cache usage: 0.0%, CPU KV cache usage: 0.0%\n",
            "INFO 08-17 04:19:20 async_llm_engine.py:171] Finished request cmpl-aaf3c765be1041c4beeb85ab5540e725.\n",
            "\u001b[32mINFO\u001b[0m:     50.27.36.141:0 - \"\u001b[1mPOST /v1/completions HTTP/1.1\u001b[0m\" \u001b[32m200 OK\u001b[0m\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!curl https://tricky-taxis-smash.loca.lt/v1/completions \\\n",
        "-H \"Content-Type: application/json\" \\\n",
        "-d '{\"model\": \"facebook/opt-125m\",\"prompt\": \"Abidjan is located in\", \"max_tokens\": 50, \"temperature\": 0.8}'"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "w_q0I-BdNam1",
        "outputId": "c554737f-eac3-4f33-99ef-a51fa8ce2d22"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "404"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
            "                                 Dload  Upload   Total   Spent    Left  Speed\n",
            "\r  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0\r100   105  100     3  100   102      7    260 --:--:-- --:--:-- --:--:--   268\n"
          ]
        }
      ]
    }
  ]
}