sagemaker-inference-server-batch.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "1ae65fec",
   "metadata": {},
   "source": [
    "# Deploy the finetuned vicuna model on Amazon SageMaker with djl server batch"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e2407531",
   "metadata": {},
   "source": [
    "## Create a SageMaker Model for Deployment\n",
    "As a first step, we'll import the relevant libraries and configure several global variables such as the hosting image that will be used nd the S3 location of our model artifacts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "id": "5806a0f1",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import sagemaker\n",
    "from sagemaker.model import Model\n",
    "from sagemaker import serializers, deserializers\n",
    "from sagemaker import image_uris\n",
    "import boto3\n",
    "import os\n",
    "import time\n",
    "import json\n",
    "import jinja2\n",
    "from pathlib import Path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "id": "24862c4a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "role = sagemaker.get_execution_role()  # execution role for the endpoint\n",
    "sess = sagemaker.session.Session()  # sagemaker session for interacting with different AWS APIs\n",
    "bucket = sess.default_bucket()  # bucket to house artifacts\n",
    "\n",
    "region = sess._region_name # region name of the current SageMaker Studio environment\n",
    "account_id = sess.account_id()  # account_id of the current SageMaker Studio environment\n",
    "\n",
    "s3_client = boto3.client(\"s3\") # client to intreract with S3 API\n",
    "sm_client = boto3.client(\"sagemaker\")  # client to intreract with SageMaker\n",
    "smr_client = boto3.client(\"sagemaker-runtime\") # client to intreract with SageMaker Endpoints\n",
    "jinja_env = jinja2.Environment() # jinja environment to generate model configuration templates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "id": "38930529",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# lookup the inference image uri based on our current region\n",
    "djl_inference_image_uri = (\n",
    "    f\"763104351884.dkr.ecr.{region}.amazonaws.com/djl-inference:0.21.0-deepspeed0.8.3-cu117\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "id": "d00f2f2b",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pretrained model will be downloaded from ---- > s3://sagemaker-us-west-2-687912291502/llama/output/2023-05-10-12-43-02/llama_out/\n"
     ]
    }
   ],
   "source": [
    "pretrained_model_location = \"s3://sagemaker-us-west-2-687912291502/llama/output/2023-05-10-12-43-02/llama_out/\"# Change to the model artifact path in S3 which we get from the fine tune job\n",
    "print(f\"Pretrained model will be downloaded from ---- > {pretrained_model_location}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "29aec093",
   "metadata": {},
   "source": [
    "## Build the inference contianer image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "id": "7b814406",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting Dockerfile.inference\n"
     ]
    }
   ],
   "source": [
    "%%writefile Dockerfile.inference\n",
    "## You should change below region code to the region you used, here sample is use us-west-2\n",
    "From 763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.21.0-deepspeed0.8.3-cu117\n",
    "\n",
    "ENV LANG=C.UTF-8\n",
    "ENV PYTHONUNBUFFERED=TRUE\n",
    "ENV PYTHONDONTWRITEBYTECODE=TRUE\n",
    "\n",
    "## Install transfomers version which support LLaMaTokenizer\n",
    "#RUN python3 -m pip install git+https://github.com/huggingface/transformers.git@68d640f7c368bcaaaecfc678f11908ebbd3d6176\n",
    "## Install transfomers version which support vicuna v1.1 LLaMaTokenizer\n",
    "#RUN python3 -m pip install transformers==4.29.0\n",
    "RUN python3 -m pip install transformers==4.28.0\n",
    "RUN python3 -m pip install bmtrain==0.2.1\n",
    "RUN python3 -m pip install jieba\n",
    "RUN python3 -m pip install tqdm\n",
    "RUN python3 -m pip install tensorboard\n",
    "RUN python3 -m pip install numpy>=1.21.0\n",
    "RUN python3 -m pip install spacy\n",
    "RUN python3 -m pip install opendelta\n",
    "\n",
    "## Make all local GPUs visible\n",
    "ENV NVIDIA_VISIBLE_DEVICES=\"all\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "id": "465b7a51",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING! Your password will be stored unencrypted in /home/ec2-user/.docker/config.json.\n",
      "Configure a credential helper to remove this warning. See\n",
      "https://docs.docker.com/engine/reference/commandline/login/#credentials-store\n",
      "\n",
      "Login Succeeded\n"
     ]
    }
   ],
   "source": [
    "## You should change below region code to the region you used, here sample is use us-west-2\n",
    "!aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin 763104351884.dkr.ecr.us-west-2.amazonaws.com"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "id": "351f761d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "## define repo name, should contain *sagemaker* in the name\n",
    "repo_name = \"sagemaker-vicuna-inference-severbatch-demo\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "id": "4a58a7ce",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Login Succeeded\n",
      "Sending build context to Docker daemon   63.9MB\n",
      "Step 1/13 : From 763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.21.0-deepspeed0.8.3-cu117\n",
      " ---> a63b0d190846\n",
      "Step 2/13 : ENV LANG=C.UTF-8\n",
      " ---> Using cache\n",
      " ---> 311767347229\n",
      "Step 3/13 : ENV PYTHONUNBUFFERED=TRUE\n",
      " ---> Using cache\n",
      " ---> 744d3881b2eb\n",
      "Step 4/13 : ENV PYTHONDONTWRITEBYTECODE=TRUE\n",
      " ---> Using cache\n",
      " ---> 114266d93af2\n",
      "Step 5/13 : RUN python3 -m pip install transformers==4.28.0\n",
      " ---> Using cache\n",
      " ---> 73084c6782b3\n",
      "Step 6/13 : RUN python3 -m pip install bmtrain==0.2.1\n",
      " ---> Running in 64d9ab5911bf\n",
      "Collecting bmtrain==0.2.1\n",
      "  Downloading bmtrain-0.2.1.tar.gz (57 kB)\n",
      "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 58.0/58.0 kB 3.5 MB/s eta 0:00:00\n",
      "  Preparing metadata (setup.py): started\n",
      "  Preparing metadata (setup.py): finished with status 'done'\n",
      "Requirement already satisfied: numpy in /usr/local/lib/python3.8/dist-packages (from bmtrain==0.2.1) (1.24.3)\n",
      "Building wheels for collected packages: bmtrain\n",
      "  Building wheel for bmtrain (setup.py): started\n",
      "  Building wheel for bmtrain (setup.py): still running...\n",
      "  Building wheel for bmtrain (setup.py): still running...\n",
      "  Building wheel for bmtrain (setup.py): finished with status 'done'\n",
      "  Created wheel for bmtrain: filename=bmtrain-0.2.1-cp38-cp38-linux_x86_64.whl size=10710707 sha256=9711727059f857d849db03e5a23dc56ff41b698c4bbe60d8eb5336a1ab857b93\n",
      "  Stored in directory: /root/.cache/pip/wheels/e3/88/b7/231ec604df76724d469371a3d6e83a4c2260e2f05237e4623c\n",
      "Successfully built bmtrain\n",
      "Installing collected packages: bmtrain\n",
      "Successfully installed bmtrain-0.2.1\n",
      "\u001b[91mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\n",
      "\u001b[0mRemoving intermediate container 64d9ab5911bf\n",
      " ---> 8c4f8bb66be4\n",
      "Step 7/13 : RUN python3 -m pip install jieba\n",
      " ---> Running in a34017f772a9\n",
      "Collecting jieba\n",
      "  Downloading jieba-0.42.1.tar.gz (19.2 MB)\n",
      "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 19.2/19.2 MB 102.2 MB/s eta 0:00:00\n",
      "  Preparing metadata (setup.py): started\n",
      "  Preparing metadata (setup.py): finished with status 'done'\n",
      "Building wheels for collected packages: jieba\n",
      "  Building wheel for jieba (setup.py): started\n",
      "  Building wheel for jieba (setup.py): finished with status 'done'\n",
      "  Created wheel for jieba: filename=jieba-0.42.1-py3-none-any.whl size=19314478 sha256=2c7c91b90d8af6f5a2c48194df08d06f201c3bc56ad74947af10d6377ff9d560\n",
      "  Stored in directory: /root/.cache/pip/wheels/ca/38/d8/dfdfe73bec1d12026b30cb7ce8da650f3f0ea2cf155ea018ae\n",
      "Successfully built jieba\n",
      "Installing collected packages: jieba\n",
      "Successfully installed jieba-0.42.1\n",
      "\u001b[91mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\n",
      "\u001b[0mRemoving intermediate container a34017f772a9\n",
      " ---> c706a99b88f3\n",
      "Step 8/13 : RUN python3 -m pip install tqdm\n",
      " ---> Running in 64918bd242b6\n",
      "Requirement already satisfied: tqdm in /usr/local/lib/python3.8/dist-packages (4.65.0)\n",
      "\u001b[91mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\n",
      "\u001b[0mRemoving intermediate container 64918bd242b6\n",
      " ---> d9a0ba2ecca3\n",
      "Step 9/13 : RUN python3 -m pip install tensorboard\n",
      " ---> Running in b74721bc19a1\n",
      "Collecting tensorboard\n",
      "  Downloading tensorboard-2.13.0-py3-none-any.whl (5.6 MB)\n",
      "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 5.6/5.6 MB 65.5 MB/s eta 0:00:00\n",
      "Collecting absl-py>=0.4 (from tensorboard)\n",
      "  Downloading absl_py-1.4.0-py3-none-any.whl (126 kB)\n",
      "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 126.5/126.5 kB 26.1 MB/s eta 0:00:00\n",
      "Collecting grpcio>=1.48.2 (from tensorboard)\n",
      "  Downloading grpcio-1.54.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (5.1 MB)\n",
      "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 5.1/5.1 MB 110.7 MB/s eta 0:00:00\n",
      "Collecting google-auth<3,>=1.6.3 (from tensorboard)\n",
      "  Downloading google_auth-2.19.0-py2.py3-none-any.whl (181 kB)\n",
      "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 181.3/181.3 kB 32.8 MB/s eta 0:00:00\n",
      "Collecting google-auth-oauthlib<1.1,>=0.5 (from tensorboard)\n",
      "  Downloading google_auth_oauthlib-1.0.0-py2.py3-none-any.whl (18 kB)\n",
      "Collecting markdown>=2.6.8 (from tensorboard)\n",
      "  Downloading Markdown-3.4.3-py3-none-any.whl (93 kB)\n",
      "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 93.9/93.9 kB 22.9 MB/s eta 0:00:00\n",
      "Requirement already satisfied: numpy>=1.12.0 in /usr/local/lib/python3.8/dist-packages (from tensorboard) (1.24.3)\n",
      "Collecting protobuf>=3.19.6 (from tensorboard)\n",
      "  Downloading protobuf-4.23.2-cp37-abi3-manylinux2014_x86_64.whl (304 kB)\n",
      "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 304.5/304.5 kB 46.6 MB/s eta 0:00:00\n",
      "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.8/dist-packages (from tensorboard) (2.29.0)\n",
      "Requirement already satisfied: setuptools>=41.0.0 in /usr/lib/python3/dist-packages (from tensorboard) (45.2.0)\n",
      "Collecting tensorboard-data-server<0.8.0,>=0.7.0 (from tensorboard)\n",
      "  Downloading tensorboard_data_server-0.7.0-py3-none-manylinux2014_x86_64.whl (6.6 MB)\n",
      "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 6.6/6.6 MB 110.4 MB/s eta 0:00:00\n",
      "Collecting werkzeug>=1.0.1 (from tensorboard)\n",
      "  Downloading Werkzeug-2.3.4-py3-none-any.whl (242 kB)\n",
      "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 242.5/242.5 kB 43.3 MB/s eta 0:00:00\n",
      "Requirement already satisfied: wheel>=0.26 in /usr/lib/python3/dist-packages (from tensorboard) (0.34.2)\n",
      "Collecting cachetools<6.0,>=2.0.0 (from google-auth<3,>=1.6.3->tensorboard)\n",
      "  Downloading cachetools-5.3.1-py3-none-any.whl (9.3 kB)\n",
      "Collecting pyasn1-modules>=0.2.1 (from google-auth<3,>=1.6.3->tensorboard)\n",
      "  Downloading pyasn1_modules-0.3.0-py2.py3-none-any.whl (181 kB)\n",
      "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 181.3/181.3 kB 36.0 MB/s eta 0:00:00\n",
      "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.8/dist-packages (from google-auth<3,>=1.6.3->tensorboard) (4.7.2)\n",
      "Requirement already satisfied: six>=1.9.0 in /usr/local/lib/python3.8/dist-packages (from google-auth<3,>=1.6.3->tensorboard) (1.16.0)\n",
      "Requirement already satisfied: urllib3<2.0 in /usr/local/lib/python3.8/dist-packages (from google-auth<3,>=1.6.3->tensorboard) (1.26.15)\n",
      "Collecting requests-oauthlib>=0.7.0 (from google-auth-oauthlib<1.1,>=0.5->tensorboard)\n",
      "  Downloading requests_oauthlib-1.3.1-py2.py3-none-any.whl (23 kB)\n",
      "Requirement already satisfied: importlib-metadata>=4.4 in /usr/local/lib/python3.8/dist-packages (from markdown>=2.6.8->tensorboard) (6.6.0)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.8/dist-packages (from requests<3,>=2.21.0->tensorboard) (3.1.0)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.8/dist-packages (from requests<3,>=2.21.0->tensorboard) (3.4)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.8/dist-packages (from requests<3,>=2.21.0->tensorboard) (2022.12.7)\n",
      "Requirement already satisfied: MarkupSafe>=2.1.1 in /usr/local/lib/python3.8/dist-packages (from werkzeug>=1.0.1->tensorboard) (2.1.2)\n",
      "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.8/dist-packages (from importlib-metadata>=4.4->markdown>=2.6.8->tensorboard) (3.15.0)\n",
      "Requirement already satisfied: pyasn1<0.6.0,>=0.4.6 in /usr/local/lib/python3.8/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard) (0.5.0)\n",
      "Collecting oauthlib>=3.0.0 (from requests-oauthlib>=0.7.0->google-auth-oauthlib<1.1,>=0.5->tensorboard)\n",
      "  Downloading oauthlib-3.2.2-py3-none-any.whl (151 kB)\n",
      "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 151.7/151.7 kB 31.3 MB/s eta 0:00:00\n",
      "Installing collected packages: werkzeug, tensorboard-data-server, pyasn1-modules, protobuf, oauthlib, grpcio, cachetools, absl-py, requests-oauthlib, markdown, google-auth, google-auth-oauthlib, tensorboard\n",
      "Successfully installed absl-py-1.4.0 cachetools-5.3.1 google-auth-2.19.0 google-auth-oauthlib-1.0.0 grpcio-1.54.2 markdown-3.4.3 oauthlib-3.2.2 protobuf-4.23.2 pyasn1-modules-0.3.0 requests-oauthlib-1.3.1 tensorboard-2.13.0 tensorboard-data-server-0.7.0 werkzeug-2.3.4\n",
      "\u001b[91mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\n",
      "\u001b[0mRemoving intermediate container b74721bc19a1\n",
      " ---> d3eeed92932c\n",
      "Step 10/13 : RUN python3 -m pip install numpy>=1.21.0\n",
      " ---> Running in 5650a9311204\n",
      "\u001b[91mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\n",
      "\u001b[0mRemoving intermediate container 5650a9311204\n",
      " ---> 24ab997914cf\n",
      "Step 11/13 : RUN python3 -m pip install spacy\n",
      " ---> Running in d959224a02dc\n",
      "Collecting spacy\n",
      "  Downloading spacy-3.5.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (6.8 MB)\n",
      "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 6.8/6.8 MB 70.5 MB/s eta 0:00:00\n",
      "Collecting spacy-legacy<3.1.0,>=3.0.11 (from spacy)\n",
      "  Downloading spacy_legacy-3.0.12-py2.py3-none-any.whl (29 kB)\n",
      "Collecting spacy-loggers<2.0.0,>=1.0.0 (from spacy)\n",
      "  Downloading spacy_loggers-1.0.4-py3-none-any.whl (11 kB)\n",
      "Collecting murmurhash<1.1.0,>=0.28.0 (from spacy)\n",
      "  Downloading murmurhash-1.0.9-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (21 kB)\n",
      "Collecting cymem<2.1.0,>=2.0.2 (from spacy)\n",
      "  Downloading cymem-2.0.7-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (36 kB)\n",
      "Collecting preshed<3.1.0,>=3.0.2 (from spacy)\n",
      "  Downloading preshed-3.0.8-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (130 kB)\n",
      "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 130.8/130.8 kB 28.9 MB/s eta 0:00:00\n",
      "Collecting thinc<8.2.0,>=8.1.8 (from spacy)\n",
      "  Downloading thinc-8.1.10-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (928 kB)\n",
      "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 928.2/928.2 kB 92.9 MB/s eta 0:00:00\n",
      "Collecting wasabi<1.2.0,>=0.9.1 (from spacy)\n",
      "  Downloading wasabi-1.1.1-py3-none-any.whl (27 kB)\n",
      "Collecting srsly<3.0.0,>=2.4.3 (from spacy)\n",
      "  Downloading srsly-2.4.6-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (493 kB)\n",
      "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 493.5/493.5 kB 69.4 MB/s eta 0:00:00\n",
      "Collecting catalogue<2.1.0,>=2.0.6 (from spacy)\n",
      "  Downloading catalogue-2.0.8-py3-none-any.whl (17 kB)\n",
      "Collecting typer<0.8.0,>=0.3.0 (from spacy)\n",
      "  Downloading typer-0.7.0-py3-none-any.whl (38 kB)\n",
      "Collecting pathy>=0.10.0 (from spacy)\n",
      "  Downloading pathy-0.10.1-py3-none-any.whl (48 kB)\n",
      "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 48.9/48.9 kB 11.4 MB/s eta 0:00:00\n",
      "Collecting smart-open<7.0.0,>=5.2.1 (from spacy)\n",
      "  Downloading smart_open-6.3.0-py3-none-any.whl (56 kB)\n",
      "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 56.8/56.8 kB 12.6 MB/s eta 0:00:00\n",
      "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.8/dist-packages (from spacy) (4.65.0)\n",
      "Requirement already satisfied: numpy>=1.15.0 in /usr/local/lib/python3.8/dist-packages (from spacy) (1.24.3)\n",
      "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.8/dist-packages (from spacy) (2.29.0)\n",
      "Requirement already satisfied: pydantic!=1.8,!=1.8.1,<1.11.0,>=1.7.4 in /usr/local/lib/python3.8/dist-packages (from spacy) (1.10.7)\n",
      "Requirement already satisfied: jinja2 in /usr/local/lib/python3.8/dist-packages (from spacy) (3.1.2)\n",
      "Requirement already satisfied: setuptools in /usr/lib/python3/dist-packages (from spacy) (45.2.0)\n",
      "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.8/dist-packages (from spacy) (23.1)\n",
      "Collecting langcodes<4.0.0,>=3.2.0 (from spacy)\n",
      "  Downloading langcodes-3.3.0-py3-none-any.whl (181 kB)\n",
      "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 181.6/181.6 kB 40.3 MB/s eta 0:00:00\n",
      "Requirement already satisfied: typing-extensions>=4.2.0 in /usr/local/lib/python3.8/dist-packages (from pydantic!=1.8,!=1.8.1,<1.11.0,>=1.7.4->spacy) (4.5.0)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.8/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (3.1.0)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.8/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (3.4)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.8/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (1.26.15)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.8/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (2022.12.7)\n",
      "Collecting blis<0.8.0,>=0.7.8 (from thinc<8.2.0,>=8.1.8->spacy)\n",
      "  Downloading blis-0.7.9-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (10.2 MB)\n",
      "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 10.2/10.2 MB 106.7 MB/s eta 0:00:00\n",
      "Collecting confection<1.0.0,>=0.0.1 (from thinc<8.2.0,>=8.1.8->spacy)\n",
      "  Downloading confection-0.0.4-py3-none-any.whl (32 kB)\n",
      "Collecting click<9.0.0,>=7.1.1 (from typer<0.8.0,>=0.3.0->spacy)\n",
      "  Downloading click-8.1.3-py3-none-any.whl (96 kB)\n",
      "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 96.6/96.6 kB 21.7 MB/s eta 0:00:00\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.8/dist-packages (from jinja2->spacy) (2.1.2)\n",
      "Installing collected packages: cymem, wasabi, spacy-loggers, spacy-legacy, smart-open, murmurhash, langcodes, click, catalogue, blis, typer, srsly, preshed, pathy, confection, thinc, spacy\n",
      "Successfully installed blis-0.7.9 catalogue-2.0.8 click-8.1.3 confection-0.0.4 cymem-2.0.7 langcodes-3.3.0 murmurhash-1.0.9 pathy-0.10.1 preshed-3.0.8 smart-open-6.3.0 spacy-3.5.3 spacy-legacy-3.0.12 spacy-loggers-1.0.4 srsly-2.4.6 thinc-8.1.10 typer-0.7.0 wasabi-1.1.1\n",
      "\u001b[91mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\n",
      "\u001b[0mRemoving intermediate container d959224a02dc\n",
      " ---> 1dfdaeaef713\n",
      "Step 12/13 : RUN python3 -m pip install opendelta\n",
      " ---> Running in 2cb4237de0cc\n",
      "Collecting opendelta\n",
      "  Downloading opendelta-0.3.2-py3-none-any.whl (88 kB)\n",
      "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 88.1/88.1 kB 6.6 MB/s eta 0:00:00\n",
      "Requirement already satisfied: torch>=1.8.0 in /usr/local/lib/python3.8/dist-packages (from opendelta) (1.13.1+cu117)\n",
      "Requirement already satisfied: transformers>=4.10.0 in /usr/local/lib/python3.8/dist-packages (from opendelta) (4.28.0)\n",
      "Collecting datasets>=1.17.0 (from opendelta)\n",
      "  Downloading datasets-2.12.0-py3-none-any.whl (474 kB)\n",
      "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 474.6/474.6 kB 26.5 MB/s eta 0:00:00\n",
      "Requirement already satisfied: sentencepiece>=0.1.96 in /usr/local/lib/python3.8/dist-packages (from opendelta) (0.1.98)\n",
      "Requirement already satisfied: tqdm>=4.62.2 in /usr/local/lib/python3.8/dist-packages (from opendelta) (4.65.0)\n",
      "Collecting decorator (from opendelta)\n",
      "  Downloading decorator-5.1.1-py3-none-any.whl (9.1 kB)\n",
      "Collecting rich (from opendelta)\n",
      "  Downloading rich-13.3.5-py3-none-any.whl (238 kB)\n",
      "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 238.7/238.7 kB 45.2 MB/s eta 0:00:00\n",
      "Collecting web.py (from opendelta)\n",
      "  Downloading web.py-0.62.tar.gz (623 kB)\n",
      "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 623.2/623.2 kB 61.5 MB/s eta 0:00:00\n",
      "  Installing build dependencies: started\n",
      "  Installing build dependencies: finished with status 'done'\n",
      "  Getting requirements to build wheel: started\n",
      "  Getting requirements to build wheel: finished with status 'done'\n",
      "  Preparing metadata (pyproject.toml): started\n",
      "  Preparing metadata (pyproject.toml): finished with status 'done'\n",
      "Collecting gitpython (from opendelta)\n",
      "  Downloading GitPython-3.1.31-py3-none-any.whl (184 kB)\n",
      "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 184.3/184.3 kB 33.1 MB/s eta 0:00:00\n",
      "Collecting scipy (from opendelta)\n",
      "  Downloading scipy-1.10.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (34.5 MB)\n",
      "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 34.5/34.5 MB 61.8 MB/s eta 0:00:00\n",
      "Collecting sklearn (from opendelta)\n",
      "  Downloading sklearn-0.0.post5.tar.gz (3.7 kB)\n",
      "  Preparing metadata (setup.py): started\n",
      "  Preparing metadata (setup.py): finished with status 'done'\n",
      "Collecting delta-center-client==0.0.4 (from opendelta)\n",
      "  Downloading delta_center_client-0.0.4-py3-none-any.whl (8.5 kB)\n",
      "Collecting oss2==2.15.0 (from delta-center-client==0.0.4->opendelta)\n",
      "  Downloading oss2-2.15.0.tar.gz (226 kB)\n",
      "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 226.4/226.4 kB 39.2 MB/s eta 0:00:00\n",
      "  Preparing metadata (setup.py): started\n",
      "  Preparing metadata (setup.py): finished with status 'done'\n",
      "Collecting yacs>=0.1.6 (from delta-center-client==0.0.4->opendelta)\n",
      "  Downloading yacs-0.1.8-py3-none-any.whl (14 kB)\n",
      "Collecting aliyun-python-sdk-core>=2.13.12 (from oss2==2.15.0->delta-center-client==0.0.4->opendelta)\n",
      "  Downloading aliyun-python-sdk-core-2.13.36.tar.gz (440 kB)\n",
      "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 440.5/440.5 kB 56.3 MB/s eta 0:00:00\n",
      "  Preparing metadata (setup.py): started\n",
      "  Preparing metadata (setup.py): finished with status 'done'\n",
      "Collecting aliyun-python-sdk-kms>=2.4.1 (from oss2==2.15.0->delta-center-client==0.0.4->opendelta)\n",
      "  Downloading aliyun_python_sdk_kms-2.16.1-py2.py3-none-any.whl (70 kB)\n",
      "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 70.8/70.8 kB 16.7 MB/s eta 0:00:00\n",
      "Collecting crcmod>=1.7 (from oss2==2.15.0->delta-center-client==0.0.4->opendelta)\n",
      "  Downloading crcmod-1.7.tar.gz (89 kB)\n",
      "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 89.7/89.7 kB 22.4 MB/s eta 0:00:00\n",
      "  Preparing metadata (setup.py): started\n",
      "  Preparing metadata (setup.py): finished with status 'done'\n",
      "Collecting pycryptodome>=3.4.7 (from oss2==2.15.0->delta-center-client==0.0.4->opendelta)\n",
      "  Downloading pycryptodome-3.18.0-cp35-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.1 MB)\n",
      "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 2.1/2.1 MB 70.0 MB/s eta 0:00:00\n",
      "Requirement already satisfied: requests!=2.9.0 in /usr/local/lib/python3.8/dist-packages (from oss2==2.15.0->delta-center-client==0.0.4->opendelta) (2.29.0)\n",
      "Requirement already satisfied: six in /usr/local/lib/python3.8/dist-packages (from oss2==2.15.0->delta-center-client==0.0.4->opendelta) (1.16.0)\n",
      "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.8/dist-packages (from datasets>=1.17.0->opendelta) (1.24.3)\n",
      "Collecting pyarrow>=8.0.0 (from datasets>=1.17.0->opendelta)\n",
      "  Downloading pyarrow-12.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (39.0 MB)\n",
      "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 39.0/39.0 MB 58.7 MB/s eta 0:00:00\n",
      "Collecting dill<0.3.7,>=0.3.0 (from datasets>=1.17.0->opendelta)\n",
      "  Downloading dill-0.3.6-py3-none-any.whl (110 kB)\n",
      "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 110.5/110.5 kB 23.3 MB/s eta 0:00:00\n",
      "Collecting pandas (from datasets>=1.17.0->opendelta)\n",
      "  Downloading pandas-2.0.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (12.3 MB)\n",
      "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 12.3/12.3 MB 113.3 MB/s eta 0:00:00\n",
      "Collecting xxhash (from datasets>=1.17.0->opendelta)\n",
      "  Downloading xxhash-3.2.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (213 kB)\n",
      "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 213.0/213.0 kB 39.3 MB/s eta 0:00:00\n",
      "Collecting multiprocess (from datasets>=1.17.0->opendelta)\n",
      "  Downloading multiprocess-0.70.14-py38-none-any.whl (132 kB)\n",
      "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 132.0/132.0 kB 29.9 MB/s eta 0:00:00\n",
      "Requirement already satisfied: fsspec[http]>=2021.11.1 in /usr/local/lib/python3.8/dist-packages (from datasets>=1.17.0->opendelta) (2023.4.0)\n",
      "Collecting aiohttp (from datasets>=1.17.0->opendelta)\n",
      "  Downloading aiohttp-3.8.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.0 MB)\n",
      "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.0/1.0 MB 90.0 MB/s eta 0:00:00\n",
      "Requirement already satisfied: huggingface-hub<1.0.0,>=0.11.0 in /usr/local/lib/python3.8/dist-packages (from datasets>=1.17.0->opendelta) (0.14.1)\n",
      "Requirement already satisfied: packaging in /usr/local/lib/python3.8/dist-packages (from datasets>=1.17.0->opendelta) (23.1)\n",
      "Collecting responses<0.19 (from datasets>=1.17.0->opendelta)\n",
      "  Downloading responses-0.18.0-py3-none-any.whl (38 kB)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.8/dist-packages (from datasets>=1.17.0->opendelta) (5.4.1)\n",
      "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.8/dist-packages (from torch>=1.8.0->opendelta) (4.5.0)\n",
      "Requirement already satisfied: filelock in /usr/local/lib/python3.8/dist-packages (from transformers>=4.10.0->opendelta) (3.12.0)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.8/dist-packages (from transformers>=4.10.0->opendelta) (2023.3.23)\n",
      "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /usr/local/lib/python3.8/dist-packages (from transformers>=4.10.0->opendelta) (0.13.3)\n",
      "Collecting gitdb<5,>=4.0.1 (from gitpython->opendelta)\n",
      "  Downloading gitdb-4.0.10-py3-none-any.whl (62 kB)\n",
      "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 62.7/62.7 kB 15.0 MB/s eta 0:00:00\n",
      "Collecting markdown-it-py<3.0.0,>=2.2.0 (from rich->opendelta)\n",
      "  Downloading markdown_it_py-2.2.0-py3-none-any.whl (84 kB)\n",
      "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 84.5/84.5 kB 18.1 MB/s eta 0:00:00\n",
      "Collecting pygments<3.0.0,>=2.13.0 (from rich->opendelta)\n",
      "  Downloading Pygments-2.15.1-py3-none-any.whl (1.1 MB)\n",
      "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.1/1.1 MB 91.4 MB/s eta 0:00:00\n",
      "Collecting cheroot (from web.py->opendelta)\n",
      "  Downloading cheroot-10.0.0-py3-none-any.whl (101 kB)\n",
      "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 101.6/101.6 kB 25.3 MB/s eta 0:00:00\n",
      "Collecting attrs>=17.3.0 (from aiohttp->datasets>=1.17.0->opendelta)\n",
      "  Downloading attrs-23.1.0-py3-none-any.whl (61 kB)\n",
      "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 61.2/61.2 kB 14.2 MB/s eta 0:00:00\n",
      "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets>=1.17.0->opendelta) (3.1.0)\n",
      "Collecting multidict<7.0,>=4.5 (from aiohttp->datasets>=1.17.0->opendelta)\n",
      "  Downloading multidict-6.0.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (121 kB)\n",
      "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 121.3/121.3 kB 24.0 MB/s eta 0:00:00\n",
      "Collecting async-timeout<5.0,>=4.0.0a3 (from aiohttp->datasets>=1.17.0->opendelta)\n",
      "  Downloading async_timeout-4.0.2-py3-none-any.whl (5.8 kB)\n",
      "Collecting yarl<2.0,>=1.0 (from aiohttp->datasets>=1.17.0->opendelta)\n",
      "  Downloading yarl-1.9.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (266 kB)\n",
      "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 266.9/266.9 kB 41.5 MB/s eta 0:00:00\n",
      "Collecting frozenlist>=1.1.1 (from aiohttp->datasets>=1.17.0->opendelta)\n",
      "  Downloading frozenlist-1.3.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (161 kB)\n",
      "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 161.3/161.3 kB 30.4 MB/s eta 0:00:00\n",
      "Collecting aiosignal>=1.1.2 (from aiohttp->datasets>=1.17.0->opendelta)\n",
      "  Downloading aiosignal-1.3.1-py3-none-any.whl (7.6 kB)\n",
      "Collecting smmap<6,>=3.0.1 (from gitdb<5,>=4.0.1->gitpython->opendelta)\n",
      "  Downloading smmap-5.0.0-py3-none-any.whl (24 kB)\n",
      "Collecting mdurl~=0.1 (from markdown-it-py<3.0.0,>=2.2.0->rich->opendelta)\n",
      "  Downloading mdurl-0.1.2-py3-none-any.whl (10.0 kB)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.8/dist-packages (from requests!=2.9.0->oss2==2.15.0->delta-center-client==0.0.4->opendelta) (3.4)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.8/dist-packages (from requests!=2.9.0->oss2==2.15.0->delta-center-client==0.0.4->opendelta) (1.26.15)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.8/dist-packages (from requests!=2.9.0->oss2==2.15.0->delta-center-client==0.0.4->opendelta) (2022.12.7)\n",
      "Collecting more-itertools>=2.6 (from cheroot->web.py->opendelta)\n",
      "  Downloading more_itertools-9.1.0-py3-none-any.whl (54 kB)\n",
      "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 54.2/54.2 kB 9.8 MB/s eta 0:00:00\n",
      "Collecting jaraco.functools (from cheroot->web.py->opendelta)\n",
      "  Downloading jaraco.functools-3.7.0-py3-none-any.whl (8.1 kB)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.8/dist-packages (from pandas->datasets>=1.17.0->opendelta) (2.8.2)\n",
      "Collecting pytz>=2020.1 (from pandas->datasets>=1.17.0->opendelta)\n",
      "  Downloading pytz-2023.3-py2.py3-none-any.whl (502 kB)\n",
      "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 502.3/502.3 kB 52.7 MB/s eta 0:00:00\n",
      "Collecting tzdata>=2022.1 (from pandas->datasets>=1.17.0->opendelta)\n",
      "  Downloading tzdata-2023.3-py2.py3-none-any.whl (341 kB)\n",
      "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 341.8/341.8 kB 45.5 MB/s eta 0:00:00\n",
      "Collecting cryptography>=2.6.0 (from aliyun-python-sdk-core>=2.13.12->oss2==2.15.0->delta-center-client==0.0.4->opendelta)\n",
      "  Downloading cryptography-41.0.0-cp37-abi3-manylinux_2_28_x86_64.whl (4.3 MB)\n",
      "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 4.3/4.3 MB 123.7 MB/s eta 0:00:00\n",
      "Collecting jmespath<1.0.0,>=0.9.3 (from aliyun-python-sdk-core>=2.13.12->oss2==2.15.0->delta-center-client==0.0.4->opendelta)\n",
      "  Downloading jmespath-0.10.0-py2.py3-none-any.whl (24 kB)\n",
      "Collecting cffi>=1.12 (from cryptography>=2.6.0->aliyun-python-sdk-core>=2.13.12->oss2==2.15.0->delta-center-client==0.0.4->opendelta)\n",
      "  Downloading cffi-1.15.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (442 kB)\n",
      "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 442.7/442.7 kB 53.5 MB/s eta 0:00:00\n",
      "Collecting pycparser (from cffi>=1.12->cryptography>=2.6.0->aliyun-python-sdk-core>=2.13.12->oss2==2.15.0->delta-center-client==0.0.4->opendelta)\n",
      "  Downloading pycparser-2.21-py2.py3-none-any.whl (118 kB)\n",
      "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 118.7/118.7 kB 23.5 MB/s eta 0:00:00\n",
      "Building wheels for collected packages: oss2, sklearn, web.py, aliyun-python-sdk-core, crcmod\n",
      "  Building wheel for oss2 (setup.py): started\n",
      "  Building wheel for oss2 (setup.py): finished with status 'done'\n",
      "  Created wheel for oss2: filename=oss2-2.15.0-py3-none-any.whl size=103450 sha256=7358268c3e58d4f030cc521371a3d29e415f8de75c4144b3d7b18874b79cde51\n",
      "  Stored in directory: /root/.cache/pip/wheels/41/aa/57/f7a33bd9e7110547c9b7b24638c688b6fbcde9a9635f41020e\n",
      "  Building wheel for sklearn (setup.py): started\n",
      "  Building wheel for sklearn (setup.py): finished with status 'done'\n",
      "  Created wheel for sklearn: filename=sklearn-0.0.post5-py3-none-any.whl size=2360 sha256=a6d533c565c883ca689a4f066230bf4181827d860cb628791904bcb7f2e63ae1\n",
      "  Stored in directory: /root/.cache/pip/wheels/44/08/18/d0b86f591e929e063b3134b126c8a77b3758e527fe1a3f6fb8\n",
      "  Building wheel for web.py (pyproject.toml): started\n",
      "  Building wheel for web.py (pyproject.toml): finished with status 'done'\n",
      "  Created wheel for web.py: filename=web.py-0.62-py3-none-any.whl size=78567 sha256=c802acfe793844997b8abace31fd7fbe7823e3c029279f380c1169cde678bfba\n",
      "  Stored in directory: /root/.cache/pip/wheels/d7/b5/8c/d4b298304a41865a1814e79218f2a33963f735796bef785829\n",
      "  Building wheel for aliyun-python-sdk-core (setup.py): started\n",
      "  Building wheel for aliyun-python-sdk-core (setup.py): finished with status 'done'\n",
      "  Created wheel for aliyun-python-sdk-core: filename=aliyun_python_sdk_core-2.13.36-py3-none-any.whl size=533182 sha256=3d65aea80d9336cfe9711244bd65c4ee67054759cf5a70753a6a2d239c1df46e\n",
      "  Stored in directory: /root/.cache/pip/wheels/50/51/7d/304d0c92509363ab568526b81e53573e2e1e249ce35d492ea4\n",
      "  Building wheel for crcmod (setup.py): started\n",
      "  Building wheel for crcmod (setup.py): finished with status 'done'\n",
      "  Created wheel for crcmod: filename=crcmod-1.7-cp38-cp38-linux_x86_64.whl size=36019 sha256=35f8548dc381903337fa21c0a6b5eed022f76193ac467e08623f625bf5fa3210\n",
      "  Stored in directory: /root/.cache/pip/wheels/ca/5a/02/f3acf982a026f3319fb3e798a8dca2d48fafee7761788562e9\n",
      "Successfully built oss2 sklearn web.py aliyun-python-sdk-core crcmod\n",
      "Installing collected packages: sklearn, pytz, crcmod, yacs, xxhash, tzdata, smmap, scipy, pygments, pycryptodome, pycparser, pyarrow, multidict, more-itertools, mdurl, jmespath, frozenlist, dill, decorator, attrs, async-timeout, yarl, responses, pandas, multiprocess, markdown-it-py, jaraco.functools, gitdb, cffi, aiosignal, rich, gitpython, cryptography, cheroot, aiohttp, web.py, aliyun-python-sdk-core, datasets, aliyun-python-sdk-kms, oss2, delta-center-client, opendelta\n",
      "  Attempting uninstall: jmespath\n",
      "    Found existing installation: jmespath 1.0.1\n",
      "    Uninstalling jmespath-1.0.1:\n",
      "      Successfully uninstalled jmespath-1.0.1\n",
      "Successfully installed aiohttp-3.8.4 aiosignal-1.3.1 aliyun-python-sdk-core-2.13.36 aliyun-python-sdk-kms-2.16.1 async-timeout-4.0.2 attrs-23.1.0 cffi-1.15.1 cheroot-10.0.0 crcmod-1.7 cryptography-41.0.0 datasets-2.12.0 decorator-5.1.1 delta-center-client-0.0.4 dill-0.3.6 frozenlist-1.3.3 gitdb-4.0.10 gitpython-3.1.31 jaraco.functools-3.7.0 jmespath-0.10.0 markdown-it-py-2.2.0 mdurl-0.1.2 more-itertools-9.1.0 multidict-6.0.4 multiprocess-0.70.14 opendelta-0.3.2 oss2-2.15.0 pandas-2.0.2 pyarrow-12.0.0 pycparser-2.21 pycryptodome-3.18.0 pygments-2.15.1 pytz-2023.3 responses-0.18.0 rich-13.3.5 scipy-1.10.1 sklearn-0.0.post5 smmap-5.0.0 tzdata-2023.3 web.py-0.62 xxhash-3.2.0 yacs-0.1.8 yarl-1.9.2\n",
      "\u001b[91mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\n",
      "\u001b[0mRemoving intermediate container 2cb4237de0cc\n",
      " ---> c97405fcaa90\n",
      "Step 13/13 : ENV NVIDIA_VISIBLE_DEVICES=\"all\"\n",
      " ---> Running in cb9220d020ac\n",
      "Removing intermediate container cb9220d020ac\n",
      " ---> 820fc2db7dc4\n",
      "Successfully built 820fc2db7dc4\n",
      "Successfully tagged sagemaker-vicuna-inference-severbatch-demo:latest\n",
      "The push refers to repository [687912291502.dkr.ecr.us-west-2.amazonaws.com/sagemaker-vicuna-inference-severbatch-demo]\n",
      "e9dff84cb791: Preparing\n",
      "bbeccbd5eef7: Preparing\n",
      "a650ad30b41e: Preparing\n",
      "570c550a0ff8: Preparing\n",
      "bf60da2e0d94: Preparing\n",
      "ce421e492e43: Preparing\n",
      "aad2023a28d3: Preparing\n",
      "f84e393f37c7: Preparing\n",
      "468437145a4d: Preparing\n",
      "279984a2f8f2: Preparing\n",
      "760a6ca6c63f: Preparing\n",
      "1dd935357a36: Preparing\n",
      "5848cd63429e: Preparing\n",
      "b936706d002a: Preparing\n",
      "3b4381888a1b: Preparing\n",
      "9fbeed575c34: Preparing\n",
      "9f47ad0160c8: Preparing\n",
      "822919f77330: Preparing\n",
      "a2654928891a: Preparing\n",
      "cdedc6c127e1: Preparing\n",
      "ac85f70f8859: Preparing\n",
      "590f3ac20d28: Preparing\n",
      "c263f12cd4e6: Preparing\n",
      "aad2023a28d3: Waiting\n",
      "99832d04a153: Preparing\n",
      "279984a2f8f2: Waiting\n",
      "a5981ed7a378: Preparing\n",
      "250519a2f830: Preparing\n",
      "760a6ca6c63f: Waiting\n",
      "f84e393f37c7: Waiting\n",
      "6cadbde53f94: Preparing\n",
      "1dd935357a36: Waiting\n",
      "468437145a4d: Waiting\n",
      "0002c93bdb37: Preparing\n",
      "5848cd63429e: Waiting\n",
      "99832d04a153: Waiting\n",
      "ce421e492e43: Waiting\n",
      "cdedc6c127e1: Waiting\n",
      "6cadbde53f94: Waiting\n",
      "250519a2f830: Waiting\n",
      "b936706d002a: Waiting\n",
      "a5981ed7a378: Waiting\n",
      "ac85f70f8859: Waiting\n",
      "0002c93bdb37: Waiting\n",
      "c263f12cd4e6: Waiting\n",
      "590f3ac20d28: Waiting\n",
      "9fbeed575c34: Waiting\n",
      "a2654928891a: Waiting\n",
      "bf60da2e0d94: Pushed\n",
      "a650ad30b41e: Pushed\n",
      "aad2023a28d3: Pushed\n",
      "f84e393f37c7: Layer already exists\n",
      "468437145a4d: Layer already exists\n",
      "279984a2f8f2: Layer already exists\n",
      "760a6ca6c63f: Layer already exists\n",
      "1dd935357a36: Layer already exists\n",
      "5848cd63429e: Layer already exists\n",
      "b936706d002a: Layer already exists\n",
      "3b4381888a1b: Layer already exists\n",
      "9fbeed575c34: Layer already exists\n",
      "9f47ad0160c8: Layer already exists\n",
      "822919f77330: Layer already exists\n",
      "a2654928891a: Layer already exists\n",
      "cdedc6c127e1: Layer already exists\n",
      "ac85f70f8859: Layer already exists\n",
      "590f3ac20d28: Layer already exists\n",
      "570c550a0ff8: Pushed\n",
      "c263f12cd4e6: Layer already exists\n",
      "99832d04a153: Layer already exists\n",
      "a5981ed7a378: Layer already exists\n",
      "250519a2f830: Layer already exists\n",
      "6cadbde53f94: Layer already exists\n",
      "0002c93bdb37: Layer already exists\n",
      "bbeccbd5eef7: Pushed\n",
      "ce421e492e43: Pushed\n",
      "e9dff84cb791: Pushed\n",
      "latest: digest: sha256:66d1488e581bdb84ee922613204ed66f34fc3781739c186c97f50bd3d2c411b9 size: 6195\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING! Your password will be stored unencrypted in /home/ec2-user/.docker/config.json.\n",
      "Configure a credential helper to remove this warning. See\n",
      "https://docs.docker.com/engine/reference/commandline/login/#credentials-store\n",
      "\n"
     ]
    }
   ],
   "source": [
    "%%script env repo_name=$repo_name bash\n",
    "\n",
    "#!/usr/bin/env bash\n",
    "\n",
    "# This script shows how to build the Docker image and push it to ECR to be ready for use\n",
    "# by SageMaker.\n",
    "\n",
    "# The argument to this script is the image name. This will be used as the image on the local\n",
    "# machine and combined with the account and region to form the repository name for ECR.\n",
    "# The name of our algorithm\n",
    "algorithm_name=${repo_name}\n",
    "\n",
    "account=$(aws sts get-caller-identity --query Account --output text)\n",
    "\n",
    "# Get the region defined in the current configuration (default to us-west-2 if none defined)\n",
    "region=$(aws configure get region)\n",
    "region=${region:-us-west-2}\n",
    "\n",
    "fullname=\"${account}.dkr.ecr.${region}.amazonaws.com/${algorithm_name}:latest\"\n",
    "\n",
    "# If the repository doesn't exist in ECR, create it.\n",
    "aws ecr describe-repositories --repository-names \"${algorithm_name}\" > /dev/null 2>&1\n",
    "\n",
    "if [ $? -ne 0 ]\n",
    "then\n",
    "    aws ecr create-repository --repository-name \"${algorithm_name}\" > /dev/null\n",
    "fi\n",
    "\n",
    "# Get the login command from ECR and execute it directly\n",
    "aws ecr get-login-password --region ${region}|docker login --username AWS --password-stdin ${fullname}\n",
    "\n",
    "# Build the docker image locally with the image name and then push it to ECR\n",
    "# with the full name.\n",
    "\n",
    "docker build -t ${algorithm_name} -f Dockerfile.inference .\n",
    "docker tag ${algorithm_name} ${fullname}\n",
    "\n",
    "docker push ${fullname}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "id": "84497325",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'687912291502.dkr.ecr.us-west-2.amazonaws.com/sagemaker-vicuna-inference-severbatch-demo:latest'"
      ]
     },
     "execution_count": 217,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## The image uri which is build and pushed above\n",
    "inference_image_uri = \"{}.dkr.ecr.{}.amazonaws.com/{}:latest\".format(account_id, region, repo_name)\n",
    "inference_image_uri"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "01e33506",
   "metadata": {},
   "source": [
    "## Deploying a Large Language Model using deepspeed engine\n",
    "The DJL Inference Image which we will be utilizing ships with a number of built-in inference handlers for a wide variety of tasks including:\n",
    "- `text-generation`\n",
    "- `question-answering`\n",
    "- `text-classification`\n",
    "- `token-classification`\n",
    "\n",
    "You can refer to this [GitRepo](https://github.com/deepjavalibrary/djl-serving/tree/master/engines/python/setup/djl_python) for a list of additional handlers and available NLP Tasks. <br>\n",
    "These handlers can be utilized as is without having to write any custom inference code. We simply need to create a `serving.properties` text file with our desired hosting options and package it up into a `tar.gz` artifact.\n",
    "\n",
    "Lets take a look at the `serving.properties` file that we'll be using for our first example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "id": "e3570119",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mkdir: cannot create directory ‘deepspeed_src’: File exists\n"
     ]
    }
   ],
   "source": [
    "!mkdir deepspeed_src"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "id": "33c253cd",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting deepspeed_src/serving.template\n"
     ]
    }
   ],
   "source": [
    "%%writefile deepspeed_src/serving.template\n",
    "engine=DeepSpeed\n",
    "option.entryPoint=model.py\n",
    "option.tensor_parallel_degree=1\n",
    "option.model_id=eachadea/vicuna-7b-1.1\n",
    "batch_size=2\n",
    "max_batch_delay=5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "id": "d1746e87-ef56-414d-a1ea-b57db076d7bb",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting deepspeed_src/model.py\n"
     ]
    }
   ],
   "source": [
    "%%writefile deepspeed_src/model.py\n",
    "from djl_python import Input, Output\n",
    "import os\n",
    "import logging\n",
    "import math\n",
    "import deepspeed\n",
    "import torch\n",
    "import torch.distributed as dist\n",
    "import sys\n",
    "import subprocess\n",
    "import time\n",
    "import transformers\n",
    "from transformers import pipeline, AutoModelForCausalLM, AutoTokenizer\n",
    "from transformers.models.llama.tokenization_llama import LlamaTokenizer\n",
    "\n",
    "\n",
    "print(\"transformers version==\"+transformers.__version__)\n",
    "os.system(\"df -h\")\n",
    "import bmtrain as bmt\n",
    "print(\"here1====\")\n",
    "print(type(bmt))\n",
    "predictor = None\n",
    "\n",
    "#for deepspeed engine\n",
    "\n",
    "def load_model(properties):\n",
    "    tensor_parallel = properties[\"tensor_parallel_degree\"]\n",
    "    model_location = properties['model_dir']\n",
    "    if \"model_id\" in properties:\n",
    "        model_location = properties['model_id']\n",
    "    #logging.info(f\"Loading model in {model_location}\")\n",
    "    \n",
    "    print(\"----------tensor parallel is {0}---------\".format(tensor_parallel))\n",
    "    #model_location=\"decapoda-research/llama-7b-hf\"\n",
    "    logging.info(f\"Loading model in {model_location}\")\n",
    "\n",
    "    model = AutoModelForCausalLM.from_pretrained(model_location, torch_dtype=torch.float16)\n",
    "    #tokenizer = LlamaTokenizer.from_pretrained(model_location)\n",
    "    tokenizer = LlamaTokenizer.from_pretrained(model_location, torch_dtype=torch.float16)\n",
    "\n",
    "    print(\"----------model dtype is {0}---------\".format(model.dtype))\n",
    "    model = deepspeed.init_inference(\n",
    "        model,\n",
    "        mp_size=tensor_parallel,\n",
    "        #dtype=model.dtype,\n",
    "        dtype=torch.half,\n",
    "        #dtype=torch.int8,\n",
    "        replace_method=\"auto\",\n",
    "        replace_with_kernel_inject=True,\n",
    "        #replace_method=\"auto\",\n",
    "        #replace_with_kernel_inject=True,\n",
    "    )\n",
    "    \n",
    "    local_rank = int(os.getenv(\"LOCAL_RANK\", \"0\"))\n",
    "    generator = pipeline(\n",
    "        task=\"text-generation\", model=model, tokenizer=tokenizer, device=local_rank\n",
    "    )\n",
    "    return generator, model, tokenizer\n",
    "\n",
    "'''\n",
    "def load_model(properties):\n",
    "    tensor_parallel = properties[\"tensor_parallel_degree\"]\n",
    "    model_location = properties['model_dir']\n",
    "    if \"model_id\" in properties:\n",
    "        model_location = properties['model_id']\n",
    "    #logging.info(f\"Loading model in {model_location}\")\n",
    "    \n",
    "    print(\"----------tensor parallel is {0}---------\".format(tensor_parallel))\n",
    "    #model_location=\"decapoda-research/llama-7b-hf\"\n",
    "    logging.info(f\"Loading model in {model_location}\")\n",
    "\n",
    "    model = AutoModelForCausalLM.from_pretrained(model_location, torch_dtype=torch.float16, device_map=\"balanced_low_0\")\n",
    "    #tokenizer = LlamaTokenizer.from_pretrained(model_location)\n",
    "    tokenizer = LlamaTokenizer.from_pretrained(model_location, torch_dtype=torch.float16)\n",
    "\n",
    "    print(\"----------model dtype is {0}---------\".format(model.dtype))\n",
    "    generator = pipeline(\n",
    "        task=\"text-generation\", model=model, tokenizer=tokenizer,\n",
    "    )\n",
    "    return generator, model, tokenizer\n",
    "'''\n",
    "\n",
    "def handle(inputs: Input) -> None:\n",
    "    global predictor, model, tokenizer\n",
    "    try:\n",
    "        if not predictor:\n",
    "            predictor,model,tokenizer = load_model(inputs.get_properties())\n",
    "\n",
    "        print(inputs)\n",
    "        if inputs.is_empty():\n",
    "            # Model server makes an empty call to warmup the model on startup\n",
    "            return None\n",
    "        \n",
    "        if inputs.is_batch():\n",
    "            print(f\"Dynamic batching size: {inputs.get_batch_size()}.\")\n",
    "            batch = inputs.get_batches()\n",
    "            print(\"batch_size==\"+str(batch))\n",
    "            tmp_inputs = []\n",
    "            params = {}\n",
    "            for _, item in enumerate(batch):\n",
    "                tmp_item = item.get_as_json()\n",
    "                tmp_inputs.append(tmp_item.get(\"input\"))\n",
    "                ##use the unified parameters for batch prediction###\n",
    "                params = tmp_item.get(\"parameters\",{})\n",
    "            \n",
    "            result = predictor(tmp_inputs,**params)\n",
    "            outputs = Output()\n",
    "            for i in range(len(result)):\n",
    "                outputs.add(result[i], key=\"generate_text\", batch_index=i)\n",
    "            return outputs\n",
    "        else:\n",
    "            inputs = inputs.get_as_json()\n",
    "            if not inputs.get(\"input\"):\n",
    "                return Output().add_as_json({\"code\":-1,\"msg\":\"input field can't be null\"})\n",
    "\n",
    "\n",
    "            #input data\n",
    "            data = inputs.get(\"input\")\n",
    "            params = inputs.get(\"parameters\",{})\n",
    "\n",
    "            #predictor\n",
    "            result = predictor(data, **params)\n",
    "\n",
    "            #return\n",
    "            return Output().add({\"code\":0,\"msg\":\"ok\",\"data\":result})\n",
    "    except Exception as e:\n",
    "        return Output().add_as_json({\"code\":-1,\"msg\":e})\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "id": "87923cef-0632-4d8a-b792-4627bf3a0203",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting deepspeed_src/requirements.txt\n"
     ]
    }
   ],
   "source": [
    "%%writefile deepspeed_src/requirements.txt\n",
    "protobuf==3.20\n",
    "#accelerate=>0.17.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "id": "1bbdefb3",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     1\t\u001b[36mengine\u001b[39;49;00m=\u001b[33mPython\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
      "     2\t\u001b[36moption.entryPoint\u001b[39;49;00m=\u001b[33mdjl_python.huggingface\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
      "     3\t\u001b[37m#option.s3url=s3://sagemaker-us-west-2-687912291502/llama/output/2023-05-10-12-43-02/llama_out/\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
      "     4\t\u001b[37m#option.model_id=pinkmanlove/llama-7b-hf\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
      "     5\t\u001b[37m#option.model_id=eachadea/vicuna-7b-1.1\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
      "     6\t\u001b[36moption.model_id\u001b[39;49;00m=\u001b[33mTheBloke/vicuna-7B-1.1-HF\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
      "     7\t\u001b[36moption.task\u001b[39;49;00m=\u001b[33mtext-generation\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
      "     8\t\u001b[36moption.device_map\u001b[39;49;00m=\u001b[33mauto\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n",
      "     9\t\u001b[36moption.load_in_8bit\u001b[39;49;00m=\u001b[33mTRUE\u001b[39;49;00m\u001b[37m\u001b[39;49;00m\n"
     ]
    }
   ],
   "source": [
    "# we plug in the appropriate model location into our `serving.properties` file based on the region in which this notebook is running\n",
    "template = jinja_env.from_string(Path(\"deepspeed_src/serving.template\").open().read())\n",
    "Path(\"deepspeed_src/serving.properties\").open(\"w\").write(template.render(s3url=pretrained_model_location))\n",
    "!pygmentize accelerate_src/serving.properties | cat -n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e46cffa8",
   "metadata": {},
   "source": [
    "There are a few options specified here. Lets go through them in turn<br>\n",
    "1. `engine` - specifies the engine that will be used for this workload. In this case we'll be hosting a model using the [DJL Python Engine](https://github.com/deepjavalibrary/djl-serving/tree/master/engines/python)\n",
    "2. `option.entryPoint` - specifies the entrypoint code that will be used to host the model. djl_python.huggingface refers to the `huggingface.py` module from [djl_python repo](https://github.com/deepjavalibrary/djl-serving/tree/master/engines/python/setup/djl_python).  \n",
    "3. `option.s3url` - specifies the location of the model files. Alternativelly an `option.model_id` option can be used instead to specifiy a model from Hugging Face Hub (e.g. `EleutherAI/gpt-j-6B`) and the model will be automatically downloaded from the Hub. The s3url approach is recommended as it allows you to host the model artifact within your own environment and enables faster deployments by utilizing optimized approach within the DJL inference container to transfer the model from S3 into the hosting instance \n",
    "4. `option.task` - This is specific to the `huggingface.py` inference handler and specifies for which task this model will be used\n",
    "5. `option.device_map` - Enables layer-wise model partitioning through [Hugging Face Accelerate](https://huggingface.co/docs/accelerate/usage_guides/big_modeling#designing-a-device-map). With `option.device_map=auto`, Accelerate will determine where to put each **layer** to maximize the use of your fastest devices (GPUs) and offload the rest on the CPU, or even the hard drive if you don’t have enough GPU RAM (or CPU RAM). Even if the model is split across several devices, it will run as you would normally expect.\n",
    "6. `option.load_in_8bit` - Quantizes the model weights to int8 thereby greatly reducing the memory footprint of the model from the initial FP32. See this [blog post](https://huggingface.co/blog/hf-bitsandbytes-integration) from Hugging Face for additional information \n",
    "\n",
    "For more information on the available options, please refer to the [SageMaker Large Model Inference Documentation](https://docs.aws.amazon.com/sagemaker/latest/dg/realtime-endpoints-large-model-configuration.html)\n",
    "\n",
    "Our initial approach here is to utilize the built-in functionality within Hugging Face Transformers to enable Large Language Model hosting. "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d156470a",
   "metadata": {},
   "source": [
    "We place the `serving.properties` file into a tarball and upload it to S3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "id": "12371518",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "deepspeed_src/\n",
      "deepspeed_src/.ipynb_checkpoints/\n",
      "deepspeed_src/.ipynb_checkpoints/model-checkpoint.py\n",
      "deepspeed_src/.ipynb_checkpoints/requirements-checkpoint.txt\n",
      "deepspeed_src/.ipynb_checkpoints/serving-checkpoint.properties\n",
      "deepspeed_src/model.py\n",
      "deepspeed_src/serving.template\n",
      "deepspeed_src/serving.properties\n",
      "deepspeed_src/requirements.txt\n"
     ]
    }
   ],
   "source": [
    "!tar czvf acc_model.tar.gz deepspeed_src/ "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "id": "3098668f",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "S3 Code or Model tar ball uploaded to --- > s3://sagemaker-us-west-2-687912291502/llama/deploy/code/acc_model.tar.gz\n"
     ]
    }
   ],
   "source": [
    "s3_code_prefix = \"llama/deploy/code\"\n",
    "\n",
    "code_artifact = sess.upload_data(\"acc_model.tar.gz\", bucket, s3_code_prefix)\n",
    "print(f\"S3 Code or Model tar ball uploaded to --- > {code_artifact}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e0807c58",
   "metadata": {},
   "source": [
    "## Deploy Model to a SageMaker Endpoint\n",
    "With a helper function we can now deploy our endpoint and invoke it with some sample inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "id": "ccb25eff-af72-4e51-8d83-9190c2d80279",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "arn:aws:sagemaker:us-west-2:687912291502:model/vicuna-7B-2023-05-31-11-57-19-752\n",
      "{'EndpointConfigArn': 'arn:aws:sagemaker:us-west-2:687912291502:endpoint-config/vicuna-7B-2023-05-31-11-57-19-752-config', 'ResponseMetadata': {'RequestId': 'e83c5062-fb70-470c-9112-19695b5041a8', 'HTTPStatusCode': 200, 'HTTPHeaders': {'x-amzn-requestid': 'e83c5062-fb70-470c-9112-19695b5041a8', 'content-type': 'application/x-amz-json-1.1', 'content-length': '121', 'date': 'Wed, 31 May 2023 11:57:19 GMT'}, 'RetryAttempts': 0}}\n",
      "{'EndpointArn': 'arn:aws:sagemaker:us-west-2:687912291502:endpoint/vicuna-7B-2023-05-31-11-57-19-752-endpoint', 'ResponseMetadata': {'RequestId': '103f1828-3f2b-400c-818a-a0dfe9403104', 'HTTPStatusCode': 200, 'HTTPHeaders': {'x-amzn-requestid': '103f1828-3f2b-400c-818a-a0dfe9403104', 'content-type': 'application/x-amz-json-1.1', 'content-length': '110', 'date': 'Wed, 31 May 2023 11:57:20 GMT'}, 'RetryAttempts': 0}}\n"
     ]
    }
   ],
   "source": [
    "from sagemaker.utils import name_from_base\n",
    "\n",
    "model_name = name_from_base(f\"vicuna-7B\")\n",
    "\n",
    "create_model_response = sm_client.create_model(\n",
    "    ModelName=model_name,\n",
    "    ExecutionRoleArn=role,\n",
    "    PrimaryContainer={\n",
    "        \"Image\": inference_image_uri,\n",
    "        \"ModelDataUrl\": code_artifact\n",
    "    }\n",
    ")\n",
    "model_arn = create_model_response[\"ModelArn\"]\n",
    "print(model_arn)\n",
    "\n",
    "endpoint_config_name = f\"{model_name}-config\"\n",
    "endpoint_name = f\"{model_name}-endpoint\"\n",
    "\n",
    "endpoint_config_response = sm_client.create_endpoint_config(\n",
    "    EndpointConfigName=endpoint_config_name,\n",
    "    ProductionVariants=[\n",
    "        {\n",
    "            \"VariantName\": \"variant1\",\n",
    "            \"ModelName\": model_name,\n",
    "            \"InstanceType\": \"ml.g5.8xlarge\",\n",
    "            \"InitialInstanceCount\": 1,\n",
    "        }\n",
    "    ]\n",
    ")\n",
    "print(endpoint_config_response)\n",
    "\n",
    "create_endpoint_response = sm_client.create_endpoint(\n",
    "    EndpointName=f\"{endpoint_name}\", EndpointConfigName=endpoint_config_name\n",
    ")\n",
    "print(create_endpoint_response)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b3c19a0c",
   "metadata": {},
   "source": [
    "Let's run an example with a basic text generation prompt Large model inference is"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "43611bc8",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import boto3\n",
    "import json\n",
    "import uuid\n",
    "client = boto3.client('sagemaker-runtime')\n",
    "endpoint_name = 'vicuna-7B-2023-05-25-00-54-30-141-endpoint'\n",
    "payload1 = '{\"input\" : \"a happy weekend with my family\", \\\n",
    "            \"parameters\": { \"max_length\": 200, \"temperature\": 0.6 }  \\\n",
    "           }'\n",
    "\n",
    "payload2 = '{\"input\" : \"I\\'d likt to go to HongKong\", \\\n",
    "            \"parameters\": { \"max_length\": 200, \"temperature\": 0.6 }  \\\n",
    "           }'\n",
    "encoded_inp = (payload1).encode(\"utf-8\")\n",
    "response = client.invoke_endpoint(\n",
    "        EndpointName=endpoint_name,\n",
    "        ContentType='application/json',\n",
    "        Accept='application/json',\n",
    "        Body=payload1\n",
    ")\n",
    "response = client.invoke_endpoint(\n",
    "        EndpointName=endpoint_name,\n",
    "        ContentType='application/json',\n",
    "        Accept='application/json',\n",
    "        Body=payload2\n",
    ")\n",
    "\n",
    "\n",
    "#predictor.predict({ \n",
    "#                    \"inputs\" : \"large model inference is\", \n",
    "#                    \"parameters\": { \"max_length\": 100, \"temperature\": 0.8 }\n",
    "#                })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "id": "9a2cb90c-4f0c-48ff-b888-cad18ca86174",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "ename": "JSONDecodeError",
     "evalue": "Expecting value: line 1 column 1 (char 0)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mJSONDecodeError\u001b[0m                           Traceback (most recent call last)",
      "\u001b[0;32m/tmp/ipykernel_28810/1073333867.py\u001b[0m in \u001b[0;36m<cell line: 2>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'Body'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mjson\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloads\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch_p39/lib/python3.9/json/__init__.py\u001b[0m in \u001b[0;36mloads\u001b[0;34m(s, cls, object_hook, parse_float, parse_int, parse_constant, object_pairs_hook, **kw)\u001b[0m\n\u001b[1;32m    344\u001b[0m             \u001b[0mparse_int\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mparse_float\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32mand\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    345\u001b[0m             parse_constant is None and object_pairs_hook is None and not kw):\n\u001b[0;32m--> 346\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0m_default_decoder\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdecode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    347\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mcls\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    348\u001b[0m         \u001b[0mcls\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mJSONDecoder\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch_p39/lib/python3.9/json/decoder.py\u001b[0m in \u001b[0;36mdecode\u001b[0;34m(self, s, _w)\u001b[0m\n\u001b[1;32m    335\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    336\u001b[0m         \"\"\"\n\u001b[0;32m--> 337\u001b[0;31m         \u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mend\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mraw_decode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0midx\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0m_w\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    338\u001b[0m         \u001b[0mend\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_w\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mend\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    339\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mend\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch_p39/lib/python3.9/json/decoder.py\u001b[0m in \u001b[0;36mraw_decode\u001b[0;34m(self, s, idx)\u001b[0m\n\u001b[1;32m    353\u001b[0m             \u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mend\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mscan_once\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0midx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    354\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mStopIteration\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0merr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 355\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mJSONDecodeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Expecting value\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0ms\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0merr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    356\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mend\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mJSONDecodeError\u001b[0m: Expecting value: line 1 column 1 (char 0)"
     ]
    }
   ],
   "source": [
    "result = response['Body'].read()\n",
    "result = json.loads(result)\n",
    "print(result)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "id": "5d5f35f7",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'predictor' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m/tmp/ipykernel_28810/3489660711.py\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m print(predictor.predict({ \"inputs\":\n\u001b[0m\u001b[1;32m      2\u001b[0m                                 \"\"\"Message: Support has been terrible for 2 weeks...\n\u001b[1;32m      3\u001b[0m                                 \u001b[0mSentiment\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mNegative\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m                                 \u001b[0;31m###\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m                                 \u001b[0mMessage\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mI\u001b[0m \u001b[0mlove\u001b[0m \u001b[0myour\u001b[0m \u001b[0mAPI\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mit\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0msimple\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mso\u001b[0m \u001b[0mfast\u001b[0m\u001b[0;31m!\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'predictor' is not defined"
     ]
    }
   ],
   "source": [
    "print(predictor.predict({ \"inputs\":\n",
    "                                \"\"\"Message: Support has been terrible for 2 weeks...\n",
    "                                Sentiment: Negative\n",
    "                                ###\n",
    "                                Message: I love your API, it is simple and so fast!\n",
    "                                Sentiment: Positive\n",
    "                                ###\n",
    "                                Message: GPT-J has been released 12 months ago.\n",
    "                                Sentiment: Neutral\n",
    "                                ###\n",
    "                                Message: The responsiveness of your team has been amazing, thank you so much!\n",
    "                                Sentiment:\"\"\",\n",
    "                      \"parameters\": { \"max_length\": 50, \"temperature\": 0.5 }\n",
    "                     }\n",
    "                    )[0][\"generated_text\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e033f0aa",
   "metadata": {},
   "source": [
    "Finally Let's do a quick benchmark to see what kind of latency we can expect from this model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "61464f7d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'generated_text': \"a happy weekend with my family.\\nI'm so glad you had a great weekend! I hope you have a great week!\\nI'm so glad you had a great weekend! I hope you have a great week! I'm so glad you had a great weekend! I hope you have a great week!\\nI'm so glad you had a great weekend! I hope you have a great week!\\nI'm so glad you had a great weekend! I hope you have a great week!\\nI'm so glad you had a great weekend! I hope you have a great week! I'm so glad you had a great weekend! I hope you have a\"}]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictor.predict({ \n",
    "                    \"inputs\" : \"a happy weekend with my family\", \n",
    "                    \"parameters\": { \"max_length\": 150, \"temperature\": 0.5 }\n",
    "                })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a2da0c7d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Clean up the endpoint before proceeding\n",
    "predictor.delete_endpoint()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7377c84c",
   "metadata": {},
   "source": [
    "## Reference"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d1a842b9",
   "metadata": {},
   "source": [
    "[sagemaker-hosting/Large-Language-Model-Hosting/](https://github.com/aws-samples/sagemaker-hosting/tree/main/Large-Language-Model-Hosting)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_pytorch_p39",
   "language": "python",
   "name": "conda_pytorch_p39"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}