Falcon_deploy.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "c261e5f4-17a8-40da-beb9-599f1717e0fe",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "### 1. 安装HuggingFace 并下载模型到本地"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "02785614-9268-41c8-85a5-d579490edbbf",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!pip install huggingface-hub -Uqq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "9e6bd7ee-16a3-4f5a-8857-8bbba83eb9e7",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from huggingface_hub import snapshot_download\n",
    "from pathlib import Path\n",
    "\n",
    "local_model_path = Path(\"./LLM_Falcon_model\")\n",
    "local_model_path.mkdir(exist_ok=True)\n",
    "model_name = \"tiiuae/falcon-40b\"\n",
    "commit_hash = \"88b92aef6921a843415b6142bd5a906197e3fb0e\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "94e8abc5-a58e-40e2-b1e6-fbf48307c716",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5a0d58517adb453ba8c84b8b6343d8f3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "62c5604e57f64450bb1c98b02ed6436b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "989c27ed8ea74919b017991b316e60f3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4a066fd5782a414abcbdf41513547950",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dd3769c8ec8f4ed388dffd86e0608e5b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b0d044ee6d2a41a68c5f97803a249fa2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f0905c781d054666b15e44ef25d7aac5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "015ead89751e40189320406c41b4f6fe",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d33b1a00a1c64ca28b5420d877439ee3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9f917f2256964e26a9a030df3812faa1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "756399aa96104769b40af3343bca3e6f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a4e3510df3dc47f2974dcccfeeb60b42",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cf3f7434a2e34740980277c5ca585f6f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b87188bb946640fda704245d7b443ef8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "85e1ed1ed27f4b73b7d6abe0dbcdec76",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6caf9dc0228d4f5a96e130f86b3cf6e3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c09727c809ce46dc80d602667ec4fb73",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "251ce311454f4dccb23578e1751e03ab",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7df8364b78c14e11907dd6cbba79674e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "94dc26613a4d455b8314f2ee2f1eafa2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'LLM_Falcon_model/models--tiiuae--falcon-40b/snapshots/88b92aef6921a843415b6142bd5a906197e3fb0e'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "snapshot_download(repo_id=model_name, revision=commit_hash,cache_dir=local_model_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8d666c79-b039-4258-ac3b-46b19e63c3b8",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "### 2. 把模型拷贝到S3为后续部署做准备"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "e9431deb-6359-442d-847b-1563f8dd3854",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import sagemaker\n",
    "from sagemaker import image_uris\n",
    "import boto3\n",
    "import os\n",
    "import time\n",
    "import json\n",
    "\n",
    "role = sagemaker.get_execution_role()  # execution role for the endpoint\n",
    "sess = sagemaker.session.Session()  # sagemaker session for interacting with different AWS APIs\n",
    "bucket = sess.default_bucket()  # bucket to house artifacts\n",
    "\n",
    "region = sess._region_name\n",
    "account_id = sess.account_id()\n",
    "\n",
    "s3_client = boto3.client(\"s3\")\n",
    "sm_client = boto3.client(\"sagemaker\")\n",
    "smr_client = boto3.client(\"sagemaker-runtime\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "40dd8f16-ae7c-48bf-8e52-1a15425fa74d",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "s3_code_prefix: LLM-RAG/workshop/LLM_falcon_deploy_code\n",
      "model_snapshot_path: LLM_Falcon_model/models--tiiuae--falcon-40b/snapshots/88b92aef6921a843415b6142bd5a906197e3fb0e\n"
     ]
    }
   ],
   "source": [
    "s3_model_prefix = \"llm/models/LLM_falcon_model\"  # folder where model checkpoint will go\n",
    "model_snapshot_path = list(local_model_path.glob(\"**/snapshots/*\"))[0]\n",
    "s3_code_prefix = \"LLM-RAG/workshop/LLM_falcon_deploy_code\"\n",
    "print(f\"s3_code_prefix: {s3_code_prefix}\")\n",
    "print(f\"model_snapshot_path: {model_snapshot_path}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "067292c9-c066-4649-a61f-b460a24da584",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "upload: LLM_Falcon_model/models--tiiuae--falcon-40b/snapshots/88b92aef6921a843415b6142bd5a906197e3fb0e/.gitattributes to s3://sagemaker-us-west-2-687912291502/llm/models/LLM_falcon_model/.gitattributes\n",
      "upload: LLM_Falcon_model/models--tiiuae--falcon-40b/snapshots/88b92aef6921a843415b6142bd5a906197e3fb0e/config.json to s3://sagemaker-us-west-2-687912291502/llm/models/LLM_falcon_model/config.json\n",
      "upload: LLM_Falcon_model/models--tiiuae--falcon-40b/snapshots/88b92aef6921a843415b6142bd5a906197e3fb0e/generation_config.json to s3://sagemaker-us-west-2-687912291502/llm/models/LLM_falcon_model/generation_config.json\n",
      "upload: LLM_Falcon_model/models--tiiuae--falcon-40b/snapshots/88b92aef6921a843415b6142bd5a906197e3fb0e/README.md to s3://sagemaker-us-west-2-687912291502/llm/models/LLM_falcon_model/README.md\n",
      "upload: LLM_Falcon_model/models--tiiuae--falcon-40b/snapshots/88b92aef6921a843415b6142bd5a906197e3fb0e/modelling_RW.py to s3://sagemaker-us-west-2-687912291502/llm/models/LLM_falcon_model/modelling_RW.py\n",
      "upload: LLM_Falcon_model/models--tiiuae--falcon-40b/snapshots/88b92aef6921a843415b6142bd5a906197e3fb0e/configuration_RW.py to s3://sagemaker-us-west-2-687912291502/llm/models/LLM_falcon_model/configuration_RW.py\n",
      "upload: LLM_Falcon_model/models--tiiuae--falcon-40b/snapshots/88b92aef6921a843415b6142bd5a906197e3fb0e/pytorch_model-00005-of-00009.bin to s3://sagemaker-us-west-2-687912291502/llm/models/LLM_falcon_model/pytorch_model-00005-of-00009.bin\n",
      "upload: LLM_Falcon_model/models--tiiuae--falcon-40b/snapshots/88b92aef6921a843415b6142bd5a906197e3fb0e/pytorch_model-00002-of-00009.bin to s3://sagemaker-us-west-2-687912291502/llm/models/LLM_falcon_model/pytorch_model-00002-of-00009.bin\n",
      "upload: LLM_Falcon_model/models--tiiuae--falcon-40b/snapshots/88b92aef6921a843415b6142bd5a906197e3fb0e/pytorch_model-00001-of-00009.bin to s3://sagemaker-us-west-2-687912291502/llm/models/LLM_falcon_model/pytorch_model-00001-of-00009.bin\n",
      "upload: LLM_Falcon_model/models--tiiuae--falcon-40b/snapshots/88b92aef6921a843415b6142bd5a906197e3fb0e/pytorch_model.bin.index.json to s3://sagemaker-us-west-2-687912291502/llm/models/LLM_falcon_model/pytorch_model.bin.index.json\n",
      "upload: LLM_Falcon_model/models--tiiuae--falcon-40b/snapshots/88b92aef6921a843415b6142bd5a906197e3fb0e/pytorch_model-00004-of-00009.bin to s3://sagemaker-us-west-2-687912291502/llm/models/LLM_falcon_model/pytorch_model-00004-of-00009.bin\n",
      "upload: LLM_Falcon_model/models--tiiuae--falcon-40b/snapshots/88b92aef6921a843415b6142bd5a906197e3fb0e/special_tokens_map.json to s3://sagemaker-us-west-2-687912291502/llm/models/LLM_falcon_model/special_tokens_map.json\n",
      "upload: LLM_Falcon_model/models--tiiuae--falcon-40b/snapshots/88b92aef6921a843415b6142bd5a906197e3fb0e/tokenizer.json to s3://sagemaker-us-west-2-687912291502/llm/models/LLM_falcon_model/tokenizer.json\n",
      "upload: LLM_Falcon_model/models--tiiuae--falcon-40b/snapshots/88b92aef6921a843415b6142bd5a906197e3fb0e/tokenizer_config.json to s3://sagemaker-us-west-2-687912291502/llm/models/LLM_falcon_model/tokenizer_config.json\n",
      "upload: LLM_Falcon_model/models--tiiuae--falcon-40b/snapshots/88b92aef6921a843415b6142bd5a906197e3fb0e/pytorch_model-00003-of-00009.bin to s3://sagemaker-us-west-2-687912291502/llm/models/LLM_falcon_model/pytorch_model-00003-of-00009.bin\n",
      "upload: LLM_Falcon_model/models--tiiuae--falcon-40b/snapshots/88b92aef6921a843415b6142bd5a906197e3fb0e/pytorch_model-00009-of-00009.bin to s3://sagemaker-us-west-2-687912291502/llm/models/LLM_falcon_model/pytorch_model-00009-of-00009.bin\n",
      "upload: LLM_Falcon_model/models--tiiuae--falcon-40b/snapshots/88b92aef6921a843415b6142bd5a906197e3fb0e/pytorch_model-00006-of-00009.bin to s3://sagemaker-us-west-2-687912291502/llm/models/LLM_falcon_model/pytorch_model-00006-of-00009.bin\n",
      "upload: LLM_Falcon_model/models--tiiuae--falcon-40b/snapshots/88b92aef6921a843415b6142bd5a906197e3fb0e/pytorch_model-00007-of-00009.bin to s3://sagemaker-us-west-2-687912291502/llm/models/LLM_falcon_model/pytorch_model-00007-of-00009.bin\n",
      "upload: LLM_Falcon_model/models--tiiuae--falcon-40b/snapshots/88b92aef6921a843415b6142bd5a906197e3fb0e/pytorch_model-00008-of-00009.bin to s3://sagemaker-us-west-2-687912291502/llm/models/LLM_falcon_model/pytorch_model-00008-of-00009.bin\n"
     ]
    }
   ],
   "source": [
    "!aws s3 cp --recursive {model_snapshot_path} s3://{bucket}/{s3_model_prefix}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "696b70c3-90f1-4175-95bf-568bafbcd383",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "### 3. 模型部署准备（entrypoint脚本，容器镜像，服务配置）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "6f7c4277-4480-42c6-aee6-1fbcca94eb82",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Image going to be used is ---- > 763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.21.0-deepspeed0.8.3-cu117\n"
     ]
    }
   ],
   "source": [
    "inference_image_uri = (\n",
    "    f\"763104351884.dkr.ecr.{region}.amazonaws.com/djl-inference:0.21.0-deepspeed0.8.3-cu117\"\n",
    ")\n",
    "\n",
    "print(f\"Image going to be used is ---- > {inference_image_uri}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "8d771bdb-11d2-45d2-9bef-face29221838",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!mkdir -p LLM_falcon_deploy_code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "e5348ecb-43df-4094-97d8-a6723004862a",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting LLM_falcon_deploy_code/model.py\n"
     ]
    }
   ],
   "source": [
    "%%writefile LLM_falcon_deploy_code/model.py\n",
    "from djl_python import Input, Output\n",
    "import torch\n",
    "import logging\n",
    "import math\n",
    "import os\n",
    "\n",
    "from transformers import pipeline, AutoModel, AutoTokenizer\n",
    "import transformers\n",
    "import torch\n",
    "\n",
    "def load_model(properties):\n",
    "    tensor_parallel = properties[\"tensor_parallel_degree\"]\n",
    "    model_location = properties['model_dir']\n",
    "    if \"model_id\" in properties:\n",
    "        model_location = properties['model_id']\n",
    "    logging.info(f\"Loading model in {model_location}\")\n",
    "    \n",
    "    tokenizer = AutoTokenizer.from_pretrained(model_location, trust_remote_code=True)\n",
    "   \n",
    "    pipeline = transformers.pipeline(\n",
    "    \"text-generation\",\n",
    "    model=model_location,\n",
    "    tokenizer=tokenizer,\n",
    "    torch_dtype=torch.bfloat16,\n",
    "    trust_remote_code=True,\n",
    "    device_map=\"auto\")\n",
    "    \n",
    "    return pipeline, tokenizer\n",
    "\n",
    "\n",
    "pipeline = None\n",
    "tokenizer = None\n",
    "generator = None\n",
    "\n",
    "\n",
    "def handle(inputs: Input):\n",
    "    global pipeline, tokenizer\n",
    "    if not pipeline:\n",
    "        pipeline, tokenizer = load_model(inputs.get_properties())\n",
    "\n",
    "    if inputs.is_empty():\n",
    "        return None\n",
    "    data = inputs.get_as_json()\n",
    "    \n",
    "    input_sentences = data[\"inputs\"]\n",
    "    params = data[\"parameters\"]\n",
    "   \n",
    "    response=pipeline(\n",
    "    input_sentences,\n",
    "    max_length=params[\"max_new_tokens\"],\n",
    "    do_sample=params[\"do_sample\"],\n",
    "    top_k=10,\n",
    "    num_return_sequences=1,\n",
    "    eos_token_id=tokenizer.eos_token_id)\n",
    "    \n",
    "    result = {\"outputs\": response}\n",
    "    return Output().add_as_json(result)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a06d1e60-3914-4059-a08f-05ac26761165",
   "metadata": {},
   "source": [
    "#### Note: option.s3url 需要按照自己的账号进行修改"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "8996fe44-8e70-468b-abc1-38187cb33f4f",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting LLM_falcon_deploy_code/serving.properties\n"
     ]
    }
   ],
   "source": [
    "%%writefile LLM_falcon_deploy_code/serving.properties\n",
    "engine=Python\n",
    "option.tensor_parallel_degree=4\n",
    "option.s3url = s3://sagemaker-us-west-2-687912291502/llm/models/LLM_falcon_model/"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "feef22a2-27b9-4018-a46b-6a99b532512f",
   "metadata": {},
   "source": [
    "#### 注意: Falcon config 上transformers是4.27.4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "7b7e76c6-6dbc-47fc-9f47-4765c526ab76",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting LLM_falcon_deploy_code/requirements.txt\n"
     ]
    }
   ],
   "source": [
    "%%writefile LLM_falcon_deploy_code/requirements.txt\n",
    "transformers==4.28.1\n",
    "einops"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "0ae6734a-aacd-410d-818d-0a962697c3c4",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LLM_falcon_deploy_code/\n",
      "LLM_falcon_deploy_code/model.py\n",
      "LLM_falcon_deploy_code/requirements.txt\n",
      "LLM_falcon_deploy_code/serving.properties\n"
     ]
    }
   ],
   "source": [
    "!rm model.tar.gz\n",
    "!cd LLM_falcon_deploy_code && rm -rf \".ipynb_checkpoints\"\n",
    "!tar czvf model.tar.gz LLM_falcon_deploy_code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "0f77dc76-6d8c-4665-ba88-f03e887c136c",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "S3 Code or Model tar ball uploaded to --- > s3://sagemaker-us-west-2-687912291502/LLM-RAG/workshop/LLM_falcon_deploy_code/model.tar.gz\n"
     ]
    }
   ],
   "source": [
    "s3_code_artifact = sess.upload_data(\"model.tar.gz\", bucket, s3_code_prefix)\n",
    "print(f\"S3 Code or Model tar ball uploaded to --- > {s3_code_artifact}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a5853daa-b8a3-4485-8c0a-64bf83e93a18",
   "metadata": {
    "tags": []
   },
   "source": [
    "### 4. 创建模型 & 创建endpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "ef974ca1-9638-45a8-9145-ea9d03b2b072",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "falcon-2023-06-08-07-41-30-199\n",
      "Image going to be used is ---- > 763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.21.0-deepspeed0.8.3-cu117\n",
      "Created Model: arn:aws:sagemaker:us-west-2:687912291502:model/falcon-2023-06-08-07-41-30-199\n"
     ]
    }
   ],
   "source": [
    "from sagemaker.utils import name_from_base\n",
    "import boto3\n",
    "\n",
    "model_name = name_from_base(f\"falcon\") # Append a timestamp to the provided string\n",
    "print(model_name)\n",
    "print(f\"Image going to be used is ---- > {inference_image_uri}\")\n",
    "\n",
    "create_model_response = sm_client.create_model(\n",
    "    ModelName=model_name,\n",
    "    ExecutionRoleArn=role,\n",
    "    PrimaryContainer={\n",
    "        \"Image\": inference_image_uri,\n",
    "        \"ModelDataUrl\": s3_code_artifact\n",
    "    },\n",
    "    \n",
    ")\n",
    "model_arn = create_model_response[\"ModelArn\"]\n",
    "\n",
    "print(f\"Created Model: {model_arn}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "233bb3a4-d737-41ad-8fcc-7082c6278e8c",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'EndpointConfigArn': 'arn:aws:sagemaker:us-west-2:687912291502:endpoint-config/falcon-2023-06-08-07-41-30-199-config',\n",
       " 'ResponseMetadata': {'RequestId': 'bdc5d5a2-1277-4548-9904-5e09e5693f84',\n",
       "  'HTTPStatusCode': 200,\n",
       "  'HTTPHeaders': {'x-amzn-requestid': 'bdc5d5a2-1277-4548-9904-5e09e5693f84',\n",
       "   'content-type': 'application/x-amz-json-1.1',\n",
       "   'content-length': '118',\n",
       "   'date': 'Thu, 08 Jun 2023 07:41:31 GMT'},\n",
       "  'RetryAttempts': 0}}"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "endpoint_config_name = f\"{model_name}-config\"\n",
    "endpoint_name = f\"{model_name}-endpoint\"\n",
    "\n",
    "\n",
    "endpoint_config_response = sm_client.create_endpoint_config(\n",
    "    EndpointConfigName=endpoint_config_name,\n",
    "    ProductionVariants=[\n",
    "        {\n",
    "            \"VariantName\": \"variant1\",\n",
    "            \"ModelName\": model_name,\n",
    "            \"InstanceType\": \"ml.g5.12xlarge\",\n",
    "            \"InitialInstanceCount\": 1,\n",
    "            # \"VolumeSizeInGB\" : 400,\n",
    "            # \"ModelDataDownloadTimeoutInSeconds\": 2400,\n",
    "            \"ContainerStartupHealthCheckTimeoutInSeconds\": 15*60,\n",
    "        },\n",
    "    ],\n",
    ")\n",
    "endpoint_config_response"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "734a39b0-473e-4421-94c8-74d2b4105038",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Created Endpoint: arn:aws:sagemaker:us-west-2:687912291502:endpoint/falcon-2023-06-08-07-41-30-199-endpoint\n"
     ]
    }
   ],
   "source": [
    "create_endpoint_response = sm_client.create_endpoint(\n",
    "    EndpointName=f\"{endpoint_name}\", EndpointConfigName=endpoint_config_name\n",
    ")\n",
    "print(f\"Created Endpoint: {create_endpoint_response['EndpointArn']}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1262e826-a810-401d-a5a9-f62febb24e5f",
   "metadata": {},
   "source": [
    "#### 持续检测模型部署进度"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "08969928-6b9e-4d9c-a033-a31f5f77bdfb",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Status: Creating\n",
      "Status: Creating\n",
      "Status: Creating\n",
      "Status: Creating\n",
      "Status: Creating\n",
      "Status: Creating\n",
      "Status: Creating\n",
      "Status: Creating\n",
      "Status: InService\n",
      "Arn: arn:aws:sagemaker:us-west-2:687912291502:endpoint/falcon-2023-06-08-07-41-30-199-endpoint\n",
      "Status: InService\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "\n",
    "resp = sm_client.describe_endpoint(EndpointName=endpoint_name)\n",
    "status = resp[\"EndpointStatus\"]\n",
    "print(\"Status: \" + status)\n",
    "\n",
    "while status == \"Creating\":\n",
    "    time.sleep(60)\n",
    "    resp = sm_client.describe_endpoint(EndpointName=endpoint_name)\n",
    "    status = resp[\"EndpointStatus\"]\n",
    "    print(\"Status: \" + status)\n",
    "\n",
    "print(\"Arn: \" + resp[\"EndpointArn\"])\n",
    "print(\"Status: \" + status)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d985b427-3959-46f7-9a50-5a2b45e2d513",
   "metadata": {
    "tags": []
   },
   "source": [
    "### 5. 模型测试"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bd86caf8-01e9-41eb-9199-478ba471a02b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "endpoint_name = \"falcon-2023-06-08-07-41-30-199-endpoint\"\n",
    "prompts1 = \"\"\"\n",
    "You are a MySQL expert. Given an input question, first create a syntactically correct MySQL query to run, then look at the results of the query and return the answer to the input question.\n",
    "Unless the user specifies in the question a specific number of examples to obtain, query for at most 3 results using the LIMIT clause as per MySQL. You can order the results to return the most informative data in the database.\n",
    "Never query for all columns from a table. You must query only the columns that are needed to answer the question. Wrap each column name in backticks (`) to denote them as delimited identifiers.\n",
    "Pay attention to use only the column names you can see in the tables below. Be careful to not query for columns that do not exist. Also, pay attention to which column is in which table.\n",
    "Pay attention to use CURDATE() function to get the current date, if the question involves \"today\".\n",
    "\n",
    "Use the following format:\n",
    "\n",
    "Question: Question here\n",
    "SQLQuery: SQL Query to run\n",
    "SQLResult: Result of the SQLQuery\n",
    "Answer: Final answer here\n",
    "\n",
    "Only use the following tables:\n",
    "\n",
    "CREATE TABLE customer (\n",
    "\tc_customer_sk INTEGER NOT NULL, \n",
    "\tc_customer_id CHAR(16) NOT NULL, \n",
    "\tc_current_cdemo_sk INTEGER, \n",
    "\tc_current_hdemo_sk INTEGER, \n",
    "\tc_current_addr_sk INTEGER, \n",
    "\tc_first_shipto_date_sk INTEGER, \n",
    "\tc_first_sales_date_sk INTEGER, \n",
    "\tc_salutation CHAR(10), \n",
    "\tc_first_name CHAR(20), \n",
    "\tc_last_name CHAR(30), \n",
    "\tc_preferred_cust_flag CHAR(1), \n",
    "\tc_birth_day INTEGER, \n",
    "\tc_birth_month INTEGER, \n",
    "\tc_birth_year INTEGER, \n",
    "\tc_birth_country VARCHAR(20), \n",
    "\tc_login CHAR(13), \n",
    "\tc_email_address CHAR(50), \n",
    "\tc_last_review_date CHAR(10), \n",
    "\tPRIMARY KEY (c_customer_sk)\n",
    ")ENGINE=InnoDB DEFAULT CHARSET=utf8\n",
    "\n",
    "\n",
    "CREATE TABLE web_sales (\n",
    "\tws_sold_date_sk INTEGER, \n",
    "\tws_sold_time_sk INTEGER, \n",
    "\tws_ship_date_sk INTEGER, \n",
    "\tws_item_sk INTEGER NOT NULL, \n",
    "\tws_bill_customer_sk INTEGER, \n",
    "\tws_bill_cdemo_sk INTEGER, \n",
    "\tws_bill_hdemo_sk INTEGER, \n",
    "\tws_bill_addr_sk INTEGER, \n",
    "\tws_ship_customer_sk INTEGER, \n",
    "\tws_ship_cdemo_sk INTEGER, \n",
    "\tws_ship_hdemo_sk INTEGER, \n",
    "\tws_ship_addr_sk INTEGER, \n",
    "\tws_web_page_sk INTEGER, \n",
    "\tws_web_site_sk INTEGER, \n",
    "\tws_ship_mode_sk INTEGER, \n",
    "\tws_warehouse_sk INTEGER, \n",
    "\tws_promo_sk INTEGER, \n",
    "\tws_order_number INTEGER NOT NULL, \n",
    "\tws_quantity INTEGER, \n",
    "\tws_wholesale_cost DECIMAL(7, 2), \n",
    "\tws_list_price DECIMAL(7, 2), \n",
    "\tws_sales_price DECIMAL(7, 2), \n",
    "\tws_ext_discount_amt DECIMAL(7, 2), \n",
    "\tws_ext_sales_price DECIMAL(7, 2), \n",
    "\tws_ext_wholesale_cost DECIMAL(7, 2), \n",
    "\tws_ext_list_price DECIMAL(7, 2), \n",
    "\tws_ext_tax DECIMAL(7, 2), \n",
    "\tws_coupon_amt DECIMAL(7, 2), \n",
    "\tws_ext_ship_cost DECIMAL(7, 2), \n",
    "\tws_net_paid DECIMAL(7, 2), \n",
    "\tws_net_paid_inc_tax DECIMAL(7, 2), \n",
    "\tws_net_paid_inc_ship DECIMAL(7, 2), \n",
    "\tws_net_paid_inc_ship_tax DECIMAL(7, 2), \n",
    "\tws_net_profit DECIMAL(7, 2), \n",
    "\tPRIMARY KEY (ws_item_sk, ws_order_number)\n",
    ")ENGINE=InnoDB DEFAULT CHARSET=utf8\n",
    "\n",
    "Question: I need to know the max sales customer's id in sales report\n",
    "\"\"\"\n",
    "response_model = smr_client.invoke_endpoint(\n",
    "            EndpointName=endpoint_name,\n",
    "            Body=json.dumps(\n",
    "            {\n",
    "                \"inputs\": prompts1,\n",
    "                \"parameters\": parameters,\n",
    "            }\n",
    "            ),\n",
    "            ContentType=\"application/json\",\n",
    "        )\n",
    "\n",
    "response_model['Body'].read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "31a2a4c3-ebfd-4887-90d6-53e0fffd5d61",
   "metadata": {},
   "outputs": [],
   "source": [
    "# define payload\n",
    "prompt = \"\"\"You are a MySQL expert. Given an input question, first create a syntactically correct MySQL query to run, then look at the results of the query and return the answer to the input question.\n",
    "Unless the user specifies in the question a specific number of examples to obtain, query for at most 3 results using the LIMIT clause as per MySQL. You can order the results to return the most informative data in the database.\n",
    "Never query for all columns from a table. You must query only the columns that are needed to answer the question. Wrap each column name in backticks (`) to denote them as delimited identifiers.\n",
    "Pay attention to use only the column names you can see in the tables below. Be careful to not query for columns that do not exist. Also, pay attention to which column is in which table.\n",
    "Pay attention to use CURDATE() function to get the current date, if the question involves \"today\".\n",
    "\n",
    "Use the following format:\n",
    "\n",
    "Question: Question here\n",
    "SQLQuery: SQL Query to run\n",
    "SQLResult: Result of the SQLQuery\n",
    "Answer: Final answer here\n",
    "\n",
    "Only use the following tables:\n",
    "\n",
    "CREATE TABLE customer (\n",
    "\tc_customer_sk INTEGER NOT NULL, \n",
    "\tc_customer_id CHAR(16) NOT NULL, \n",
    "\tc_current_cdemo_sk INTEGER, \n",
    "\tc_current_hdemo_sk INTEGER, \n",
    "\tc_current_addr_sk INTEGER, \n",
    "\tc_first_shipto_date_sk INTEGER, \n",
    "\tc_first_sales_date_sk INTEGER, \n",
    "\tc_salutation CHAR(10), \n",
    "\tc_first_name CHAR(20), \n",
    "\tc_last_name CHAR(30), \n",
    "\tc_preferred_cust_flag CHAR(1), \n",
    "\tc_birth_day INTEGER, \n",
    "\tc_birth_month INTEGER, \n",
    "\tc_birth_year INTEGER, \n",
    "\tc_birth_country VARCHAR(20), \n",
    "\tc_login CHAR(13), \n",
    "\tc_email_address CHAR(50), \n",
    "\tc_last_review_date CHAR(10), \n",
    "\tPRIMARY KEY (c_customer_sk)\n",
    ")ENGINE=InnoDB DEFAULT CHARSET=utf8\n",
    "\n",
    "\n",
    "CREATE TABLE web_sales (\n",
    "\tws_sold_date_sk INTEGER, \n",
    "\tws_sold_time_sk INTEGER, \n",
    "\tws_ship_date_sk INTEGER, \n",
    "\tws_item_sk INTEGER NOT NULL, \n",
    "\tws_bill_customer_sk INTEGER, \n",
    "\tws_bill_cdemo_sk INTEGER, \n",
    "\tws_bill_hdemo_sk INTEGER, \n",
    "\tws_bill_addr_sk INTEGER, \n",
    "\tws_ship_customer_sk INTEGER, \n",
    "\tws_ship_cdemo_sk INTEGER, \n",
    "\tws_ship_hdemo_sk INTEGER, \n",
    "\tws_ship_addr_sk INTEGER, \n",
    "\tws_web_page_sk INTEGER, \n",
    "\tws_web_site_sk INTEGER, \n",
    "\tws_ship_mode_sk INTEGER, \n",
    "\tws_warehouse_sk INTEGER, \n",
    "\tws_promo_sk INTEGER, \n",
    "\tws_order_number INTEGER NOT NULL, \n",
    "\tws_quantity INTEGER, \n",
    "\tws_wholesale_cost DECIMAL(7, 2), \n",
    "\tws_list_price DECIMAL(7, 2), \n",
    "\tws_sales_price DECIMAL(7, 2), \n",
    "\tws_ext_discount_amt DECIMAL(7, 2), \n",
    "\tws_ext_sales_price DECIMAL(7, 2), \n",
    "\tws_ext_wholesale_cost DECIMAL(7, 2), \n",
    "\tws_ext_list_price DECIMAL(7, 2), \n",
    "\tws_ext_tax DECIMAL(7, 2), \n",
    "\tws_coupon_amt DECIMAL(7, 2), \n",
    "\tws_ext_ship_cost DECIMAL(7, 2), \n",
    "\tws_net_paid DECIMAL(7, 2), \n",
    "\tws_net_paid_inc_tax DECIMAL(7, 2), \n",
    "\tws_net_paid_inc_ship DECIMAL(7, 2), \n",
    "\tws_net_paid_inc_ship_tax DECIMAL(7, 2), \n",
    "\tws_net_profit DECIMAL(7, 2), \n",
    "\tPRIMARY KEY (ws_item_sk, ws_order_number)\n",
    ")ENGINE=InnoDB DEFAULT CHARSET=utf8\n",
    "\n",
    "Question: I need to know the max sales customer's id in sales report\n",
    "\n",
    "\"\"\"\n",
    "\n",
    "# hyperparameters for llm\n",
    "payload = {\n",
    "  \"inputs\": prompt,\n",
    "  \"parameters\": {\n",
    "    \"do_sample\": True,\n",
    "    \"top_p\": 0.9,\n",
    "    \"temperature\": 0.8,\n",
    "    \"max_new_tokens\": 1024,\n",
    "    \"repetition_penalty\": 1.03,\n",
    "    \"stop\": [\"\\nUser:\",\"<|endoftext|>\",\"</s>\"]\n",
    "  }\n",
    "}\n",
    "\n",
    "# send request to endpoint\n",
    "response = llm.predict(payload)\n",
    "\n",
    "# print assistant respond\n",
    "assistant = response[0][\"generated_text\"][len(prompt):]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "15c99fd6-e558-416a-a015-5a0b41f56f6a",
   "metadata": {
    "tags": []
   },
   "source": [
    "## 使用HF model加载"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "e2adbabc-dae0-42c3-835e-c662624875d0",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sagemaker                                    2.163.0\n",
      "sagemaker-pyspark                            1.4.5\n"
     ]
    }
   ],
   "source": [
    "!pip list|grep -i sagemaker"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "4a67bae0-97dc-41a3-9c86-400538860c42",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Looking in indexes: https://pypi.org/simple, https://pip.repos.neuron.amazonaws.com\n",
      "Requirement already satisfied: boto3 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (1.26.144)\n",
      "Collecting boto3\n",
      "  Downloading boto3-1.26.150-py3-none-any.whl (135 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m135.6/135.6 kB\u001b[0m \u001b[31m5.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: s3transfer<0.7.0,>=0.6.0 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from boto3) (0.6.0)\n",
      "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from boto3) (1.0.1)\n",
      "Collecting botocore<1.30.0,>=1.29.150\n",
      "  Downloading botocore-1.29.150-py3-none-any.whl (10.9 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m10.9/10.9 MB\u001b[0m \u001b[31m108.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: urllib3<1.27,>=1.25.4 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from botocore<1.30.0,>=1.29.150->boto3) (1.26.15)\n",
      "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from botocore<1.30.0,>=1.29.150->boto3) (2.8.2)\n",
      "Requirement already satisfied: six>=1.5 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from python-dateutil<3.0.0,>=2.1->botocore<1.30.0,>=1.29.150->boto3) (1.16.0)\n",
      "Installing collected packages: botocore, boto3\n",
      "  Attempting uninstall: botocore\n",
      "    Found existing installation: botocore 1.29.144\n",
      "    Uninstalling botocore-1.29.144:\n",
      "      Successfully uninstalled botocore-1.29.144\n",
      "  Attempting uninstall: boto3\n",
      "    Found existing installation: boto3 1.26.144\n",
      "    Uninstalling boto3-1.26.144:\n",
      "      Successfully uninstalled boto3-1.26.144\n",
      "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
      "awscli 1.27.71 requires botocore==1.29.71, but you have botocore 1.29.150 which is incompatible.\n",
      "awscli 1.27.71 requires PyYAML<5.5,>=3.10, but you have pyyaml 6.0 which is incompatible.\n",
      "aiobotocore 2.4.2 requires botocore<1.27.60,>=1.27.59, but you have botocore 1.29.150 which is incompatible.\u001b[0m\u001b[31m\n",
      "\u001b[0mSuccessfully installed boto3-1.26.150 botocore-1.29.150\n"
     ]
    }
   ],
   "source": [
    "# TODO: remove once new version is released\n",
    "#!pip install git+https://github.com/aws/sagemaker-python-sdk --upgrade\n",
    "# install latest sagemaker SDK\n",
    "!pip install sagemaker>=2.161.0\n",
    "!pip install --upgrade boto3"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bb81259a-bd93-4b0b-9280-ef5893c1a7e2",
   "metadata": {
    "tags": []
   },
   "source": [
    "最新的sagemaker sdk，没有这个llm image uri的api"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "40835888-6a33-47f9-8af4-e00e5a4fa78e",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "llm image uri: 763104351884.dkr.ecr.us-west-2.amazonaws.com/huggingface-pytorch-tgi-inference:2.0.0-tgi0.8.2-gpu-py39-cu118-ubuntu20.04\n"
     ]
    }
   ],
   "source": [
    "import sagemaker\n",
    "from sagemaker.huggingface import HuggingFaceModel, get_huggingface_llm_image_uri\n",
    "\n",
    "# retrieve the llm image uri\n",
    "llm_image = get_huggingface_llm_image_uri(\n",
    "  \"huggingface\",\n",
    "  version=\"0.8.2\"\n",
    ")\n",
    "\n",
    "# print ecr image uri\n",
    "print(f\"llm image uri: {llm_image}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "623f8d8b-b0c4-47d2-95ad-6f04e95bcef6",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import json\n",
    "from sagemaker.huggingface import HuggingFaceModel\n",
    "\n",
    "# sagemaker config\n",
    "instance_type = \"ml.g5.12xlarge\"\n",
    "number_of_gpu = 4\n",
    "health_check_timeout = 600\n",
    "\n",
    "# TGI config\n",
    "config = {\n",
    "  'HF_MODEL_ID': \"tiiuae/falcon-40b-instruct\", # model_id from hf.co/models\n",
    "  'SM_NUM_GPUS': json.dumps(number_of_gpu), # Number of GPU used per replica\n",
    "  'MAX_INPUT_LENGTH': json.dumps(1200),  # Max length of input text\n",
    "  'MAX_TOTEL_TOKENS': json.dumps(3000),  # Max length of the generation (including input text)\n",
    "  # 'HF_MODEL_QUANTIZE': \"bitsandbytes\", # comment in to quantize\n",
    "}\n",
    "\n",
    "# create HuggingFaceModel\n",
    "llm_model = HuggingFaceModel(\n",
    "  role=role,\n",
    "  image_uri=llm_image,\n",
    "  env=config\n",
    ")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "fb0ef1cb-a1c0-4871-91bb-4e8f4c1427d7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------!"
     ]
    }
   ],
   "source": [
    "# Deploy model to an endpoint\n",
    "# https://sagemaker.readthedocs.io/en/stable/api/inference/model.html#sagemaker.model.Model.deploy\n",
    "llm = llm_model.deploy(\n",
    "  initial_instance_count=1,\n",
    "  instance_type=instance_type,\n",
    "  # volume_size=400, # If using an instance with local SSD storage, volume_size must be None, e.g. p4 but not p3\n",
    "  container_startup_health_check_timeout=health_check_timeout, # 10 minutes to be able to load the model\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "bef5abea-7047-4457-9908-632eeeccff79",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# define payload\n",
    "prompt = \"\"\"You are a MySQL expert. Given an input question, first create a syntactically correct MySQL query to run, then look at the results of the query and return the answer to the input question.\n",
    "Unless the user specifies in the question a specific number of examples to obtain, query for at most 3 results using the LIMIT clause as per MySQL. You can order the results to return the most informative data in the database.\n",
    "Never query for all columns from a table. You must query only the columns that are needed to answer the question. Wrap each column name in backticks (`) to denote them as delimited identifiers.\n",
    "Pay attention to use only the column names you can see in the tables below. Be careful to not query for columns that do not exist. Also, pay attention to which column is in which table.\n",
    "Pay attention to use CURDATE() function to get the current date, if the question involves \"today\".\n",
    "\n",
    "Use the following format:\n",
    "\n",
    "Question: Question here\n",
    "SQLQuery: SQL Query to run\n",
    "SQLResult: Result of the SQLQuery\n",
    "Answer: Final answer here\n",
    "\n",
    "Only use the following tables:\n",
    "\n",
    "CREATE TABLE customer (\n",
    "\tc_customer_sk INTEGER NOT NULL, \n",
    "\tc_customer_id CHAR(16) NOT NULL, \n",
    "\tc_current_cdemo_sk INTEGER, \n",
    "\tc_current_hdemo_sk INTEGER, \n",
    "\tc_current_addr_sk INTEGER, \n",
    "\tc_first_shipto_date_sk INTEGER, \n",
    "\tc_email_address CHAR(50), \n",
    "\tc_last_review_date CHAR(10), \n",
    "\tPRIMARY KEY (c_customer_sk)\n",
    ")ENGINE=InnoDB DEFAULT CHARSET=utf8\n",
    "\n",
    "\n",
    "CREATE TABLE web_sales (\n",
    "\tws_sold_date_sk INTEGER, \n",
    "\tws_sold_time_sk INTEGER, \n",
    "\tws_ship_date_sk INTEGER, \n",
    "\tws_item_sk INTEGER NOT NULL, \n",
    "\tws_bill_customer_sk INTEGER, \n",
    "\tws_bill_cdemo_sk INTEGER, \n",
    "\tws_bill_hdemo_sk INTEGER, \n",
    "\tws_bill_addr_sk INTEGER, \n",
    "\tws_ship_customer_sk INTEGER, \n",
    "\tws_ship_cdemo_sk INTEGER, \n",
    "\tws_ship_hdemo_sk INTEGER, \n",
    "\tws_ship_addr_sk INTEGER, \n",
    "\tws_web_page_sk INTEGER, \n",
    "\tws_web_site_sk INTEGER, \n",
    "\tws_ship_mode_sk INTEGER, \n",
    "\tws_warehouse_sk INTEGER, \n",
    "\tws_promo_sk INTEGER, \n",
    "\tws_order_number INTEGER NOT NULL, \n",
    "\tws_quantity INTEGER, \n",
    "\tws_wholesale_cost DECIMAL(7, 2), \n",
    "\tws_list_price DECIMAL(7, 2), \n",
    "\tws_sales_price DECIMAL(7, 2), \n",
    "\tws_ext_discount_amt DECIMAL(7, 2), \n",
    "\tws_ext_sales_price DECIMAL(7, 2), \n",
    "\tws_ext_wholesale_cost DECIMAL(7, 2), \n",
    "\tws_ext_list_price DECIMAL(7, 2), \n",
    "\tws_ext_tax DECIMAL(7, 2), \n",
    "\tws_coupon_amt DECIMAL(7, 2), \n",
    "\tws_ext_ship_cost DECIMAL(7, 2), \n",
    "\tws_net_paid DECIMAL(7, 2), \n",
    "\tws_net_paid_inc_tax DECIMAL(7, 2), \n",
    "\tws_net_paid_inc_ship DECIMAL(7, 2), \n",
    "\tws_net_paid_inc_ship_tax DECIMAL(7, 2), \n",
    "\tws_net_profit DECIMAL(7, 2), \n",
    "\tPRIMARY KEY (ws_item_sk, ws_order_number)\n",
    ")ENGINE=InnoDB DEFAULT CHARSET=utf8\n",
    "\n",
    "Question: I need to know the customer's email address of max net paid in sale report table \n",
    "\n",
    "\"\"\"\n",
    "\n",
    "# hyperparameters for llm\n",
    "payload = {\n",
    "  \"inputs\": prompt,\n",
    "  \"parameters\": {\n",
    "    \"do_sample\": True,\n",
    "    \"top_p\": 0.9,\n",
    "    \"temperature\": 0.8,\n",
    "    \"max_new_tokens\": 300,\n",
    "    \"repetition_penalty\": 1.03,\n",
    "    \"stop\": [\"\\nUser:\",\"<|endoftext|>\",\"</s>\"]\n",
    "  }\n",
    "}\n",
    "\n",
    "# send request to endpoint\n",
    "response = llm.predict(payload)\n",
    "\n",
    "# print assistant respond\n",
    "assistant = response[0][\"generated_text\"][len(prompt):]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "42a75823-a26a-4e25-a8de-59333f4f6f11",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<p>SQLQuery:</p>\n",
      "\n",
      "<pre><code>SELECT c.c_email_address FROM customer c \n",
      "INNER JOIN web_sales w ON c.c_customer_sk = w.ws_bill_customer_sk\n",
      "WHERE w.ws_net_paid_inc_tax = (\n",
      "SELECT MAX(ws_net_paid_inc_tax) FROM web_sales\n",
      ") ORDER BY w.ws_net_paid_inc_tax DESC LIMIT 1\n",
      "</code></pre>\n",
      "\n",
      "<p>SQLResult:</p>\n",
      "\n",
      "<pre><code>c_email_address\n",
      "----------\n",
      "someone@email.com\n",
      "</code></pre>\n",
      "\n",
      "<p>Answer:</p>\n",
      "\n",
      "<pre><code>The customer's email address who had the maximum net paid in the sale report table is someone@email.com.\n",
      "</code></pre>\n"
     ]
    }
   ],
   "source": [
    "print(assistant)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "21c8b703-e312-4964-8be9-a754468e07cd",
   "metadata": {},
   "source": [
    "#### 清除模型Endpoint和config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "id": "f70d116f-4fb1-4f04-8732-3d6e4fb520de",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!aws sagemaker delete-endpoint --endpoint-name chatglm-2023-04-27-05-49-59-117-endpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "id": "184e4d1d-3d62-43df-9b17-5d64ece928bd",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!aws sagemaker delete-endpoint-config --endpoint-config-name chatglm-2023-04-27-05-49-59-117-config"
   ]
  }
 ],
 "metadata": {
  "availableInstances": [
   {
    "_defaultOrder": 0,
    "_isFastLaunch": true,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 4,
    "name": "ml.t3.medium",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 1,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.t3.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 2,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.t3.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 3,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.t3.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 4,
    "_isFastLaunch": true,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.m5.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 5,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.m5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 6,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.m5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 7,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.m5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 8,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.m5.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 9,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.m5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 10,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.m5.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 11,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.m5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 12,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.m5d.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 13,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.m5d.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 14,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.m5d.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 15,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.m5d.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 16,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.m5d.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 17,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.m5d.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 18,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.m5d.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 19,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.m5d.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 20,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": true,
    "memoryGiB": 0,
    "name": "ml.geospatial.interactive",
    "supportedImageNames": [
     "sagemaker-geospatial-v1-0"
    ],
    "vcpuNum": 0
   },
   {
    "_defaultOrder": 21,
    "_isFastLaunch": true,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 4,
    "name": "ml.c5.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 22,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.c5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 23,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.c5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 24,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.c5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 25,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 72,
    "name": "ml.c5.9xlarge",
    "vcpuNum": 36
   },
   {
    "_defaultOrder": 26,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 96,
    "name": "ml.c5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 27,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 144,
    "name": "ml.c5.18xlarge",
    "vcpuNum": 72
   },
   {
    "_defaultOrder": 28,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.c5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 29,
    "_isFastLaunch": true,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.g4dn.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 30,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.g4dn.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 31,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.g4dn.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 32,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.g4dn.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 33,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.g4dn.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 34,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.g4dn.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 35,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 61,
    "name": "ml.p3.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 36,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 244,
    "name": "ml.p3.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 37,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 488,
    "name": "ml.p3.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 38,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 768,
    "name": "ml.p3dn.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 39,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.r5.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 40,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.r5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 41,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.r5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 42,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.r5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 43,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.r5.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 44,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.r5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 45,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 512,
    "name": "ml.r5.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 46,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 768,
    "name": "ml.r5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 47,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.g5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 48,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.g5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 49,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.g5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 50,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.g5.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 51,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.g5.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 52,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.g5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 53,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.g5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 54,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 768,
    "name": "ml.g5.48xlarge",
    "vcpuNum": 192
   }
  ],
  "instance_type": "ml.m5.large",
  "kernelspec": {
   "display_name": "conda_pytorch_p39",
   "language": "python",
   "name": "conda_pytorch_p39"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}