baichuan_7B_deploy.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "c261e5f4-17a8-40da-beb9-599f1717e0fe",
   "metadata": {
    "tags": []
   },
   "source": [
    "### 1. 安装HuggingFace 并下载模型到本地"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "02785614-9268-41c8-85a5-d579490edbbf",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Looking in indexes: https://pypi.org/simple, https://pip.repos.neuron.amazonaws.com\n",
      "Requirement already satisfied: accelerate>=0.17.2 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (0.20.3)\n",
      "Requirement already satisfied: torch>=1.6.0 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from accelerate>=0.17.2) (1.13.1)\n",
      "Requirement already satisfied: pyyaml in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from accelerate>=0.17.2) (6.0)\n",
      "Requirement already satisfied: numpy>=1.17 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from accelerate>=0.17.2) (1.23.5)\n",
      "Requirement already satisfied: packaging>=20.0 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from accelerate>=0.17.2) (21.3)\n",
      "Requirement already satisfied: psutil in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from accelerate>=0.17.2) (5.9.4)\n",
      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from packaging>=20.0->accelerate>=0.17.2) (3.0.9)\n",
      "Requirement already satisfied: typing_extensions in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from torch>=1.6.0->accelerate>=0.17.2) (4.4.0)\n"
     ]
    }
   ],
   "source": [
    "#!pip install huggingface-hub -Uqq\n",
    "!pip install accelerate\">=0.17.2\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "9e6bd7ee-16a3-4f5a-8857-8bbba83eb9e7",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from huggingface_hub import snapshot_download\n",
    "from pathlib import Path\n",
    "\n",
    "local_model_path = Path(\"./LLM_baichuan_model\")\n",
    "local_model_path.mkdir(exist_ok=True)\n",
    "model_name = \"baichuan-inc/baichuan-7B\"\n",
    "commit_hash = \"ba9db8ed916eb8c4d4349d40ef7a0b6b68a0b930\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "94e8abc5-a58e-40e2-b1e6-fbf48307c716",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.01471090316772461,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": null,
       "postfix": null,
       "prefix": "Fetching 13 files",
       "rate": null,
       "total": 13,
       "unit": "it",
       "unit_divisor": 1000,
       "unit_scale": false
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "28b581264ae143dcbbb0c23445eb6732",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Fetching 13 files:   0%|          | 0/13 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.0053920745849609375,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": null,
       "postfix": null,
       "prefix": "Downloading (…)modeling_baichuan.py",
       "rate": null,
       "total": 28598,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "87a72ed2e3314194abd75f5a15b6d061",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)modeling_baichuan.py:   0%|          | 0.00/28.6k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.00509953498840332,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": null,
       "postfix": null,
       "prefix": "Downloading (…)0b930/.gitattributes",
       "rate": null,
       "total": 1477,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "188e4a30517b46a9aa4caf71ccdda19d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)0b930/.gitattributes:   0%|          | 0.00/1.48k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.0051517486572265625,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": null,
       "postfix": null,
       "prefix": "Downloading (…)neration_config.json",
       "rate": null,
       "total": 132,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d2269ff59c0442daa21171a7374c4a96",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)neration_config.json:   0%|          | 0.00/132 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.00568842887878418,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": null,
       "postfix": null,
       "prefix": "Downloading (…)68a0b930/config.json",
       "rate": null,
       "total": 656,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d8015d8bc29c498a81e8ac08e10d61cc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)68a0b930/config.json:   0%|          | 0.00/656 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.004865407943725586,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": null,
       "postfix": null,
       "prefix": "Downloading pytorch_model.bin",
       "rate": null,
       "total": 14001182896,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "057bb8eb8500498bac70aae133d20486",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading pytorch_model.bin:   0%|          | 0.00/14.0G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.006084442138671875,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": null,
       "postfix": null,
       "prefix": "Downloading (…)nization_baichuan.py",
       "rate": null,
       "total": 9574,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c3ba52a5fcd840a7be23927b4c6d106b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)nization_baichuan.py:   0%|          | 0.00/9.57k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.005783796310424805,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": null,
       "postfix": null,
       "prefix": "Downloading (…)guration_baichuan.py",
       "rate": null,
       "total": 2345,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "11f0bc0359bd4b10b8f23208df12df19",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)guration_baichuan.py:   0%|          | 0.00/2.35k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.006860017776489258,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": null,
       "postfix": null,
       "prefix": "Downloading (…)cial_tokens_map.json",
       "rate": null,
       "total": 411,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dd98a2e4a5ce4fb2a9855dde2320f349",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)cial_tokens_map.json:   0%|          | 0.00/411 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.0058290958404541016,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": null,
       "postfix": null,
       "prefix": "Downloading (…)okenizer_config.json",
       "rate": null,
       "total": 802,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "759a7cbcffaa443bb1e906b0712eab18",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)okenizer_config.json:   0%|          | 0.00/802 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.007354259490966797,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": null,
       "postfix": null,
       "prefix": "Downloading (…)5%8D%8F%E8%AE%AE.pdf",
       "rate": null,
       "total": 774879,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "490b116077b34305b89bb1704fc337a7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)5%8D%8F%E8%AE%AE.pdf:   0%|          | 0.00/775k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.00601959228515625,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": null,
       "postfix": null,
       "prefix": "Downloading tokenizer.model",
       "rate": null,
       "total": 2095253,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b4bce219cd2f4bb6bb81f5e9782d034c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading tokenizer.model:   0%|          | 0.00/2.10M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.006313323974609375,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": null,
       "postfix": null,
       "prefix": "Downloading (…)6b68a0b930/README.md",
       "rate": null,
       "total": 13122,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c18ffce722294d4781113002a87acf64",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)6b68a0b930/README.md:   0%|          | 0.00/13.1k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.0067996978759765625,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": null,
       "postfix": null,
       "prefix": "Downloading (…)b68a0b930/handler.py",
       "rate": null,
       "total": 1052,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9fa3127c4b7b44c8b3bbcfc61a39d8ce",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)b68a0b930/handler.py:   0%|          | 0.00/1.05k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'LLM_baichuan_model/models--baichuan-inc--baichuan-7B/snapshots/ba9db8ed916eb8c4d4349d40ef7a0b6b68a0b930'"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "snapshot_download(repo_id=model_name, revision=commit_hash,cache_dir=local_model_path)\n",
    "#snapshot_download(repo_id=model_name,cache_dir=local_model_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8d666c79-b039-4258-ac3b-46b19e63c3b8",
   "metadata": {
    "tags": []
   },
   "source": [
    "### 2. 把模型拷贝到S3为后续部署做准备"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e9431deb-6359-442d-847b-1563f8dd3854",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import sagemaker\n",
    "from sagemaker import image_uris\n",
    "import boto3\n",
    "import os\n",
    "import time\n",
    "import json\n",
    "\n",
    "role = sagemaker.get_execution_role()  # execution role for the endpoint\n",
    "sess = sagemaker.session.Session()  # sagemaker session for interacting with different AWS APIs\n",
    "bucket = sess.default_bucket()  # bucket to house artifacts\n",
    "\n",
    "region = sess._region_name\n",
    "account_id = sess.account_id()\n",
    "\n",
    "s3_client = boto3.client(\"s3\")\n",
    "sm_client = boto3.client(\"sagemaker\")\n",
    "smr_client = boto3.client(\"sagemaker-runtime\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "40dd8f16-ae7c-48bf-8e52-1a15425fa74d",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "s3_code_prefix: LLM-RAG/workshop/LLM_baichuan_deploy_code\n",
      "model_snapshot_path: LLM_baichuan_model/models--baichuan-inc--baichuan-7B/snapshots/ba9db8ed916eb8c4d4349d40ef7a0b6b68a0b930\n"
     ]
    }
   ],
   "source": [
    "s3_model_prefix = \"llm/models/LLM_baichuan_model\"  # folder where model checkpoint will go\n",
    "model_snapshot_path = list(local_model_path.glob(\"**/snapshots/*\"))[0]\n",
    "s3_code_prefix = \"LLM-RAG/workshop/LLM_baichuan_deploy_code\"\n",
    "print(f\"s3_code_prefix: {s3_code_prefix}\")\n",
    "print(f\"model_snapshot_path: {model_snapshot_path}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "067292c9-c066-4649-a61f-b460a24da584",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "upload: LLM_baichuan_model/models--baichuan-inc--baichuan-7B/snapshots/ba9db8ed916eb8c4d4349d40ef7a0b6b68a0b930/.gitattributes to s3://sagemaker-us-west-2-687912291502/llm/models/LLM_baichuan_model/.gitattributes\n",
      "upload: LLM_baichuan_model/models--baichuan-inc--baichuan-7B/snapshots/ba9db8ed916eb8c4d4349d40ef7a0b6b68a0b930/handler.py to s3://sagemaker-us-west-2-687912291502/llm/models/LLM_baichuan_model/handler.py\n",
      "upload: LLM_baichuan_model/models--baichuan-inc--baichuan-7B/snapshots/ba9db8ed916eb8c4d4349d40ef7a0b6b68a0b930/generation_config.json to s3://sagemaker-us-west-2-687912291502/llm/models/LLM_baichuan_model/generation_config.json\n",
      "upload: LLM_baichuan_model/models--baichuan-inc--baichuan-7B/snapshots/ba9db8ed916eb8c4d4349d40ef7a0b6b68a0b930/config.json to s3://sagemaker-us-west-2-687912291502/llm/models/LLM_baichuan_model/config.json\n",
      "upload: LLM_baichuan_model/models--baichuan-inc--baichuan-7B/snapshots/ba9db8ed916eb8c4d4349d40ef7a0b6b68a0b930/modeling_baichuan.py to s3://sagemaker-us-west-2-687912291502/llm/models/LLM_baichuan_model/modeling_baichuan.py\n",
      "upload: LLM_baichuan_model/models--baichuan-inc--baichuan-7B/snapshots/ba9db8ed916eb8c4d4349d40ef7a0b6b68a0b930/special_tokens_map.json to s3://sagemaker-us-west-2-687912291502/llm/models/LLM_baichuan_model/special_tokens_map.json\n",
      "upload: LLM_baichuan_model/models--baichuan-inc--baichuan-7B/snapshots/ba9db8ed916eb8c4d4349d40ef7a0b6b68a0b930/tokenizer_config.json to s3://sagemaker-us-west-2-687912291502/llm/models/LLM_baichuan_model/tokenizer_config.json\n",
      "upload: LLM_baichuan_model/models--baichuan-inc--baichuan-7B/snapshots/ba9db8ed916eb8c4d4349d40ef7a0b6b68a0b930/tokenization_baichuan.py to s3://sagemaker-us-west-2-687912291502/llm/models/LLM_baichuan_model/tokenization_baichuan.py\n",
      "upload: LLM_baichuan_model/models--baichuan-inc--baichuan-7B/snapshots/ba9db8ed916eb8c4d4349d40ef7a0b6b68a0b930/configuration_baichuan.py to s3://sagemaker-us-west-2-687912291502/llm/models/LLM_baichuan_model/configuration_baichuan.py\n",
      "upload: LLM_baichuan_model/models--baichuan-inc--baichuan-7B/snapshots/ba9db8ed916eb8c4d4349d40ef7a0b6b68a0b930/baichuan-7B 模型许可协议.pdf to s3://sagemaker-us-west-2-687912291502/llm/models/LLM_baichuan_model/baichuan-7B 模型许可协议.pdf\n",
      "upload: LLM_baichuan_model/models--baichuan-inc--baichuan-7B/snapshots/ba9db8ed916eb8c4d4349d40ef7a0b6b68a0b930/README.md to s3://sagemaker-us-west-2-687912291502/llm/models/LLM_baichuan_model/README.md\n",
      "upload: LLM_baichuan_model/models--baichuan-inc--baichuan-7B/snapshots/ba9db8ed916eb8c4d4349d40ef7a0b6b68a0b930/tokenizer.model to s3://sagemaker-us-west-2-687912291502/llm/models/LLM_baichuan_model/tokenizer.model\n",
      "upload: LLM_baichuan_model/models--baichuan-inc--baichuan-7B/snapshots/ba9db8ed916eb8c4d4349d40ef7a0b6b68a0b930/pytorch_model.bin to s3://sagemaker-us-west-2-687912291502/llm/models/LLM_baichuan_model/pytorch_model.bin\n"
     ]
    }
   ],
   "source": [
    "!aws s3 cp --recursive {model_snapshot_path} s3://{bucket}/{s3_model_prefix}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "696b70c3-90f1-4175-95bf-568bafbcd383",
   "metadata": {
    "tags": []
   },
   "source": [
    "### 3. 模型部署准备（entrypoint脚本，容器镜像，服务配置）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "6f7c4277-4480-42c6-aee6-1fbcca94eb82",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Image going to be used is ---- > 763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.22.1-deepspeed0.9.2-cu118\n"
     ]
    }
   ],
   "source": [
    "inference_image_uri = (\n",
    "    f\"763104351884.dkr.ecr.{region}.amazonaws.com/djl-inference:0.22.1-deepspeed0.9.2-cu118\"\n",
    ")\n",
    "\n",
    "print(f\"Image going to be used is ---- > {inference_image_uri}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "8d771bdb-11d2-45d2-9bef-face29221838",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!mkdir -p LLM_baichuan_deploy_code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "e5348ecb-43df-4094-97d8-a6723004862a",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting LLM_baichuan_deploy_code/model.py\n"
     ]
    }
   ],
   "source": [
    "%%writefile LLM_baichuan_deploy_code/model.py\n",
    "from djl_python import Input, Output\n",
    "import torch\n",
    "import logging\n",
    "import math\n",
    "import os\n",
    "from transformers import pipeline, AutoModel, AutoTokenizer, AutoModelForCausalLM\n",
    "import transformers\n",
    "import torch\n",
    "import deepspeed\n",
    "\n",
    "\n",
    "def load_model(properties):\n",
    "    tensor_parallel_degree = properties[\"tensor_parallel_degree\"]\n",
    "    model_location = properties['model_dir']\n",
    "    if \"model_id\" in properties:\n",
    "        model_location = properties['model_id']\n",
    "    logging.info(f\"Loading model in {model_location}\")\n",
    "    \n",
    "    tokenizer = AutoTokenizer.from_pretrained(model_location,trust_remote_code=True)\n",
    "    \n",
    "    pipeline = AutoModelForCausalLM.from_pretrained(  \n",
    "    pretrained_model_name_or_path=model_location,  \n",
    "    device_map=\"auto\",  \n",
    "    trust_remote_code=True,  \n",
    "    torch_dtype=torch.float16)\n",
    "    \n",
    "    #pipeline = deepspeed.init_inference(pipeline,\n",
    "    #      tensor_parallel={\"tp_size\": tensor_parallel_degree},\n",
    "    #      dtype=pipeline.dtype,\n",
    "    #      replace_method='auto',\n",
    "    #      replace_with_kernel_inject=True)\n",
    "    \n",
    "    return pipeline, tokenizer\n",
    "\n",
    "\n",
    "pipeline = None\n",
    "tokenizer = None\n",
    "generator = None\n",
    "\n",
    "\n",
    "def handle(inputs: Input):\n",
    "    global pipeline, tokenizer\n",
    "    if not pipeline:\n",
    "        pipeline, tokenizer = load_model(inputs.get_properties())\n",
    "\n",
    "    if inputs.is_empty():\n",
    "        return None\n",
    "    data = inputs.get_as_json()\n",
    "    \n",
    "    #input_sentences = prefix_prompt+\"User: \"+data[\"inputs\"]\n",
    "    input_sentences = data[\"inputs\"]\n",
    "    params = data[\"parameters\"]\n",
    "    input_ids = tokenizer.encode(input_sentences, return_tensors='pt').to('cuda')\n",
    "\n",
    "    with torch.no_grad():  \n",
    "       output_ids = pipeline.generate(  \n",
    "            input_ids,\n",
    "            **params)\n",
    "\n",
    "    response=tokenizer.decode(output_ids[0], skip_special_tokens=True)\n",
    "    result = {\"outputs\": response}\n",
    "    return Output().add_as_json(result)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a06d1e60-3914-4059-a08f-05ac26761165",
   "metadata": {},
   "source": [
    "#### Note: option.s3url 需要按照自己的账号进行修改"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "8996fe44-8e70-468b-abc1-38187cb33f4f",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting LLM_baichuan_deploy_code/serving.properties\n"
     ]
    }
   ],
   "source": [
    "%%writefile LLM_baichuan_deploy_code/serving.properties\n",
    "engine=DeepSpeed\n",
    "option.tensor_parallel_degree=4\n",
    "option.s3url=s3://sagemaker-us-west-2-687912291502/llm/models/LLM_baichuan_model/"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "feef22a2-27b9-4018-a46b-6a99b532512f",
   "metadata": {},
   "source": [
    "#### 注意: baichuan config 上transformers是4.29.2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "7b7e76c6-6dbc-47fc-9f47-4765c526ab76",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting LLM_baichuan_deploy_code/requirements.txt\n"
     ]
    }
   ],
   "source": [
    "%%writefile LLM_baichuan_deploy_code/requirements.txt\n",
    "transformers==4.29.1\n",
    "accelerate>=0.17.1\n",
    "einops"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "0ae6734a-aacd-410d-818d-0a962697c3c4",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LLM_baichuan_deploy_code/\n",
      "LLM_baichuan_deploy_code/model.py\n",
      "LLM_baichuan_deploy_code/serving.properties\n",
      "LLM_baichuan_deploy_code/requirements.txt\n"
     ]
    }
   ],
   "source": [
    "!rm model.tar.gz\n",
    "!cd LLM_baichuan_deploy_code && rm -rf \".ipynb_checkpoints\"\n",
    "!tar czvf model.tar.gz LLM_baichuan_deploy_code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "0f77dc76-6d8c-4665-ba88-f03e887c136c",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "S3 Code or Model tar ball uploaded to --- > s3://sagemaker-us-west-2-687912291502/LLM-RAG/workshop/LLM_baichuan_deploy_code/model.tar.gz\n"
     ]
    }
   ],
   "source": [
    "s3_code_artifact = sess.upload_data(\"model.tar.gz\", bucket, s3_code_prefix)\n",
    "print(f\"S3 Code or Model tar ball uploaded to --- > {s3_code_artifact}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a5853daa-b8a3-4485-8c0a-64bf83e93a18",
   "metadata": {
    "tags": []
   },
   "source": [
    "### 4. 创建模型 & 创建endpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "ef974ca1-9638-45a8-9145-ea9d03b2b072",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "baichuan-2023-06-16-08-32-34-825\n",
      "Image going to be used is ---- > 763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.22.1-deepspeed0.9.2-cu118\n",
      "Created Model: arn:aws:sagemaker:us-west-2:687912291502:model/baichuan-2023-06-16-08-32-34-825\n"
     ]
    }
   ],
   "source": [
    "from sagemaker.utils import name_from_base\n",
    "import boto3\n",
    "\n",
    "model_name = name_from_base(f\"baichuan\") # Append a timestamp to the provided string\n",
    "print(model_name)\n",
    "print(f\"Image going to be used is ---- > {inference_image_uri}\")\n",
    "\n",
    "create_model_response = sm_client.create_model(\n",
    "    ModelName=model_name,\n",
    "    ExecutionRoleArn=role,\n",
    "    PrimaryContainer={\n",
    "        \"Image\": inference_image_uri,\n",
    "        \"ModelDataUrl\": s3_code_artifact\n",
    "    },\n",
    "    \n",
    ")\n",
    "model_arn = create_model_response[\"ModelArn\"]\n",
    "\n",
    "print(f\"Created Model: {model_arn}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "233bb3a4-d737-41ad-8fcc-7082c6278e8c",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'EndpointConfigArn': 'arn:aws:sagemaker:us-west-2:687912291502:endpoint-config/baichuan-2023-06-16-08-32-34-825-config',\n",
       " 'ResponseMetadata': {'RequestId': '1c23d527-1c70-44bd-9ea4-04c1323719cc',\n",
       "  'HTTPStatusCode': 200,\n",
       "  'HTTPHeaders': {'x-amzn-requestid': '1c23d527-1c70-44bd-9ea4-04c1323719cc',\n",
       "   'content-type': 'application/x-amz-json-1.1',\n",
       "   'content-length': '120',\n",
       "   'date': 'Fri, 16 Jun 2023 08:32:35 GMT'},\n",
       "  'RetryAttempts': 0}}"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "endpoint_config_name = f\"{model_name}-config\"\n",
    "endpoint_name = f\"{model_name}-endpoint\"\n",
    "\n",
    "\n",
    "endpoint_config_response = sm_client.create_endpoint_config(\n",
    "    EndpointConfigName=endpoint_config_name,\n",
    "    ProductionVariants=[\n",
    "        {\n",
    "            \"VariantName\": \"variant1\",\n",
    "            \"ModelName\": model_name,\n",
    "            \"InstanceType\": \"ml.g5.12xlarge\",\n",
    "            \"InitialInstanceCount\": 1,\n",
    "            # \"VolumeSizeInGB\" : 400,\n",
    "            # \"ModelDataDownloadTimeoutInSeconds\": 2400,\n",
    "            \"ContainerStartupHealthCheckTimeoutInSeconds\": 15*60,\n",
    "        },\n",
    "    ],\n",
    ")\n",
    "endpoint_config_response"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "734a39b0-473e-4421-94c8-74d2b4105038",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Created Endpoint: arn:aws:sagemaker:us-west-2:687912291502:endpoint/baichuan-2023-06-16-08-32-34-825-endpoint\n"
     ]
    }
   ],
   "source": [
    "create_endpoint_response = sm_client.create_endpoint(\n",
    "    EndpointName=f\"{endpoint_name}\", EndpointConfigName=endpoint_config_name\n",
    ")\n",
    "print(f\"Created Endpoint: {create_endpoint_response['EndpointArn']}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1262e826-a810-401d-a5a9-f62febb24e5f",
   "metadata": {},
   "source": [
    "#### 持续检测模型部署进度"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "08969928-6b9e-4d9c-a033-a31f5f77bdfb",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Status: Creating\n",
      "Status: Creating\n",
      "Status: Creating\n",
      "Status: Creating\n",
      "Status: Creating\n",
      "Status: Creating\n",
      "Status: Creating\n",
      "Status: Creating\n",
      "Status: Creating\n",
      "Status: Creating\n",
      "Status: InService\n",
      "Arn: arn:aws:sagemaker:us-west-2:687912291502:endpoint/baichuan-2023-06-16-08-32-34-825-endpoint\n",
      "Status: InService\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "\n",
    "resp = sm_client.describe_endpoint(EndpointName=endpoint_name)\n",
    "status = resp[\"EndpointStatus\"]\n",
    "print(\"Status: \" + status)\n",
    "\n",
    "while status == \"Creating\":\n",
    "    time.sleep(60)\n",
    "    resp = sm_client.describe_endpoint(EndpointName=endpoint_name)\n",
    "    status = resp[\"EndpointStatus\"]\n",
    "    print(\"Status: \" + status)\n",
    "\n",
    "print(\"Arn: \" + resp[\"EndpointArn\"])\n",
    "print(\"Status: \" + status)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d985b427-3959-46f7-9a50-5a2b45e2d513",
   "metadata": {
    "tags": []
   },
   "source": [
    "### 5. 模型测试"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "bd86caf8-01e9-41eb-9199-478ba471a02b",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'{\\n  \"outputs\":\"好累啊,我该怎么办?\\\\n我今年20岁了,在一家公司做文员,工资不高,工作很轻松,但是每天上班都好无聊,同事们也都不怎么说话,我也不想和他们说话,因为我觉得他们都是些没素质的人,而且还很自私,我不想和他们一起工作,可是又不知道该做什么,我该怎么办?\\\\n你应该多和同事交流,这样才能了解他们,知道他们的性格,这样你才可以更好的相处。\"\\n}'"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "endpoint_name = \"baichuan-2023-06-16-08-32-34-825-endpoint\"\n",
    "prompts1 = \"\"\"\n",
    "你是MySQL的专家。给定一个输入问题，创建一个语法正确的MySQL查询语句。\n",
    "除非用户在问题中指定了要获得的特定数量的示例，否则使用LIMIT子句查询最多3个结果。您可以对结果进行排序，以返回数据库中信息量最大的数据。您必须仅查询回答问题所需的列。将每个列名用反引号（`）括起来，表示为分隔的标识符。\n",
    "请注意，仅可以使用在下面这些表中看到的列名，不要查询不存在的列。此外，还要注意哪个列在哪个表中。如果问题涉及”今天”，请注意使用CURDATE()函数获取当前日期.\n",
    "\n",
    "使用如下格式:\n",
    "Question: 具体的问题\n",
    "SQLQuery: 运行的sql语句\n",
    "SQLResult: SQLQuery运行的结果\n",
    "Answer: 最终的回答\n",
    "\n",
    "\n",
    "使用如下的表:\n",
    "CREATE TABLE customer (\n",
    "\tc_customer_sk INTEGER NOT NULL, \n",
    "\tc_customer_id CHAR(16) NOT NULL, \n",
    "\tc_current_cdemo_sk INTEGER, \n",
    "\tc_current_hdemo_sk INTEGER, \n",
    "\tc_current_addr_sk INTEGER, \n",
    "\tc_first_shipto_date_sk INTEGER, \n",
    "\tc_first_sales_date_sk INTEGER, \n",
    "\tc_salutation CHAR(10), \n",
    "\tc_first_name CHAR(20), \n",
    "\tc_last_name CHAR(30), \n",
    "\tc_preferred_cust_flag CHAR(1), \n",
    "\tc_birth_day INTEGER, \n",
    "\tc_birth_month INTEGER, \n",
    "\tc_birth_year INTEGER, \n",
    "\tc_birth_country VARCHAR(20), \n",
    "\tc_login CHAR(13), \n",
    "\tc_email_address CHAR(50), \n",
    "\tc_last_review_date CHAR(10), \n",
    "\tPRIMARY KEY (c_customer_sk)\n",
    ")ENGINE=InnoDB DEFAULT CHARSET=utf8\n",
    "\n",
    "\n",
    "CREATE TABLE web_sales (\n",
    "\tws_sold_date_sk INTEGER, \n",
    "\tws_sold_time_sk INTEGER, \n",
    "\tws_ship_date_sk INTEGER, \n",
    "\tws_item_sk INTEGER NOT NULL, \n",
    "\tws_bill_customer_sk INTEGER, \n",
    "\tws_bill_cdemo_sk INTEGER, \n",
    "\tws_bill_hdemo_sk INTEGER, \n",
    "\tws_bill_addr_sk INTEGER, \n",
    "\tws_ship_customer_sk INTEGER, \n",
    "\tws_ship_cdemo_sk INTEGER, \n",
    "\tws_ship_hdemo_sk INTEGER, \n",
    "\tws_ship_addr_sk INTEGER, \n",
    "\tws_web_page_sk INTEGER, \n",
    "\tws_web_site_sk INTEGER, \n",
    "\tws_ship_mode_sk INTEGER, \n",
    "\tws_warehouse_sk INTEGER, \n",
    "\tws_promo_sk INTEGER, \n",
    "\tws_order_number INTEGER NOT NULL, \n",
    "\tws_quantity INTEGER, \n",
    "\tws_wholesale_cost DECIMAL(7, 2), \n",
    "\tws_list_price DECIMAL(7, 2), \n",
    "\tws_sales_price DECIMAL(7, 2), \n",
    "\tws_ext_discount_amt DECIMAL(7, 2), \n",
    "\tws_ext_sales_price DECIMAL(7, 2), \n",
    "\tws_ext_wholesale_cost DECIMAL(7, 2), \n",
    "\tws_ext_list_price DECIMAL(7, 2), \n",
    "\tws_ext_tax DECIMAL(7, 2), \n",
    "\tws_coupon_amt DECIMAL(7, 2), \n",
    "\tws_ext_ship_cost DECIMAL(7, 2), \n",
    "\tws_net_paid DECIMAL(7, 2), \n",
    "\tws_net_paid_inc_tax DECIMAL(7, 2), \n",
    "\tws_net_paid_inc_ship DECIMAL(7, 2), \n",
    "\tws_net_paid_inc_ship_tax DECIMAL(7, 2), \n",
    "\tws_net_profit DECIMAL(7, 2), \n",
    "\tPRIMARY KEY (ws_item_sk, ws_order_number)\n",
    ")ENGINE=InnoDB DEFAULT CHARSET=utf8\n",
    "\n",
    "Question: 我需要知道销售报表中，下单金额最大的客户email地址\n",
    "\"\"\"\n",
    "\n",
    "prompts2=\"给我一个青海和甘肃旅游的路线，8天7晚\"\n",
    "prompts3=\"好累啊\"\n",
    "parameters={\n",
    "    \"do_sample\": False,\n",
    "    \"top_p\": 0.9,\n",
    "    \"temperature\": 1,\n",
    "    \"max_new_tokens\": 300,\n",
    "    \"repetition_penalty\": 1.03\n",
    "}\n",
    "response_model = smr_client.invoke_endpoint(\n",
    "            EndpointName=endpoint_name,\n",
    "            Body=json.dumps(\n",
    "            {\n",
    "                \"inputs\": prompts3,\n",
    "                \"parameters\": parameters,\n",
    "            }\n",
    "            ),\n",
    "            ContentType=\"application/json\",\n",
    "        )\n",
    "\n",
    "response_model['Body'].read().decode(\"utf-8\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "21c8b703-e312-4964-8be9-a754468e07cd",
   "metadata": {},
   "source": [
    "#### 清除模型Endpoint和config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f70d116f-4fb1-4f04-8732-3d6e4fb520de",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!aws sagemaker delete-endpoint --endpoint-name chatglm-2023-04-27-05-49-59-117-endpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "184e4d1d-3d62-43df-9b17-5d64ece928bd",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!aws sagemaker delete-endpoint-config --endpoint-config-name chatglm-2023-04-27-05-49-59-117-config"
   ]
  }
 ],
 "metadata": {
  "availableInstances": [
   {
    "_defaultOrder": 0,
    "_isFastLaunch": true,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 4,
    "name": "ml.t3.medium",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 1,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.t3.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 2,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.t3.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 3,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.t3.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 4,
    "_isFastLaunch": true,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.m5.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 5,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.m5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 6,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.m5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 7,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.m5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 8,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.m5.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 9,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.m5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 10,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.m5.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 11,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.m5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 12,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.m5d.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 13,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.m5d.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 14,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.m5d.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 15,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.m5d.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 16,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.m5d.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 17,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.m5d.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 18,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.m5d.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 19,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.m5d.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 20,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": true,
    "memoryGiB": 0,
    "name": "ml.geospatial.interactive",
    "supportedImageNames": [
     "sagemaker-geospatial-v1-0"
    ],
    "vcpuNum": 0
   },
   {
    "_defaultOrder": 21,
    "_isFastLaunch": true,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 4,
    "name": "ml.c5.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 22,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.c5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 23,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.c5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 24,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.c5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 25,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 72,
    "name": "ml.c5.9xlarge",
    "vcpuNum": 36
   },
   {
    "_defaultOrder": 26,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 96,
    "name": "ml.c5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 27,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 144,
    "name": "ml.c5.18xlarge",
    "vcpuNum": 72
   },
   {
    "_defaultOrder": 28,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.c5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 29,
    "_isFastLaunch": true,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.g4dn.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 30,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.g4dn.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 31,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.g4dn.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 32,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.g4dn.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 33,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.g4dn.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 34,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.g4dn.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 35,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 61,
    "name": "ml.p3.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 36,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 244,
    "name": "ml.p3.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 37,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 488,
    "name": "ml.p3.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 38,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 768,
    "name": "ml.p3dn.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 39,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.r5.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 40,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.r5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 41,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.r5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 42,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.r5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 43,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.r5.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 44,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.r5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 45,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 512,
    "name": "ml.r5.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 46,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 768,
    "name": "ml.r5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 47,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.g5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 48,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.g5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 49,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.g5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 50,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.g5.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 51,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.g5.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 52,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.g5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 53,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.g5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 54,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 768,
    "name": "ml.g5.48xlarge",
    "vcpuNum": 192
   }
  ],
  "instance_type": "ml.m5.large",
  "kernelspec": {
   "display_name": "conda_pytorch_p39",
   "language": "python",
   "name": "conda_pytorch_p39"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}