sagemaker-training.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "f4416c96",
   "metadata": {},
   "source": [
    "# An sample to finetune vicuna on SageMaker"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "95febd44",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "## Update sagemaker python sdk version\n",
    "!pip install -U sagemaker"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "158e94ab",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:botocore.credentials:Found credentials from IAM Role: BaseNotebookInstanceEc2InstanceRole\n",
      "INFO:botocore.credentials:Found credentials from IAM Role: BaseNotebookInstanceEc2InstanceRole\n"
     ]
    }
   ],
   "source": [
    "import sagemaker\n",
    "import boto3\n",
    "from sagemaker import get_execution_role\n",
    "\n",
    "sess = sagemaker.Session()\n",
    "role = get_execution_role()\n",
    "sagemaker_default_bucket = sess.default_bucket()\n",
    "\n",
    "account = sess.boto_session.client(\"sts\").get_caller_identity()[\"Account\"]\n",
    "region = sess.boto_session.region_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "82a0c829",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cloning into 'FastChat'...\n",
      "remote: Enumerating objects: 3587, done.\u001b[K\n",
      "remote: Counting objects: 100% (1627/1627), done.\u001b[K\n",
      "remote: Compressing objects: 100% (461/461), done.\u001b[K\n",
      "remote: Total 3587 (delta 1431), reused 1222 (delta 1164), pack-reused 1960\u001b[K\n",
      "Receiving objects: 100% (3587/3587), 30.06 MiB | 38.24 MiB/s, done.\n",
      "Resolving deltas: 100% (2519/2519), done.\n"
     ]
    }
   ],
   "source": [
    "## download training script from github\n",
    "!rm -rf ./FastChat\n",
    "!git clone https://github.com/lm-sys/FastChat.git\n",
    "!cp ./s5cmd ./FastChat/"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2bc5a450",
   "metadata": {},
   "source": [
    "## Download pretrained model from HuggingFace Hub"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a2513fac",
   "metadata": {},
   "source": [
    "To avoid download model from Huggingface hub failure, we download first and push those model files to S3 bucket first."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "88f942fc",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Looking in indexes: https://pypi.org/simple, https://pip.repos.neuron.amazonaws.com\n",
      "Requirement already satisfied: huggingface_hub in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (0.14.1)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from huggingface_hub) (6.0)\n",
      "Requirement already satisfied: filelock in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from huggingface_hub) (3.6.0)\n",
      "Requirement already satisfied: tqdm>=4.42.1 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from huggingface_hub) (4.63.2)\n",
      "Requirement already satisfied: requests in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from huggingface_hub) (2.28.1)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from huggingface_hub) (4.4.0)\n",
      "Requirement already satisfied: fsspec in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from huggingface_hub) (2022.11.0)\n",
      "Requirement already satisfied: packaging>=20.9 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from huggingface_hub) (21.3)\n",
      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from packaging>=20.9->huggingface_hub) (3.0.9)\n",
      "Requirement already satisfied: charset-normalizer<3,>=2 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from requests->huggingface_hub) (2.1.1)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from requests->huggingface_hub) (2022.12.7)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from requests->huggingface_hub) (1.26.8)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from requests->huggingface_hub) (3.4)\n"
     ]
    }
   ],
   "source": [
    "!pip install huggingface_hub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "0cd25c4c",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.009912490844726562,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": null,
       "postfix": null,
       "prefix": "Fetching 10 files",
       "rate": null,
       "total": 10,
       "unit": "it",
       "unit_divisor": 1000,
       "unit_scale": false
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dbeec60bec674d3b8d360926f608f760",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Fetching 10 files:   0%|          | 0/10 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.010442972183227539,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": null,
       "postfix": null,
       "prefix": "Downloading (…)l-00002-of-00002.bin",
       "rate": null,
       "total": 3500315539,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d80b724d350b45bf885f763bb3a17cb7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)l-00002-of-00002.bin:   0%|          | 0.00/3.50G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.009791135787963867,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": null,
       "postfix": null,
       "prefix": "Downloading (…)l-00001-of-00002.bin",
       "rate": null,
       "total": 9976634558,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5bb11e6cae824660bebe86c3fc6cdb61",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)l-00001-of-00002.bin:   0%|          | 0.00/9.98G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.006239652633666992,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": null,
       "postfix": null,
       "prefix": "Downloading (…)cial_tokens_map.json",
       "rate": null,
       "total": 2,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2bce6c3b0ee045c59a8ad8b1ea4f3d7b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)cial_tokens_map.json:   0%|          | 0.00/2.00 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.006548643112182617,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": null,
       "postfix": null,
       "prefix": "Downloading (…)121a4ca0/config.json",
       "rate": null,
       "total": 472,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f7dd8637ca0c48cfbf7a5c63081bbfa6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)121a4ca0/config.json:   0%|          | 0.00/472 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.006217479705810547,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": null,
       "postfix": null,
       "prefix": "Downloading (…)model.bin.index.json",
       "rate": null,
       "total": 26788,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8668505c392c436092bad684ade074a4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)model.bin.index.json:   0%|          | 0.00/26.8k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.010134458541870117,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": null,
       "postfix": null,
       "prefix": "Downloading (…)a4ca0/.gitattributes",
       "rate": null,
       "total": 1477,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b509d826c1eb43889391502705c61c8e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)a4ca0/.gitattributes:   0%|          | 0.00/1.48k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.008573770523071289,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": null,
       "postfix": null,
       "prefix": "Downloading (…)neration_config.json",
       "rate": null,
       "total": 137,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "90f865d163384604bd68365d925b6440",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)neration_config.json:   0%|          | 0.00/137 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.007856369018554688,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": null,
       "postfix": null,
       "prefix": "Downloading (…)81121a4ca0/README.md",
       "rate": null,
       "total": 28,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "78d9e46910c44b4f91ec4c76addf0098",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)81121a4ca0/README.md:   0%|          | 0.00/28.0 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.007498979568481445,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": null,
       "postfix": null,
       "prefix": "Downloading tokenizer.model",
       "rate": null,
       "total": 499723,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bd877c280fa04f4c847272751e7acead",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading tokenizer.model:   0%|          | 0.00/500k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.008463382720947266,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": null,
       "postfix": null,
       "prefix": "Downloading (…)okenizer_config.json",
       "rate": null,
       "total": 141,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "735ddd3b77b849d8976f99b6c5328129",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)okenizer_config.json:   0%|          | 0.00/141 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from huggingface_hub import snapshot_download\n",
    "from pathlib import Path\n",
    "\n",
    "local_cache_path = Path(\"./model\")\n",
    "local_cache_path.mkdir(exist_ok=True)\n",
    "\n",
    "model_name = \"pinkmanlove/llama-7b-hf\"#decapoda-research/llama-13b-hf\n",
    "\n",
    "# Only download pytorch checkpoint files\n",
    "allow_patterns = [\"*.json\", \"*.pt\", \"*.bin\", \"*.model\"]\n",
    "\n",
    "model_download_path = snapshot_download(\n",
    "    repo_id=model_name,\n",
    "    cache_dir=local_cache_path,\n",
    "    #allow_patterns=allow_patterns,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a40a2647",
   "metadata": {},
   "source": [
    "**Upload model files to S3**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "918df42f",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "./model/models--pinkmanlove--llama-7b-hf/snapshots/b3cde76468bad3c085ead29707ee7481121a4ca0/config.json\n",
      "./model/models--pinkmanlove--llama-7b-hf/snapshots/b3cde76468bad3c085ead29707ee7481121a4ca0/\n"
     ]
    }
   ],
   "source": [
    "# Get the model files path\n",
    "import os\n",
    "from glob import glob\n",
    "\n",
    "local_model_path = None\n",
    "\n",
    "paths = os.walk(r'./model')\n",
    "for root, dirs, files in paths:\n",
    "    for file in files:\n",
    "        if file == 'config.json':\n",
    "            print(os.path.join(root,file))\n",
    "            local_model_path = str(os.path.join(root,file))[0:-11]\n",
    "            print(local_model_path)\n",
    "if local_model_path == None:\n",
    "    print(\"Model download may failed, please check prior step!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "38de4454",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Name:\n",
      "  sync - sync objects\n",
      "\n",
      "Usage:\n",
      "  sync [options] source destination\n",
      "\n",
      "Options:\n",
      "  --delete                       delete objects in destination but not in source (default: false)\n",
      "  --size-only                    make size of object only criteria to decide whether an object should be synced (default: false)\n",
      "  --no-follow-symlinks           do not follow symbolic links (default: false)\n",
      "  --storage-class value          set storage class for target ('STANDARD','REDUCED_REDUNDANCY','GLACIER','STANDARD_IA','ONEZONE_IA','INTELLIGENT_TIERING','DEEP_ARCHIVE')\n",
      "  --concurrency value, -c value  number of concurrent parts transferred between host and remote server (default: 5)\n",
      "  --part-size value, -p value    size of each part transferred between host and remote server, in MiB (default: 50)\n",
      "  --sse value                    perform server side encryption of the data at its destination, e.g. aws:kms\n",
      "  --sse-kms-key-id value         customer master key (CMK) id for SSE-KMS encryption; leave it out if server-side generated key is desired\n",
      "  --acl value                    set acl for target: defines granted accesses and their types on different accounts/groups, e.g. cp --acl 'public-read'\n",
      "  --cache-control value          set cache control for target: defines cache control header for object, e.g. cp --cache-control 'public, max-age=345600'\n",
      "  --expires value                set expires for target (uses RFC3339 format): defines expires header for object, e.g. cp  --expires '2024-10-01T20:30:00Z'\n",
      "  --force-glacier-transfer       force transfer of glacier objects whether they are restored or not (default: false)\n",
      "  --ignore-glacier-warnings      turns off glacier warnings: ignore errors encountered during copying, downloading and moving glacier objects (default: false)\n",
      "  --source-region value          set the region of source bucket; the region of the source bucket will be automatically discovered if --source-region is not specified\n",
      "  --destination-region value     set the region of destination bucket: the region of the destination bucket will be automatically discovered if --destination-region is not specified\n",
      "  --exclude value                exclude objects with given pattern\n",
      "  --raw                          disable the wildcard operations, useful with filenames that contains glob characters (default: false)\n",
      "  --help, -h                     show help (default: false)\n",
      "  \n",
      "Examples:\n",
      "  01. Sync local folder to s3 bucket\n",
      "     > s5cmd sync folder/ s3://bucket/\n",
      "\n",
      "  02. Sync S3 bucket to local folder\n",
      "     > s5cmd sync s3://bucket/* folder/\n",
      "\n",
      "  03. Sync S3 bucket objects under prefix to S3 bucket.\n",
      "     > s5cmd sync s3://sourcebucket/prefix/* s3://destbucket/\n",
      "\n",
      "  04. Sync local folder to S3 but delete the files that S3 bucket has but local does not have.\n",
      "     > s5cmd sync --delete folder/ s3://bucket/\n",
      "\n",
      "  05. Sync S3 bucket to local folder but use size as only comparison criteria.\n",
      "     > s5cmd sync --size-only s3://bucket/* folder/\n",
      "\n",
      "  06. Sync a file to S3 bucket\n",
      "     > s5cmd sync myfile.gz s3://bucket/\n",
      "\n",
      "  07. Sync matching S3 objects to another bucket\n",
      "     > s5cmd sync s3://bucket/*.gz s3://target-bucket/prefix/\n",
      "\n",
      "  08. Perform KMS Server Side Encryption of the object(s) at the destination\n",
      "     > s5cmd sync --sse aws:kms s3://bucket/object s3://target-bucket/prefix/object\n",
      "\n",
      "  09. Perform KMS-SSE of the object(s) at the destination using customer managed Customer Master Key (CMK) key id\n",
      "     > s5cmd sync --sse aws:kms --sse-kms-key-id <your-kms-key-id> s3://bucket/object s3://target-bucket/prefix/object\n",
      "\n",
      "  10. Sync all files to S3 bucket but exclude the ones with txt and gz extension\n",
      "     > s5cmd sync --exclude \"*.txt\" --exclude \"*.gz\" dir/ s3://bucket\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR \"sync $local_model_path s3://$sagemaker_default_bucket/llama/pretrain/pinkmanlove/llama-7b-hf/\": given object not found\n"
     ]
    }
   ],
   "source": [
    "%%script env sagemaker_default_bucket=$sagemaker_default_bucket local_model_path=$local_model_path bash\n",
    "\n",
    "chmod +x ./s5cmd\n",
    "./s5cmd sync ${local_model_path} s3://${sagemaker_default_bucket}/llama/pretrain/pinkmanlove/llama-7b-hf/ \n",
    "\n",
    "rm -rf model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9588d498",
   "metadata": {},
   "source": [
    "## Prepare docker image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "d2057f24",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting Dockerfile\n"
     ]
    }
   ],
   "source": [
    "%%writefile Dockerfile\n",
    "## You should change below region code to the region you used, here sample is use us-west-2\n",
    "From 763104351884.dkr.ecr.us-west-2.amazonaws.com/huggingface-pytorch-training:1.13.1-transformers4.26.0-gpu-py39-cu117-ubuntu20.04 \n",
    "#From pytorch/pytorch:1.5-cuda10.1-cudnn7-runtime\n",
    "\n",
    "ENV LANG=C.UTF-8\n",
    "ENV PYTHONUNBUFFERED=TRUE\n",
    "ENV PYTHONDONTWRITEBYTECODE=TRUE\n",
    "\n",
    "# RUN python3 -m pip install git+https://github.com/huggingface/transformers.git@97a3d16a6941294d7d76d24f36f26617d224278e\n",
    "\n",
    "RUN pip3 uninstall -y deepspeed && pip3 install deepspeed\n",
    "RUN python3 -m pip install transformers==4.28.0\n",
    "\n",
    "\n",
    "## Make all local GPUs visible\n",
    "ENV NVIDIA_VISIBLE_DEVICES=\"all\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "3b8ee553",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING! Your password will be stored unencrypted in /home/ec2-user/.docker/config.json.\n",
      "Configure a credential helper to remove this warning. See\n",
      "https://docs.docker.com/engine/reference/commandline/login/#credentials-store\n",
      "\n",
      "Login Succeeded\n"
     ]
    }
   ],
   "source": [
    "## You should change below region code to the region you used, here sample is use us-west-2\n",
    "!aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin 763104351884.dkr.ecr.us-west-2.amazonaws.com"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "985b0fb6",
   "metadata": {},
   "source": [
    "**Build image and push to ECR.**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "53800617",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "## define repo name, should contain *sagemaker* in the name\n",
    "repo_name = \"sagemaker-vicuna-demo\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "a9d98c7c",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Login Succeeded\n",
      "Sending build context to Docker daemon  68.01GB\n",
      "Step 1/7 : From 763104351884.dkr.ecr.us-west-2.amazonaws.com/huggingface-pytorch-training:1.13.1-transformers4.26.0-gpu-py39-cu117-ubuntu20.04\n",
      " ---> c5a6ef695006\n",
      "Step 2/7 : ENV LANG=C.UTF-8\n",
      " ---> Using cache\n",
      " ---> af49cfa7feae\n",
      "Step 3/7 : ENV PYTHONUNBUFFERED=TRUE\n",
      " ---> Using cache\n",
      " ---> 287106637dc6\n",
      "Step 4/7 : ENV PYTHONDONTWRITEBYTECODE=TRUE\n",
      " ---> Using cache\n",
      " ---> 773b4cf30c90\n",
      "Step 5/7 : RUN pip3 uninstall -y deepspeed && pip3 install deepspeed\n",
      " ---> Using cache\n",
      " ---> ce72201e73cd\n",
      "Step 6/7 : RUN python3 -m pip install transformers==4.28.0\n",
      " ---> Using cache\n",
      " ---> e234794bbe5c\n",
      "Step 7/7 : ENV NVIDIA_VISIBLE_DEVICES=\"all\"\n",
      " ---> Using cache\n",
      " ---> bb43a66885f0\n",
      "Successfully built bb43a66885f0\n",
      "Successfully tagged sagemaker-vicuna-demo:latest\n",
      "The push refers to repository [687912291502.dkr.ecr.us-west-2.amazonaws.com/sagemaker-vicuna-demo]\n",
      "1e9d9d5ddefd: Preparing\n",
      "02a87473f68b: Preparing\n",
      "f8dae5c3df1e: Preparing\n",
      "e3221f18601a: Preparing\n",
      "b6f286626882: Preparing\n",
      "76fe97d80cdb: Preparing\n",
      "f5f76489fff8: Preparing\n",
      "621c3f07daa7: Preparing\n",
      "9b484bb42e11: Preparing\n",
      "54c7c0b58471: Preparing\n",
      "c34adc3ab668: Preparing\n",
      "bbf651e48b84: Preparing\n",
      "f61045791108: Preparing\n",
      "4e2ac0cda74a: Preparing\n",
      "f5f76489fff8: Waiting\n",
      "658a33d555eb: Preparing\n",
      "621c3f07daa7: Waiting\n",
      "bd16d9a61a98: Preparing\n",
      "f0c0cd2accfa: Preparing\n",
      "9b484bb42e11: Waiting\n",
      "1275469c066c: Preparing\n",
      "bbf651e48b84: Waiting\n",
      "b802dd3babf4: Preparing\n",
      "54c7c0b58471: Waiting\n",
      "a3834ec63558: Preparing\n",
      "f61045791108: Waiting\n",
      "63edcef6dedf: Preparing\n",
      "4e2ac0cda74a: Waiting\n",
      "0154e84cc2dd: Preparing\n",
      "7085d1c151f6: Preparing\n",
      "a77a2104cfb6: Preparing\n",
      "6808e7f9da2f: Preparing\n",
      "3bc059a9dec6: Preparing\n",
      "de783f3fec23: Preparing\n",
      "c34adc3ab668: Waiting\n",
      "18ca52d74b2f: Preparing\n",
      "73df6ccd636c: Preparing\n",
      "6738b73ff7a8: Preparing\n",
      "2a8292d9bfcc: Preparing\n",
      "76fe97d80cdb: Waiting\n",
      "5b75a5ef32a7: Preparing\n",
      "25a5f55a11f0: Preparing\n",
      "707f484816ae: Preparing\n",
      "bd16d9a61a98: Waiting\n",
      "b802dd3babf4: Waiting\n",
      "0430aa1e47d4: Preparing\n",
      "65448e793131: Preparing\n",
      "1275469c066c: Waiting\n",
      "a3834ec63558: Waiting\n",
      "15af6e2d42ba: Preparing\n",
      "63edcef6dedf: Waiting\n",
      "0154e84cc2dd: Waiting\n",
      "b46caef92993: Preparing\n",
      "53ce33a12646: Preparing\n",
      "3bc059a9dec6: Waiting\n",
      "aad68760f4ce: Preparing\n",
      "6808e7f9da2f: Waiting\n",
      "18ca52d74b2f: Waiting\n",
      "7085d1c151f6: Waiting\n",
      "323d67ab1719: Preparing\n",
      "73df6ccd636c: Waiting\n",
      "de783f3fec23: Waiting\n",
      "e72743a0fdfe: Preparing\n",
      "a77a2104cfb6: Waiting\n",
      "3996353f5820: Preparing\n",
      "0430aa1e47d4: Waiting\n",
      "ea87e0b9c30f: Preparing\n",
      "af18356cdf10: Preparing\n",
      "b46caef92993: Waiting\n",
      "65448e793131: Waiting\n",
      "6738b73ff7a8: Waiting\n",
      "2a8292d9bfcc: Waiting\n",
      "f6e30dd4497e: Preparing\n",
      "99832d04a153: Preparing\n",
      "15af6e2d42ba: Waiting\n",
      "5b75a5ef32a7: Waiting\n",
      "a5981ed7a378: Preparing\n",
      "250519a2f830: Preparing\n",
      "707f484816ae: Waiting\n",
      "6cadbde53f94: Preparing\n",
      "0002c93bdb37: Preparing\n",
      "25a5f55a11f0: Waiting\n",
      "3996353f5820: Waiting\n",
      "658a33d555eb: Waiting\n",
      "aad68760f4ce: Waiting\n",
      "e72743a0fdfe: Waiting\n",
      "323d67ab1719: Waiting\n",
      "f6e30dd4497e: Waiting\n",
      "53ce33a12646: Waiting\n",
      "6cadbde53f94: Waiting\n",
      "0002c93bdb37: Waiting\n",
      "250519a2f830: Waiting\n",
      "ea87e0b9c30f: Waiting\n",
      "a5981ed7a378: Waiting\n",
      "f8dae5c3df1e: Layer already exists\n",
      "1e9d9d5ddefd: Layer already exists\n",
      "e3221f18601a: Layer already exists\n",
      "b6f286626882: Layer already exists\n",
      "02a87473f68b: Layer already exists\n",
      "76fe97d80cdb: Layer already exists\n",
      "f5f76489fff8: Layer already exists\n",
      "621c3f07daa7: Layer already exists\n",
      "54c7c0b58471: Layer already exists\n",
      "9b484bb42e11: Layer already exists\n",
      "c34adc3ab668: Layer already exists\n",
      "bbf651e48b84: Layer already exists\n",
      "f61045791108: Layer already exists\n",
      "4e2ac0cda74a: Layer already exists\n",
      "658a33d555eb: Layer already exists\n",
      "bd16d9a61a98: Layer already exists\n",
      "f0c0cd2accfa: Layer already exists\n",
      "1275469c066c: Layer already exists\n",
      "b802dd3babf4: Layer already exists\n",
      "a3834ec63558: Layer already exists\n",
      "63edcef6dedf: Layer already exists\n",
      "7085d1c151f6: Layer already exists\n",
      "0154e84cc2dd: Layer already exists\n",
      "a77a2104cfb6: Layer already exists\n",
      "6808e7f9da2f: Layer already exists\n",
      "3bc059a9dec6: Layer already exists\n",
      "de783f3fec23: Layer already exists\n",
      "18ca52d74b2f: Layer already exists\n",
      "73df6ccd636c: Layer already exists\n",
      "6738b73ff7a8: Layer already exists\n",
      "2a8292d9bfcc: Layer already exists\n",
      "5b75a5ef32a7: Layer already exists\n",
      "25a5f55a11f0: Layer already exists\n",
      "707f484816ae: Layer already exists\n",
      "0430aa1e47d4: Layer already exists\n",
      "65448e793131: Layer already exists\n",
      "15af6e2d42ba: Layer already exists\n",
      "b46caef92993: Layer already exists\n",
      "53ce33a12646: Layer already exists\n",
      "aad68760f4ce: Layer already exists\n",
      "323d67ab1719: Layer already exists\n",
      "e72743a0fdfe: Layer already exists\n",
      "3996353f5820: Layer already exists\n",
      "ea87e0b9c30f: Layer already exists\n",
      "f6e30dd4497e: Layer already exists\n",
      "af18356cdf10: Layer already exists\n",
      "99832d04a153: Layer already exists\n",
      "a5981ed7a378: Layer already exists\n",
      "0002c93bdb37: Layer already exists\n",
      "250519a2f830: Layer already exists\n",
      "6cadbde53f94: Layer already exists\n",
      "latest: digest: sha256:f6e450c5c980f1e2580625eb16a3221a1583d3e81114287d6d4aa668f7df8d47 size: 11044\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING! Your password will be stored unencrypted in /home/ec2-user/.docker/config.json.\n",
      "Configure a credential helper to remove this warning. See\n",
      "https://docs.docker.com/engine/reference/commandline/login/#credentials-store\n",
      "\n"
     ]
    }
   ],
   "source": [
    "%%script env repo_name=$repo_name bash\n",
    "\n",
    "#!/usr/bin/env bash\n",
    "\n",
    "# This script shows how to build the Docker image and push it to ECR to be ready for use\n",
    "# by SageMaker.\n",
    "\n",
    "# The argument to this script is the image name. This will be used as the image on the local\n",
    "# machine and combined with the account and region to form the repository name for ECR.\n",
    "# The name of our algorithm\n",
    "algorithm_name=${repo_name}\n",
    "\n",
    "account=$(aws sts get-caller-identity --query Account --output text)\n",
    "\n",
    "# Get the region defined in the current configuration (default to us-west-2 if none defined)\n",
    "region=$(aws configure get region)\n",
    "region=${region:-us-west-2}\n",
    "\n",
    "fullname=\"${account}.dkr.ecr.${region}.amazonaws.com/${algorithm_name}:latest\"\n",
    "\n",
    "# If the repository doesn't exist in ECR, create it.\n",
    "aws ecr describe-repositories --repository-names \"${algorithm_name}\" > /dev/null 2>&1\n",
    "\n",
    "if [ $? -ne 0 ]\n",
    "then\n",
    "    aws ecr create-repository --repository-name \"${algorithm_name}\" > /dev/null\n",
    "fi\n",
    "\n",
    "# Get the login command from ECR and execute it directly\n",
    "aws ecr get-login-password --region ${region}|docker login --username AWS --password-stdin ${fullname}\n",
    "\n",
    "# Build the docker image locally with the image name and then push it to ECR\n",
    "# with the full name.\n",
    "\n",
    "docker build -t ${algorithm_name} .\n",
    "docker tag ${algorithm_name} ${fullname}\n",
    "\n",
    "docker push ${fullname}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8dd6dc0d",
   "metadata": {},
   "source": [
    "### Generate the deepspeed config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "f7b56758",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting ds.json\n"
     ]
    }
   ],
   "source": [
    "%%writefile ds.json\n",
    "{\n",
    "  \"fp16\": {\n",
    "    \"enabled\": true,\n",
    "    \"auto_cast\": false,\n",
    "    \"loss_scale\": 0,\n",
    "    \"initial_scale_power\": 16,\n",
    "    \"loss_scale_window\": 1000,\n",
    "    \"hysteresis\": 2,\n",
    "    \"min_loss_scale\": 1\n",
    "  },\n",
    "  \"optimizer\": {\n",
    "    \"type\": \"AdamW\",\n",
    "    \"params\": {\n",
    "      \"lr\": \"auto\",\n",
    "      \"betas\": \"auto\",\n",
    "      \"eps\": \"auto\",\n",
    "      \"weight_decay\": \"auto\"\n",
    "    }\n",
    "  },\n",
    "  \"scheduler\": {\n",
    "    \"type\": \"WarmupLR\",\n",
    "    \"params\": {\n",
    "      \"warmup_min_lr\": \"auto\",\n",
    "      \"warmup_max_lr\": \"auto\",\n",
    "      \"warmup_num_steps\": \"auto\"\n",
    "    }\n",
    "  },\n",
    "  \"zero_optimization\": {\n",
    "    \"stage\": 3,\n",
    "    \"overlap_comm\": true,\n",
    "    \"contiguous_gradients\": true,\n",
    "    \"sub_group_size\": 1e9,\n",
    "    \"reduce_bucket_size\": \"auto\",\n",
    "    \"stage3_prefetch_bucket_size\": \"auto\",\n",
    "    \"stage3_param_persistence_threshold\": \"auto\",\n",
    "    \"stage3_max_live_parameters\": 1e9,\n",
    "    \"stage3_max_reuse_distance\": 1e9,\n",
    "    \"stage3_gather_16bit_weights_on_model_save\": true\n",
    "  },\n",
    "  \"gradient_accumulation_steps\": \"auto\",\n",
    "  \"gradient_clipping\": \"auto\",\n",
    "  \"steps_per_print\": 2000,\n",
    "  \"train_batch_size\": \"auto\",\n",
    "  \"train_micro_batch_size_per_gpu\": \"auto\",\n",
    "  \"wall_clock_breakdown\": false\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9e5fc567",
   "metadata": {},
   "source": [
    "**Generate training entrypoint script.**\n",
    "\n",
    "**Note: DO NOT CHANGE BELOW VAlUE OF \"output_dir\" and \"cache_dir\", keep it \"/tmp/llama_out\" and \"/tmp\".**"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "683e3d70",
   "metadata": {},
   "source": [
    "Below is just a testing to fine-tune on a sample dataset (just 8 samples), you could change ```data_path``` to your dataset for furthur fine tune.\n",
    "\n",
    "For the dataset download, you could follow the way how to download pretrain model:\n",
    "```\n",
    "./s5cmd sync s3://$MODEL_S3_BUCKET/llama/pretrain/7B/* /tmp/llama_pretrain/\n",
    "```\n",
    "\n",
    "It is recommend to use the folder ```/tmp/dataset/```."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "06c51767",
   "metadata": {},
   "source": [
    "## Notice\n",
    "\n",
    "We modified some parts of ```FastChat/fastchat/train/train.py```, such as how to save model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "f94daee8",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!mv FastChat/fastchat/train/train.py FastChat/fastchat/train/train_bak.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "286ccd3d",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting FastChat/fastchat/train/train.py\n"
     ]
    }
   ],
   "source": [
    "%%writefile FastChat/fastchat/train/train.py\n",
    "# Adopted from tatsu-lab@stanford_alpaca. Below is the original copyright:\n",
    "#    Copyright 2023 Rohan Taori, Ishaan Gulrajani, Tianyi Zhang, Yann Dubois, Xuechen Li\n",
    "#\n",
    "#    Licensed under the Apache License, Version 2.0 (the \"License\");\n",
    "#    you may not use this file except in compliance with the License.\n",
    "#    You may obtain a copy of the License at\n",
    "#\n",
    "#        http://www.apache.org/licenses/LICENSE-2.0\n",
    "#\n",
    "#    Unless required by applicable law or agreed to in writing, software\n",
    "#    distributed under the License is distributed on an \"AS IS\" BASIS,\n",
    "#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
    "#    See the License for the specific language governing permissions and\n",
    "#    limitations under the License.\n",
    "\n",
    "import copy\n",
    "from dataclasses import dataclass, field\n",
    "import json\n",
    "import pathlib\n",
    "from typing import Dict, Optional, Sequence\n",
    "\n",
    "import torch\n",
    "from torch.utils.data import Dataset\n",
    "import transformers\n",
    "from transformers import Trainer\n",
    "####\n",
    "from transformers import AutoModelForCausalLM, Trainer, TrainingArguments, AutoTokenizer\n",
    "from transformers.models.llama.tokenization_llama import LlamaTokenizer\n",
    "####\n",
    "from transformers.trainer_pt_utils import LabelSmoother\n",
    "\n",
    "from fastchat.conversation import get_conv_template, SeparatorStyle\n",
    "\n",
    "IGNORE_TOKEN_ID = LabelSmoother.ignore_index\n",
    "\n",
    "\n",
    "@dataclass\n",
    "class ModelArguments:\n",
    "    model_name_or_path: Optional[str] = field(default=\"facebook/opt-125m\")\n",
    "\n",
    "\n",
    "@dataclass\n",
    "class DataArguments:\n",
    "    data_path: str = field(default=None,\n",
    "                           metadata={\"help\": \"Path to the training data.\"})\n",
    "    lazy_preprocess: bool = False\n",
    "\n",
    "\n",
    "@dataclass\n",
    "class TrainingArguments(transformers.TrainingArguments):\n",
    "    cache_dir: Optional[str] = field(default=None)\n",
    "    optim: str = field(default=\"adamw_torch\")\n",
    "    model_max_length: int = field(\n",
    "        default=512,\n",
    "        metadata={\n",
    "            \"help\":\n",
    "            \"Maximum sequence length. Sequences will be right padded (and possibly truncated).\"\n",
    "        },\n",
    "    )\n",
    "\n",
    "\n",
    "local_rank = None\n",
    "\n",
    "\n",
    "def rank0_print(*args):\n",
    "    if local_rank == 0:\n",
    "        print(*args)\n",
    "\n",
    "\n",
    "def safe_save_model_for_hf_trainer(trainer: transformers.Trainer,\n",
    "                                   output_dir: str):\n",
    "    \"\"\"Collects the state dict and dump to disk.\"\"\"\n",
    "    state_dict = trainer.model.state_dict()\n",
    "    if trainer.args.should_save:\n",
    "        cpu_state_dict = {\n",
    "            key: value.cpu()\n",
    "            for key, value in state_dict.items()\n",
    "        }\n",
    "        del state_dict\n",
    "        trainer._save(output_dir, state_dict=cpu_state_dict)  # noqa\n",
    "\n",
    "\n",
    "def preprocess(\n",
    "    sources,\n",
    "    tokenizer: transformers.PreTrainedTokenizer,\n",
    ") -> Dict:\n",
    "    conv = get_conv_template(\"vicuna_v1.1\").copy()\n",
    "    roles = {\"human\": conv.roles[0], \"gpt\": conv.roles[1]}\n",
    "\n",
    "    # Apply prompt templates\n",
    "    conversations = []\n",
    "    for i, source in enumerate(sources):\n",
    "        if roles[source[0][\"from\"]] != conv.roles[0]:\n",
    "            # Skip the first one if it is not from human\n",
    "            source = source[1:]\n",
    "\n",
    "        conv.messages = []\n",
    "        for j, sentence in enumerate(source):\n",
    "            role = roles[sentence[\"from\"]]\n",
    "            assert role == conv.roles[j % 2], f\"{i}\"\n",
    "            conv.append_message(role, sentence[\"value\"])\n",
    "        conversations.append(conv.get_prompt())\n",
    "\n",
    "    # Tokenize conversations\n",
    "    input_ids = tokenizer(\n",
    "        conversations,\n",
    "        return_tensors=\"pt\",\n",
    "        padding=\"max_length\",\n",
    "        max_length=tokenizer.model_max_length,\n",
    "        truncation=True,\n",
    "    ).input_ids\n",
    "    targets = input_ids.clone()\n",
    "\n",
    "    assert conv.sep_style == SeparatorStyle.ADD_COLON_TWO\n",
    "\n",
    "    # Mask targets\n",
    "    sep = conv.sep + conv.roles[1] + \": \"\n",
    "    for conversation, target in zip(conversations, targets):\n",
    "        total_len = int(target.ne(tokenizer.pad_token_id).sum())\n",
    "\n",
    "        rounds = conversation.split(conv.sep2)\n",
    "        cur_len = 1\n",
    "        for i, rou in enumerate(rounds):\n",
    "            if rou == \"\":\n",
    "                break\n",
    "\n",
    "            parts = rou.split(sep)\n",
    "            if len(parts) != 2:\n",
    "                break\n",
    "            parts[0] += sep\n",
    "            round_len = len(tokenizer(rou).input_ids)\n",
    "            instruction_len = len(tokenizer(parts[0]).input_ids) - 2\n",
    "\n",
    "            target[cur_len:cur_len+instruction_len] = (\n",
    "                IGNORE_TOKEN_ID)\n",
    "\n",
    "            #rank0_print(tokenizer.decode(target[cur_len+instruction_len:cur_len+round_len]))\n",
    "\n",
    "            cur_len += round_len\n",
    "        target[cur_len:] = IGNORE_TOKEN_ID\n",
    "\n",
    "        if cur_len < tokenizer.model_max_length:\n",
    "            if cur_len != total_len:\n",
    "                rank0_print(f\"WARNING: tokenization mismatch \"\n",
    "                            f\"{cur_len} vs. {total_len}\")\n",
    "\n",
    "    return dict(input_ids=input_ids, labels=targets,\n",
    "                attention_mask=input_ids.ne(tokenizer.pad_token_id))\n",
    "\n",
    "\n",
    "class SupervisedDataset(Dataset):\n",
    "    \"\"\"Dataset for supervised fine-tuning.\"\"\"\n",
    "\n",
    "    def __init__(self, data_path: str,\n",
    "                 tokenizer: transformers.PreTrainedTokenizer):\n",
    "        super(SupervisedDataset, self).__init__()\n",
    "        rank0_print(\"Loading data...\")\n",
    "        list_data_dict = json.load(open(data_path, \"r\"))\n",
    "\n",
    "        rank0_print(\"Formatting inputs...\")\n",
    "        sources = [example[\"conversations\"] for example in list_data_dict]\n",
    "        data_dict = preprocess(sources, tokenizer)\n",
    "\n",
    "        self.input_ids = data_dict[\"input_ids\"]\n",
    "        self.labels = data_dict[\"labels\"]\n",
    "        self.attention_mask = data_dict[\"attention_mask\"]\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.input_ids)\n",
    "\n",
    "    def __getitem__(self, i) -> Dict[str, torch.Tensor]:\n",
    "        return dict(input_ids=self.input_ids[i],\n",
    "                    labels=self.labels[i],\n",
    "                    attention_mask=self.attention_mask[i])\n",
    "\n",
    "\n",
    "class LazySupervisedDataset(Dataset):\n",
    "    \"\"\"Dataset for supervised fine-tuning.\"\"\"\n",
    "\n",
    "    def __init__(self, data_path: str,\n",
    "                 tokenizer: transformers.PreTrainedTokenizer):\n",
    "        super(LazySupervisedDataset, self).__init__()\n",
    "        self.tokenizer = tokenizer\n",
    "\n",
    "        rank0_print(\"Loading data...\")\n",
    "        list_data_dict = json.load(open(data_path, \"r\"))\n",
    "\n",
    "        rank0_print(\"Formatting inputs...Skip in lazy mode\")\n",
    "        self.tokenizer = tokenizer\n",
    "        self.list_data_dict = list_data_dict\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.list_data_dict)\n",
    "\n",
    "    def __getitem__(self, i) -> Dict[str, torch.Tensor]:\n",
    "        sources = self.list_data_dict[i]\n",
    "        if isinstance(i, int):\n",
    "            sources = [sources]\n",
    "        data_dict = preprocess([e[\"conversations\"] for e in sources],\n",
    "            self.tokenizer)\n",
    "        if isinstance(i, int):\n",
    "            data_dict = dict(input_ids=data_dict[\"input_ids\"][0],\n",
    "                             labels=data_dict[\"labels\"][0],\n",
    "                             attention_mask=data_dict[\"attention_mask\"][0])\n",
    "        return data_dict\n",
    "\n",
    "\n",
    "def make_supervised_data_module(tokenizer: transformers.PreTrainedTokenizer,\n",
    "                                data_args) -> Dict:\n",
    "    \"\"\"Make dataset and collator for supervised fine-tuning.\"\"\"\n",
    "    dataset_cls = (LazySupervisedDataset\n",
    "                   if data_args.lazy_preprocess else SupervisedDataset)\n",
    "    train_dataset = dataset_cls(tokenizer=tokenizer,\n",
    "                                data_path=data_args.data_path)\n",
    "    return dict(train_dataset=train_dataset,\n",
    "                eval_dataset=None)\n",
    "\n",
    "\n",
    "def train():\n",
    "    global local_rank\n",
    "\n",
    "    parser = transformers.HfArgumentParser(\n",
    "        (ModelArguments, DataArguments, TrainingArguments))\n",
    "    model_args, data_args, training_args = parser.parse_args_into_dataclasses()\n",
    "    local_rank = training_args.local_rank\n",
    "    model = transformers.AutoModelForCausalLM.from_pretrained(\n",
    "        model_args.model_name_or_path,\n",
    "        cache_dir=training_args.cache_dir,\n",
    "    )\n",
    "    tokenizer = LlamaTokenizer.from_pretrained( #transformers.AutoTokenizer\n",
    "        model_args.model_name_or_path,\n",
    "        cache_dir=training_args.cache_dir,\n",
    "        model_max_length=training_args.model_max_length,\n",
    "        padding_side=\"right\",\n",
    "        use_fast=False,\n",
    "    )\n",
    "#####\n",
    "#     tokenizer.pad_token = tokenizer.unk_token\n",
    "    if tokenizer.pad_token is None:\n",
    "        print(\"-----------no pad token and add special token PAD----\")\n",
    "        tokenizer.add_special_tokens({'pad_token': '[PAD]'})\n",
    "        model.resize_token_embeddings(len(tokenizer))\n",
    "######\n",
    "    data_module = make_supervised_data_module(tokenizer=tokenizer,\n",
    "                                              data_args=data_args)\n",
    "    trainer = Trainer(model=model,\n",
    "                      tokenizer=tokenizer,\n",
    "                      args=training_args,\n",
    "                      **data_module)\n",
    "\n",
    "    if list(pathlib.Path(training_args.output_dir).glob(\"checkpoint-*\")):\n",
    "        trainer.train(resume_from_checkpoint=True)\n",
    "    else:\n",
    "        trainer.train()\n",
    "#     trainer.save_state()\n",
    "#     safe_save_model_for_hf_trainer(trainer=trainer,\n",
    "#                                    output_dir=training_args.output_dir)\n",
    "\n",
    "\n",
    "    tokenizer.save_pretrained(training_args.output_dir)\n",
    "    trainer.save_model(training_args.output_dir)\n",
    "\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    train()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6aef55e2",
   "metadata": {},
   "source": [
    "Here we use sample dataset - sharegpt_test.json for testing."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b5a3d106-42f5-4241-a8ae-000939f11c5e",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "### 单机多卡 deepspeed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "id": "55cedcb2",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting ./FastChat/ds-train.sh\n"
     ]
    }
   ],
   "source": [
    "%%writefile ./FastChat/ds-train.sh\n",
    "#!/bin/bash\n",
    "\n",
    "chmod +x ./s5cmd\n",
    "./s5cmd sync s3://$MODEL_S3_BUCKET/llama/pretrain/pinkmanlove/llama-7b-hf/* /tmp/llama_pretrain/\n",
    "\n",
    "#cd FastChat && pip install -e . && cd ..\n",
    "pip install -e .\n",
    "\n",
    "deepspeed --num_gpus=8 ./fastchat/train/train_mem.py \\\n",
    "    --deepspeed ds.json \\\n",
    "    --model_name_or_path \"/tmp/llama_pretrain/\" \\\n",
    "    --data_path data/dummy_conversation.json \\\n",
    "    --output_dir \"/tmp/llama_out\" \\\n",
    "    --num_train_epochs 1 \\\n",
    "    --per_device_train_batch_size 2 \\\n",
    "    --per_device_eval_batch_size  2 \\\n",
    "    --gradient_accumulation_steps 8 \\\n",
    "    --evaluation_strategy \"no\" \\\n",
    "    --save_strategy \"no\" \\\n",
    "    --save_steps 2000 \\\n",
    "    --save_total_limit 1 \\\n",
    "    --learning_rate 2e-5 \\\n",
    "    --weight_decay 0. \\\n",
    "    --warmup_ratio 0.03 \\\n",
    "    --lr_scheduler_type \"cosine\" \\\n",
    "    --logging_steps 1 \\\n",
    "    --cache_dir '/tmp' \\\n",
    "    --model_max_length 2048 \\\n",
    "    --gradient_checkpointing True \\\n",
    "    --lazy_preprocess True \\\n",
    "    --fp16 True \\\n",
    "    --tf32 True \\\n",
    "    --report_to \"none\"\n",
    "\n",
    "if [ $? -eq 1 ]; then\n",
    "    echo \"Training script error, please check CloudWatch logs\"\n",
    "    exit 1\n",
    "fi\n",
    "\n",
    "./s5cmd sync /tmp/llama_out s3://$MODEL_S3_BUCKET/llama/output/$(date +%Y-%m-%d-%H-%M-%S)/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "id": "517889b1",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'687912291502.dkr.ecr.us-west-2.amazonaws.com/sagemaker-vicuna-demo:latest'"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## The image uri which is build and pushed above\n",
    "image_uri = \"{}.dkr.ecr.{}.amazonaws.com/{}:latest\".format(account, region, repo_name)\n",
    "image_uri"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "id": "ad795754",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "## set train_data_path to your training dataset path in s3\n",
    "train_data_path = f's3://{sagemaker_default_bucket}/llama/train_data/'\n",
    "\n",
    "inputs = {'train': train_data_path}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8ab36100",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "import time\n",
    "from sagemaker.estimator import Estimator\n",
    "\n",
    "environment = {\n",
    "              'MODEL_S3_BUCKET': sagemaker_default_bucket # The bucket to store pretrained model and fine-tune model\n",
    "}\n",
    "\n",
    "base_job_name = 'vicuna-demo'         \n",
    "\n",
    "instance_type = 'ml.p4d.24xlarge'\n",
    "\n",
    "estimator = Estimator(role=role,\n",
    "                      entry_point='ds-train.sh',\n",
    "                      source_dir='./FastChat/',\n",
    "                      base_job_name=base_job_name,\n",
    "                      instance_count=1,\n",
    "                      instance_type=instance_type,\n",
    "                      image_uri=image_uri,\n",
    "                      environment=environment,\n",
    "                      disable_profiler=True,\n",
    "                      debugger_hook_config=False,\n",
    "                      max_run=24*60*60*2)\n",
    "\n",
    "estimator.fit()\n",
    "#estimator.fit(inputs)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2134d55e-cdb1-4e9d-be23-252755bbffce",
   "metadata": {
    "tags": []
   },
   "source": [
    "### 多机多卡 torch distribute + deepspeed "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "id": "7fcca39f-029e-4c95-81f8-a5cabf717607",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting ./FastChat/ds-train-distribute.sh\n"
     ]
    }
   ],
   "source": [
    "%%writefile ./FastChat/ds-train-distribute.sh\n",
    "#!/bin/bash\n",
    "SM_MASTER=\"${SM_MASTER}\"\n",
    "SM_MASTER_ADDR=\"${SM_MASTER_ADDR}\"\n",
    "CURRENT_HOST=\"${SM_CURRENT_HOST}\"\n",
    "\n",
    "\n",
    "IFS=',' read -ra hosts_array <<< \"${SM_HOSTS}\"\n",
    "NNODES=${#hosts_array[@]}\n",
    "NODE_RANK=0\n",
    "\n",
    "for i in \"${!hosts_array[@]}\"; do\n",
    "    if [[ \"${hosts_array[$i]}\" == *${CURRENT_HOST}* ]]; then\n",
    "        echo \"host index：$i\"\n",
    "        NODE_RANK=\"$i\" \n",
    "    fi\n",
    "done\n",
    "   \n",
    "    \n",
    "MASTER_PORT=\"23456\"\n",
    "export NCCL_SOCKET_IFNAME=\"eth0\"\n",
    "\n",
    "#Configure the distributed arguments for torch.distributed.launch.\n",
    "GPUS_PER_NODE=\"$SM_NUM_GPUS\"\n",
    "DISTRIBUTED_ARGS=\"--nproc_per_node $GPUS_PER_NODE \\\n",
    "                  --nnodes $NNODES --node_rank $NODE_RANK \\\n",
    "                  --master_addr $MASTER_ADDR \\\n",
    "                  --master_port $MASTER_PORT\"\n",
    "\n",
    "\n",
    "SAVE_PATH=\"${SM_WORKING_DIR}/results\"\n",
    "LOG_FILE=\"${SAVE_PATH}/log.txt\"\n",
    "\n",
    "\n",
    "chmod +x ./s5cmd\n",
    "./s5cmd sync s3://$MODEL_S3_BUCKET/llama/pretrain/pinkmanlove/llama-7b-hf/* /tmp/llama_pretrain/\n",
    "\n",
    "#cd FastChat && pip install -e . && cd ..\n",
    "pip install -e .\n",
    "\n",
    "\n",
    "DEEPSPEED_OPTS=\"\"\"\n",
    "  ./fastchat/train/train_mem.py \n",
    "    --deepspeed ds.json \n",
    "    --model_name_or_path \"/tmp/llama_pretrain/\" \n",
    "    --data_path data/dummy_conversation.json \n",
    "    --output_dir \"/tmp/llama_out\" \n",
    "    --num_train_epochs 1 \n",
    "    --per_device_train_batch_size 1 \n",
    "    --per_device_eval_batch_size  1 \n",
    "    --gradient_accumulation_steps 4 \n",
    "    --evaluation_strategy \"no\" \n",
    "    --save_strategy \"no\" \n",
    "    --save_steps 2000 \n",
    "    --save_total_limit 1 \n",
    "    --learning_rate 2e-5 \n",
    "    --weight_decay 0. \n",
    "    --warmup_ratio 0.03 \n",
    "    --lr_scheduler_type \"cosine\" \n",
    "    --logging_steps 1 \n",
    "    --cache_dir '/tmp' \n",
    "    --model_max_length 2048 \n",
    "    --gradient_checkpointing True \n",
    "    --lazy_preprocess True \n",
    "    --fp16 True \n",
    "    --tf32 True \n",
    "    --report_to \"none\"\n",
    "\"\"\"    \n",
    "\n",
    "CMD=\"torchrun ${DISTRIBUTED_ARGS} ${DEEPSPEED_OPTS}\"\n",
    "echo ${CMD}\n",
    "${CMD} 2>&1 \n",
    "echo \"begin to upload trained model\"\n",
    "echo \"current host==\"${CURRENT_HOST}\n",
    "echo \"master host==\"${MASTER_ADDR}\n",
    "if [[ \"${CURRENT_HOST}\" == \"${MASTER_ADDR}\" ]]; then  \n",
    "    ./s5cmd sync /tmp/llama_out s3://$MODEL_S3_BUCKET/llama/output/$(date +%Y-%m-%d-%H-%M-%S)/\n",
    "fi\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5a3b0782-a59a-4457-b8a1-f44888f6c82a",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:botocore.credentials:Found credentials from IAM Role: BaseNotebookInstanceEc2InstanceRole\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using provided s3_resource\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:sagemaker:Creating training-job with name: vicuna-demo-2023-07-21-09-44-24-675\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2023-07-21 09:44:28 Starting - Starting the training job......\n",
      "2023-07-21 09:45:05 Starting - Preparing the instances for training.....................\n",
      "2023-07-21 09:48:49 Downloading - Downloading input data...\n",
      "2023-07-21 09:49:04 Training - Downloading the training image.....................\n",
      "2023-07-21 09:52:46 Training - Training image download completed. Training in progress......\u001b[34mbash: cannot set terminal process group (-1): Inappropriate ioctl for device\u001b[0m\n",
      "\u001b[34mbash: no job control in this shell\u001b[0m\n",
      "\u001b[34m2023-07-21 09:53:40,264 sagemaker-training-toolkit INFO     Imported framework sagemaker_pytorch_container.training\u001b[0m\n",
      "\u001b[34m2023-07-21 09:53:40,326 sagemaker-training-toolkit INFO     No Neurons detected (normal if no neurons installed)\u001b[0m\n",
      "\u001b[34m2023-07-21 09:53:40,335 sagemaker_pytorch_container.training INFO     Block until all host DNS lookups succeed.\u001b[0m\n",
      "\u001b[34m2023-07-21 09:53:40,337 sagemaker_pytorch_container.training INFO     Invoking user training script.\u001b[0m\n",
      "\u001b[34m2023-07-21 09:53:41,532 sagemaker-training-toolkit INFO     No Neurons detected (normal if no neurons installed)\u001b[0m\n",
      "\u001b[34m2023-07-21 09:53:41,606 sagemaker-training-toolkit INFO     No Neurons detected (normal if no neurons installed)\u001b[0m\n",
      "\u001b[34m2023-07-21 09:53:41,677 sagemaker-training-toolkit INFO     No Neurons detected (normal if no neurons installed)\u001b[0m\n",
      "\u001b[34m2023-07-21 09:53:41,687 sagemaker-training-toolkit INFO     Invoking user script\u001b[0m\n",
      "\u001b[34mTraining Env:\u001b[0m\n",
      "\u001b[34m{\n",
      "    \"additional_framework_parameters\": {},\n",
      "    \"channel_input_dirs\": {},\n",
      "    \"current_host\": \"algo-1\",\n",
      "    \"current_instance_group\": \"homogeneousCluster\",\n",
      "    \"current_instance_group_hosts\": [\n",
      "        \"algo-1\",\n",
      "        \"algo-2\"\n",
      "    ],\n",
      "    \"current_instance_type\": \"ml.p4d.24xlarge\",\n",
      "    \"distribution_hosts\": [],\n",
      "    \"distribution_instance_groups\": [],\n",
      "    \"framework_module\": \"sagemaker_pytorch_container.training:main\",\n",
      "    \"hosts\": [\n",
      "        \"algo-1\",\n",
      "        \"algo-2\"\n",
      "    ],\n",
      "    \"hyperparameters\": {},\n",
      "    \"input_config_dir\": \"/opt/ml/input/config\",\n",
      "    \"input_data_config\": {},\n",
      "    \"input_dir\": \"/opt/ml/input\",\n",
      "    \"instance_groups\": [\n",
      "        \"homogeneousCluster\"\n",
      "    ],\n",
      "    \"instance_groups_dict\": {\n",
      "        \"homogeneousCluster\": {\n",
      "            \"instance_group_name\": \"homogeneousCluster\",\n",
      "            \"instance_type\": \"ml.p4d.24xlarge\",\n",
      "            \"hosts\": [\n",
      "                \"algo-1\",\n",
      "                \"algo-2\"\n",
      "            ]\n",
      "        }\n",
      "    },\n",
      "    \"is_hetero\": false,\n",
      "    \"is_master\": true,\n",
      "    \"is_modelparallel_enabled\": null,\n",
      "    \"is_smddpmprun_installed\": true,\n",
      "    \"job_name\": \"vicuna-demo-2023-07-21-09-44-24-675\",\n",
      "    \"log_level\": 20,\n",
      "    \"master_hostname\": \"algo-1\",\n",
      "    \"model_dir\": \"/opt/ml/model\",\n",
      "    \"module_dir\": \"s3://sagemaker-us-west-2-687912291502/vicuna-demo-2023-07-21-09-44-24-675/source/sourcedir.tar.gz\",\n",
      "    \"module_name\": \"ds-train-distribute.sh\",\n",
      "    \"network_interface_name\": \"eth0\",\n",
      "    \"num_cpus\": 96,\n",
      "    \"num_gpus\": 8,\n",
      "    \"num_neurons\": 0,\n",
      "    \"output_data_dir\": \"/opt/ml/output/data\",\n",
      "    \"output_dir\": \"/opt/ml/output\",\n",
      "    \"output_intermediate_dir\": \"/opt/ml/output/intermediate\",\n",
      "    \"resource_config\": {\n",
      "        \"current_host\": \"algo-1\",\n",
      "        \"current_instance_type\": \"ml.p4d.24xlarge\",\n",
      "        \"current_group_name\": \"homogeneousCluster\",\n",
      "        \"hosts\": [\n",
      "            \"algo-1\",\n",
      "            \"algo-2\"\n",
      "        ],\n",
      "        \"instance_groups\": [\n",
      "            {\n",
      "                \"instance_group_name\": \"homogeneousCluster\",\n",
      "                \"instance_type\": \"ml.p4d.24xlarge\",\n",
      "                \"hosts\": [\n",
      "                    \"algo-1\",\n",
      "                    \"algo-2\"\n",
      "                ]\n",
      "            }\n",
      "        ],\n",
      "        \"network_interface_name\": \"eth0\"\n",
      "    },\n",
      "    \"user_entry_point\": \"ds-train-distribute.sh\"\u001b[0m\n",
      "\u001b[34m}\u001b[0m\n",
      "\u001b[34mEnvironment variables:\u001b[0m\n",
      "\u001b[34mSM_HOSTS=[\"algo-1\",\"algo-2\"]\u001b[0m\n",
      "\u001b[34mSM_NETWORK_INTERFACE_NAME=eth0\u001b[0m\n",
      "\u001b[34mSM_HPS={}\u001b[0m\n",
      "\u001b[34mSM_USER_ENTRY_POINT=ds-train-distribute.sh\u001b[0m\n",
      "\u001b[34mSM_FRAMEWORK_PARAMS={}\u001b[0m\n",
      "\u001b[34mSM_RESOURCE_CONFIG={\"current_group_name\":\"homogeneousCluster\",\"current_host\":\"algo-1\",\"current_instance_type\":\"ml.p4d.24xlarge\",\"hosts\":[\"algo-1\",\"algo-2\"],\"instance_groups\":[{\"hosts\":[\"algo-1\",\"algo-2\"],\"instance_group_name\":\"homogeneousCluster\",\"instance_type\":\"ml.p4d.24xlarge\"}],\"network_interface_name\":\"eth0\"}\u001b[0m\n",
      "\u001b[34mSM_INPUT_DATA_CONFIG={}\u001b[0m\n",
      "\u001b[34mSM_OUTPUT_DATA_DIR=/opt/ml/output/data\u001b[0m\n",
      "\u001b[34mSM_CHANNELS=[]\u001b[0m\n",
      "\u001b[34mSM_CURRENT_HOST=algo-1\u001b[0m\n",
      "\u001b[34mSM_CURRENT_INSTANCE_TYPE=ml.p4d.24xlarge\u001b[0m\n",
      "\u001b[34mSM_CURRENT_INSTANCE_GROUP=homogeneousCluster\u001b[0m\n",
      "\u001b[34mSM_CURRENT_INSTANCE_GROUP_HOSTS=[\"algo-1\",\"algo-2\"]\u001b[0m\n",
      "\u001b[34mSM_INSTANCE_GROUPS=[\"homogeneousCluster\"]\u001b[0m\n",
      "\u001b[34mSM_INSTANCE_GROUPS_DICT={\"homogeneousCluster\":{\"hosts\":[\"algo-1\",\"algo-2\"],\"instance_group_name\":\"homogeneousCluster\",\"instance_type\":\"ml.p4d.24xlarge\"}}\u001b[0m\n",
      "\u001b[34mSM_DISTRIBUTION_INSTANCE_GROUPS=[]\u001b[0m\n",
      "\u001b[34mSM_IS_HETERO=false\u001b[0m\n",
      "\u001b[34mSM_MODULE_NAME=ds-train-distribute.sh\u001b[0m\n",
      "\u001b[34mSM_LOG_LEVEL=20\u001b[0m\n",
      "\u001b[34mSM_FRAMEWORK_MODULE=sagemaker_pytorch_container.training:main\u001b[0m\n",
      "\u001b[34mSM_INPUT_DIR=/opt/ml/input\u001b[0m\n",
      "\u001b[34mSM_INPUT_CONFIG_DIR=/opt/ml/input/config\u001b[0m\n",
      "\u001b[34mSM_OUTPUT_DIR=/opt/ml/output\u001b[0m\n",
      "\u001b[34mSM_NUM_CPUS=96\u001b[0m\n",
      "\u001b[34mSM_NUM_GPUS=8\u001b[0m\n",
      "\u001b[34mSM_NUM_NEURONS=0\u001b[0m\n",
      "\u001b[34mSM_MODEL_DIR=/opt/ml/model\u001b[0m\n",
      "\u001b[34mSM_MODULE_DIR=s3://sagemaker-us-west-2-687912291502/vicuna-demo-2023-07-21-09-44-24-675/source/sourcedir.tar.gz\u001b[0m\n",
      "\u001b[34mSM_TRAINING_ENV={\"additional_framework_parameters\":{},\"channel_input_dirs\":{},\"current_host\":\"algo-1\",\"current_instance_group\":\"homogeneousCluster\",\"current_instance_group_hosts\":[\"algo-1\",\"algo-2\"],\"current_instance_type\":\"ml.p4d.24xlarge\",\"distribution_hosts\":[],\"distribution_instance_groups\":[],\"framework_module\":\"sagemaker_pytorch_container.training:main\",\"hosts\":[\"algo-1\",\"algo-2\"],\"hyperparameters\":{},\"input_config_dir\":\"/opt/ml/input/config\",\"input_data_config\":{},\"input_dir\":\"/opt/ml/input\",\"instance_groups\":[\"homogeneousCluster\"],\"instance_groups_dict\":{\"homogeneousCluster\":{\"hosts\":[\"algo-1\",\"algo-2\"],\"instance_group_name\":\"homogeneousCluster\",\"instance_type\":\"ml.p4d.24xlarge\"}},\"is_hetero\":false,\"is_master\":true,\"is_modelparallel_enabled\":null,\"is_smddpmprun_installed\":true,\"job_name\":\"vicuna-demo-2023-07-21-09-44-24-675\",\"log_level\":20,\"master_hostname\":\"algo-1\",\"model_dir\":\"/opt/ml/model\",\"module_dir\":\"s3://sagemaker-us-west-2-687912291502/vicuna-demo-2023-07-21-09-44-24-675/source/sourcedir.tar.gz\",\"module_name\":\"ds-train-distribute.sh\",\"network_interface_name\":\"eth0\",\"num_cpus\":96,\"num_gpus\":8,\"num_neurons\":0,\"output_data_dir\":\"/opt/ml/output/data\",\"output_dir\":\"/opt/ml/output\",\"output_intermediate_dir\":\"/opt/ml/output/intermediate\",\"resource_config\":{\"current_group_name\":\"homogeneousCluster\",\"current_host\":\"algo-1\",\"current_instance_type\":\"ml.p4d.24xlarge\",\"hosts\":[\"algo-1\",\"algo-2\"],\"instance_groups\":[{\"hosts\":[\"algo-1\",\"algo-2\"],\"instance_group_name\":\"homogeneousCluster\",\"instance_type\":\"ml.p4d.24xlarge\"}],\"network_interface_name\":\"eth0\"},\"user_entry_point\":\"ds-train-distribute.sh\"}\u001b[0m\n",
      "\u001b[34mSM_USER_ARGS=[]\u001b[0m\n",
      "\u001b[34mSM_OUTPUT_INTERMEDIATE_DIR=/opt/ml/output/intermediate\u001b[0m\n",
      "\u001b[34mPYTHONPATH=/opt/ml/code:/opt/conda/bin:/opt/conda/lib/python39.zip:/opt/conda/lib/python3.9:/opt/conda/lib/python3.9/lib-dynload:/opt/conda/lib/python3.9/site-packages\u001b[0m\n",
      "\u001b[34mInvoking script with the following command:\u001b[0m\n",
      "\u001b[34m/bin/sh -c \"./ds-train-distribute.sh \"\u001b[0m\n",
      "\u001b[34m[2023-07-21 09:53:43,020] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)\u001b[0m\n",
      "\u001b[35mbash: cannot set terminal process group (-1): Inappropriate ioctl for device\u001b[0m\n",
      "\u001b[35mbash: no job control in this shell\u001b[0m\n",
      "\u001b[35m2023-07-21 09:53:41,529 sagemaker-training-toolkit INFO     Imported framework sagemaker_pytorch_container.training\u001b[0m\n",
      "\u001b[35m2023-07-21 09:53:41,591 sagemaker-training-toolkit INFO     No Neurons detected (normal if no neurons installed)\u001b[0m\n",
      "\u001b[35m2023-07-21 09:53:41,600 sagemaker_pytorch_container.training INFO     Block until all host DNS lookups succeed.\u001b[0m\n",
      "\u001b[35m2023-07-21 09:53:41,602 sagemaker_pytorch_container.training INFO     Invoking user training script.\u001b[0m\n",
      "\u001b[35m2023-07-21 09:53:42,504 sagemaker-training-toolkit INFO     No Neurons detected (normal if no neurons installed)\u001b[0m\n",
      "\u001b[35m2023-07-21 09:53:42,576 sagemaker-training-toolkit INFO     No Neurons detected (normal if no neurons installed)\u001b[0m\n",
      "\u001b[35m2023-07-21 09:53:42,647 sagemaker-training-toolkit INFO     No Neurons detected (normal if no neurons installed)\u001b[0m\n",
      "\u001b[35m2023-07-21 09:53:42,657 sagemaker-training-toolkit INFO     Invoking user script\u001b[0m\n",
      "\u001b[35mTraining Env:\u001b[0m\n",
      "\u001b[35m{\n",
      "    \"additional_framework_parameters\": {},\n",
      "    \"channel_input_dirs\": {},\n",
      "    \"current_host\": \"algo-2\",\n",
      "    \"current_instance_group\": \"homogeneousCluster\",\n",
      "    \"current_instance_group_hosts\": [\n",
      "        \"algo-1\",\n",
      "        \"algo-2\"\n",
      "    ],\n",
      "    \"current_instance_type\": \"ml.p4d.24xlarge\",\n",
      "    \"distribution_hosts\": [],\n",
      "    \"distribution_instance_groups\": [],\n",
      "    \"framework_module\": \"sagemaker_pytorch_container.training:main\",\n",
      "    \"hosts\": [\n",
      "        \"algo-1\",\n",
      "        \"algo-2\"\n",
      "    ],\n",
      "    \"hyperparameters\": {},\n",
      "    \"input_config_dir\": \"/opt/ml/input/config\",\n",
      "    \"input_data_config\": {},\n",
      "    \"input_dir\": \"/opt/ml/input\",\n",
      "    \"instance_groups\": [\n",
      "        \"homogeneousCluster\"\n",
      "    ],\n",
      "    \"instance_groups_dict\": {\n",
      "        \"homogeneousCluster\": {\n",
      "            \"instance_group_name\": \"homogeneousCluster\",\n",
      "            \"instance_type\": \"ml.p4d.24xlarge\",\n",
      "            \"hosts\": [\n",
      "                \"algo-1\",\n",
      "                \"algo-2\"\n",
      "            ]\n",
      "        }\n",
      "    },\n",
      "    \"is_hetero\": false,\n",
      "    \"is_master\": false,\n",
      "    \"is_modelparallel_enabled\": null,\n",
      "    \"is_smddpmprun_installed\": true,\n",
      "    \"job_name\": \"vicuna-demo-2023-07-21-09-44-24-675\",\n",
      "    \"log_level\": 20,\n",
      "    \"master_hostname\": \"algo-1\",\n",
      "    \"model_dir\": \"/opt/ml/model\",\n",
      "    \"module_dir\": \"s3://sagemaker-us-west-2-687912291502/vicuna-demo-2023-07-21-09-44-24-675/source/sourcedir.tar.gz\",\n",
      "    \"module_name\": \"ds-train-distribute.sh\",\n",
      "    \"network_interface_name\": \"eth0\",\n",
      "    \"num_cpus\": 96,\n",
      "    \"num_gpus\": 8,\n",
      "    \"num_neurons\": 0,\n",
      "    \"output_data_dir\": \"/opt/ml/output/data\",\n",
      "    \"output_dir\": \"/opt/ml/output\",\n",
      "    \"output_intermediate_dir\": \"/opt/ml/output/intermediate\",\n",
      "    \"resource_config\": {\n",
      "        \"current_host\": \"algo-2\",\n",
      "        \"current_instance_type\": \"ml.p4d.24xlarge\",\n",
      "        \"current_group_name\": \"homogeneousCluster\",\n",
      "        \"hosts\": [\n",
      "            \"algo-1\",\n",
      "            \"algo-2\"\n",
      "        ],\n",
      "        \"instance_groups\": [\n",
      "            {\n",
      "                \"instance_group_name\": \"homogeneousCluster\",\n",
      "                \"instance_type\": \"ml.p4d.24xlarge\",\n",
      "                \"hosts\": [\n",
      "                    \"algo-1\",\n",
      "                    \"algo-2\"\n",
      "                ]\n",
      "            }\n",
      "        ],\n",
      "        \"network_interface_name\": \"eth0\"\n",
      "    },\n",
      "    \"user_entry_point\": \"ds-train-distribute.sh\"\u001b[0m\n",
      "\u001b[35m}\u001b[0m\n",
      "\u001b[35mEnvironment variables:\u001b[0m\n",
      "\u001b[35mSM_HOSTS=[\"algo-1\",\"algo-2\"]\u001b[0m\n",
      "\u001b[35mSM_NETWORK_INTERFACE_NAME=eth0\u001b[0m\n",
      "\u001b[35mSM_HPS={}\u001b[0m\n",
      "\u001b[35mSM_USER_ENTRY_POINT=ds-train-distribute.sh\u001b[0m\n",
      "\u001b[35mSM_FRAMEWORK_PARAMS={}\u001b[0m\n",
      "\u001b[35mSM_RESOURCE_CONFIG={\"current_group_name\":\"homogeneousCluster\",\"current_host\":\"algo-2\",\"current_instance_type\":\"ml.p4d.24xlarge\",\"hosts\":[\"algo-1\",\"algo-2\"],\"instance_groups\":[{\"hosts\":[\"algo-1\",\"algo-2\"],\"instance_group_name\":\"homogeneousCluster\",\"instance_type\":\"ml.p4d.24xlarge\"}],\"network_interface_name\":\"eth0\"}\u001b[0m\n",
      "\u001b[35mSM_INPUT_DATA_CONFIG={}\u001b[0m\n",
      "\u001b[35mSM_OUTPUT_DATA_DIR=/opt/ml/output/data\u001b[0m\n",
      "\u001b[35mSM_CHANNELS=[]\u001b[0m\n",
      "\u001b[35mSM_CURRENT_HOST=algo-2\u001b[0m\n",
      "\u001b[35mSM_CURRENT_INSTANCE_TYPE=ml.p4d.24xlarge\u001b[0m\n",
      "\u001b[35mSM_CURRENT_INSTANCE_GROUP=homogeneousCluster\u001b[0m\n",
      "\u001b[35mSM_CURRENT_INSTANCE_GROUP_HOSTS=[\"algo-1\",\"algo-2\"]\u001b[0m\n",
      "\u001b[35mSM_INSTANCE_GROUPS=[\"homogeneousCluster\"]\u001b[0m\n",
      "\u001b[35mSM_INSTANCE_GROUPS_DICT={\"homogeneousCluster\":{\"hosts\":[\"algo-1\",\"algo-2\"],\"instance_group_name\":\"homogeneousCluster\",\"instance_type\":\"ml.p4d.24xlarge\"}}\u001b[0m\n",
      "\u001b[35mSM_DISTRIBUTION_INSTANCE_GROUPS=[]\u001b[0m\n",
      "\u001b[35mSM_IS_HETERO=false\u001b[0m\n",
      "\u001b[35mSM_MODULE_NAME=ds-train-distribute.sh\u001b[0m\n",
      "\u001b[35mSM_LOG_LEVEL=20\u001b[0m\n",
      "\u001b[35mSM_FRAMEWORK_MODULE=sagemaker_pytorch_container.training:main\u001b[0m\n",
      "\u001b[35mSM_INPUT_DIR=/opt/ml/input\u001b[0m\n",
      "\u001b[35mSM_INPUT_CONFIG_DIR=/opt/ml/input/config\u001b[0m\n",
      "\u001b[35mSM_OUTPUT_DIR=/opt/ml/output\u001b[0m\n",
      "\u001b[35mSM_NUM_CPUS=96\u001b[0m\n",
      "\u001b[35mSM_NUM_GPUS=8\u001b[0m\n",
      "\u001b[35mSM_NUM_NEURONS=0\u001b[0m\n",
      "\u001b[35mSM_MODEL_DIR=/opt/ml/model\u001b[0m\n",
      "\u001b[35mSM_MODULE_DIR=s3://sagemaker-us-west-2-687912291502/vicuna-demo-2023-07-21-09-44-24-675/source/sourcedir.tar.gz\u001b[0m\n",
      "\u001b[35mSM_TRAINING_ENV={\"additional_framework_parameters\":{},\"channel_input_dirs\":{},\"current_host\":\"algo-2\",\"current_instance_group\":\"homogeneousCluster\",\"current_instance_group_hosts\":[\"algo-1\",\"algo-2\"],\"current_instance_type\":\"ml.p4d.24xlarge\",\"distribution_hosts\":[],\"distribution_instance_groups\":[],\"framework_module\":\"sagemaker_pytorch_container.training:main\",\"hosts\":[\"algo-1\",\"algo-2\"],\"hyperparameters\":{},\"input_config_dir\":\"/opt/ml/input/config\",\"input_data_config\":{},\"input_dir\":\"/opt/ml/input\",\"instance_groups\":[\"homogeneousCluster\"],\"instance_groups_dict\":{\"homogeneousCluster\":{\"hosts\":[\"algo-1\",\"algo-2\"],\"instance_group_name\":\"homogeneousCluster\",\"instance_type\":\"ml.p4d.24xlarge\"}},\"is_hetero\":false,\"is_master\":false,\"is_modelparallel_enabled\":null,\"is_smddpmprun_installed\":true,\"job_name\":\"vicuna-demo-2023-07-21-09-44-24-675\",\"log_level\":20,\"master_hostname\":\"algo-1\",\"model_dir\":\"/opt/ml/model\",\"module_dir\":\"s3://sagemaker-us-west-2-687912291502/vicuna-demo-2023-07-21-09-44-24-675/source/sourcedir.tar.gz\",\"module_name\":\"ds-train-distribute.sh\",\"network_interface_name\":\"eth0\",\"num_cpus\":96,\"num_gpus\":8,\"num_neurons\":0,\"output_data_dir\":\"/opt/ml/output/data\",\"output_dir\":\"/opt/ml/output\",\"output_intermediate_dir\":\"/opt/ml/output/intermediate\",\"resource_config\":{\"current_group_name\":\"homogeneousCluster\",\"current_host\":\"algo-2\",\"current_instance_type\":\"ml.p4d.24xlarge\",\"hosts\":[\"algo-1\",\"algo-2\"],\"instance_groups\":[{\"hosts\":[\"algo-1\",\"algo-2\"],\"instance_group_name\":\"homogeneousCluster\",\"instance_type\":\"ml.p4d.24xlarge\"}],\"network_interface_name\":\"eth0\"},\"user_entry_point\":\"ds-train-distribute.sh\"}\u001b[0m\n",
      "\u001b[35mSM_USER_ARGS=[]\u001b[0m\n",
      "\u001b[35mSM_OUTPUT_INTERMEDIATE_DIR=/opt/ml/output/intermediate\u001b[0m\n",
      "\u001b[35mPYTHONPATH=/opt/ml/code:/opt/conda/bin:/opt/conda/lib/python39.zip:/opt/conda/lib/python3.9:/opt/conda/lib/python3.9/lib-dynload:/opt/conda/lib/python3.9/site-packages\u001b[0m\n",
      "\u001b[35mInvoking script with the following command:\u001b[0m\n",
      "\u001b[35m/bin/sh -c \"./ds-train-distribute.sh \"\u001b[0m\n",
      "\u001b[35m[2023-07-21 09:53:43,948] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)\u001b[0m\n",
      "\u001b[34m[2023-07-21 09:53:45.779: W smdistributed/modelparallel/torch/nn/predefined_hooks.py:78] Found unsupported HuggingFace version 4.28.0 for automated tensor parallelism. HuggingFace modules will not be automatically distributed. You can use smp.tp_register_with_module API to register desired modules for tensor parallelism, or directly instantiate an smp.nn.DistributedModule. Supported HuggingFace transformers versions for automated tensor parallelism: ['4.17.0', '4.20.1', '4.21.0']\u001b[0m\n",
      "\u001b[34m2023-07-21 09:53:45,783 root         INFO     Using NamedTuple = typing._NamedTuple instead.\u001b[0m\n",
      "\u001b[34m2023-07-21 09:53:45,802 sagemaker-training-toolkit INFO     Exceptions not imported for SageMaker TF as Tensorflow is not installed.\u001b[0m\n",
      "\u001b[34mhost index：0\u001b[0m\n",
      "\u001b[34mcp s3://sagemaker-us-west-2-687912291502/llama/pretrain/pinkmanlove/llama-7b-hf/config.json /tmp/llama_pretrain/config.json\u001b[0m\n",
      "\u001b[34mcp s3://sagemaker-us-west-2-687912291502/llama/pretrain/pinkmanlove/llama-7b-hf/generation_config.json /tmp/llama_pretrain/generation_config.json\u001b[0m\n",
      "\u001b[34mcp s3://sagemaker-us-west-2-687912291502/llama/pretrain/pinkmanlove/llama-7b-hf/tokenizer_config.json /tmp/llama_pretrain/tokenizer_config.json\u001b[0m\n",
      "\u001b[34mcp s3://sagemaker-us-west-2-687912291502/llama/pretrain/pinkmanlove/llama-7b-hf/pytorch_model.bin.index.json /tmp/llama_pretrain/pytorch_model.bin.index.json\u001b[0m\n",
      "\u001b[34mcp s3://sagemaker-us-west-2-687912291502/llama/pretrain/pinkmanlove/llama-7b-hf/special_tokens_map.json /tmp/llama_pretrain/special_tokens_map.json\u001b[0m\n",
      "\u001b[34mcp s3://sagemaker-us-west-2-687912291502/llama/pretrain/pinkmanlove/llama-7b-hf/tokenizer.model /tmp/llama_pretrain/tokenizer.model\u001b[0m\n",
      "\u001b[35m[2023-07-21 09:53:46.642: W smdistributed/modelparallel/torch/nn/predefined_hooks.py:78] Found unsupported HuggingFace version 4.28.0 for automated tensor parallelism. HuggingFace modules will not be automatically distributed. You can use smp.tp_register_with_module API to register desired modules for tensor parallelism, or directly instantiate an smp.nn.DistributedModule. Supported HuggingFace transformers versions for automated tensor parallelism: ['4.17.0', '4.20.1', '4.21.0']\u001b[0m\n",
      "\u001b[35m2023-07-21 09:53:46,646 root         INFO     Using NamedTuple = typing._NamedTuple instead.\u001b[0m\n",
      "\u001b[35m2023-07-21 09:53:46,664 sagemaker-training-toolkit INFO     Exceptions not imported for SageMaker TF as Tensorflow is not installed.\u001b[0m\n",
      "\u001b[35mhost index：1\u001b[0m\n",
      "\u001b[35mcp s3://sagemaker-us-west-2-687912291502/llama/pretrain/pinkmanlove/llama-7b-hf/config.json /tmp/llama_pretrain/config.json\u001b[0m\n",
      "\u001b[35mcp s3://sagemaker-us-west-2-687912291502/llama/pretrain/pinkmanlove/llama-7b-hf/generation_config.json /tmp/llama_pretrain/generation_config.json\u001b[0m\n",
      "\u001b[35mcp s3://sagemaker-us-west-2-687912291502/llama/pretrain/pinkmanlove/llama-7b-hf/special_tokens_map.json /tmp/llama_pretrain/special_tokens_map.json\u001b[0m\n",
      "\u001b[35mcp s3://sagemaker-us-west-2-687912291502/llama/pretrain/pinkmanlove/llama-7b-hf/pytorch_model.bin.index.json /tmp/llama_pretrain/pytorch_model.bin.index.json\u001b[0m\n",
      "\u001b[35mcp s3://sagemaker-us-west-2-687912291502/llama/pretrain/pinkmanlove/llama-7b-hf/tokenizer_config.json /tmp/llama_pretrain/tokenizer_config.json\u001b[0m\n",
      "\u001b[35mcp s3://sagemaker-us-west-2-687912291502/llama/pretrain/pinkmanlove/llama-7b-hf/tokenizer.model /tmp/llama_pretrain/tokenizer.model\u001b[0m\n",
      "\u001b[34mcp s3://sagemaker-us-west-2-687912291502/llama/pretrain/pinkmanlove/llama-7b-hf/pytorch_model-00002-of-00002.bin /tmp/llama_pretrain/pytorch_model-00002-of-00002.bin\u001b[0m\n",
      "\u001b[35mcp s3://sagemaker-us-west-2-687912291502/llama/pretrain/pinkmanlove/llama-7b-hf/pytorch_model-00002-of-00002.bin /tmp/llama_pretrain/pytorch_model-00002-of-00002.bin\u001b[0m\n",
      "\u001b[34mcp s3://sagemaker-us-west-2-687912291502/llama/pretrain/pinkmanlove/llama-7b-hf/pytorch_model-00001-of-00002.bin /tmp/llama_pretrain/pytorch_model-00001-of-00002.bin\u001b[0m\n",
      "\u001b[35mcp s3://sagemaker-us-west-2-687912291502/llama/pretrain/pinkmanlove/llama-7b-hf/pytorch_model-00001-of-00002.bin /tmp/llama_pretrain/pytorch_model-00001-of-00002.bin\u001b[0m\n",
      "\u001b[34mObtaining file:///opt/ml/code\u001b[0m\n",
      "\u001b[34mInstalling build dependencies: started\u001b[0m\n",
      "\u001b[35mObtaining file:///opt/ml/code\u001b[0m\n",
      "\u001b[35mInstalling build dependencies: started\u001b[0m\n",
      "\u001b[34mInstalling build dependencies: finished with status 'done'\u001b[0m\n",
      "\u001b[34mChecking if build backend supports build_editable: started\u001b[0m\n",
      "\u001b[35mInstalling build dependencies: finished with status 'done'\u001b[0m\n",
      "\u001b[35mChecking if build backend supports build_editable: started\u001b[0m\n",
      "\u001b[35mChecking if build backend supports build_editable: finished with status 'done'\u001b[0m\n",
      "\u001b[35mGetting requirements to build editable: started\u001b[0m\n",
      "\u001b[34mChecking if build backend supports build_editable: finished with status 'done'\u001b[0m\n",
      "\u001b[34mGetting requirements to build editable: started\u001b[0m\n",
      "\u001b[34mGetting requirements to build editable: finished with status 'done'\u001b[0m\n",
      "\u001b[34mInstalling backend dependencies: started\u001b[0m\n",
      "\u001b[35mGetting requirements to build editable: finished with status 'done'\u001b[0m\n",
      "\u001b[35mInstalling backend dependencies: started\u001b[0m\n",
      "\u001b[34mInstalling backend dependencies: finished with status 'done'\u001b[0m\n",
      "\u001b[34mPreparing editable metadata (pyproject.toml): started\u001b[0m\n",
      "\u001b[34mPreparing editable metadata (pyproject.toml): finished with status 'done'\u001b[0m\n",
      "\u001b[34mCollecting wandb\u001b[0m\n",
      "\u001b[34mDownloading wandb-0.15.5-py3-none-any.whl (2.1 MB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 2.1/2.1 MB 12.6 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting httpx\u001b[0m\n",
      "\u001b[34mDownloading httpx-0.24.1-py3-none-any.whl (75 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 75.4/75.4 kB 17.7 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[35mInstalling backend dependencies: finished with status 'done'\u001b[0m\n",
      "\u001b[35mPreparing editable metadata (pyproject.toml): started\u001b[0m\n",
      "\u001b[35mPreparing editable metadata (pyproject.toml): finished with status 'done'\u001b[0m\n",
      "\u001b[35mCollecting nh3\u001b[0m\n",
      "\u001b[35mDownloading nh3-0.2.14-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.7 MB)\u001b[0m\n",
      "\u001b[35m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.7/1.7 MB 24.6 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[35mRequirement already satisfied: torch in /opt/conda/lib/python3.9/site-packages (from fschat==0.2.19) (1.13.1+cu117)\u001b[0m\n",
      "\u001b[35mRequirement already satisfied: pydantic<=2.0 in /opt/conda/lib/python3.9/site-packages (from fschat==0.2.19) (1.10.4)\u001b[0m\n",
      "\u001b[35mRequirement already satisfied: transformers<4.29.0,>=4.28.0 in /opt/conda/lib/python3.9/site-packages (from fschat==0.2.19) (4.28.0)\u001b[0m\n",
      "\u001b[35mCollecting tiktoken\u001b[0m\n",
      "\u001b[35mDownloading tiktoken-0.4.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.7 MB)\u001b[0m\n",
      "\u001b[35m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.7/1.7 MB 83.8 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[35mCollecting httpx\u001b[0m\n",
      "\u001b[35mDownloading httpx-0.24.1-py3-none-any.whl (75 kB)\u001b[0m\n",
      "\u001b[35m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 75.4/75.4 kB 18.8 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[35mRequirement already satisfied: sentencepiece in /opt/conda/lib/python3.9/site-packages (from fschat==0.2.19) (0.1.97)\u001b[0m\n",
      "\u001b[35mCollecting shortuuid\u001b[0m\n",
      "\u001b[35mDownloading shortuuid-1.0.11-py3-none-any.whl (10 kB)\u001b[0m\n",
      "\u001b[35mCollecting wandb\u001b[0m\n",
      "\u001b[34mCollecting gradio\u001b[0m\n",
      "\u001b[34mDownloading gradio-3.38.0-py3-none-any.whl (19.8 MB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 19.8/19.8 MB 49.1 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: prompt-toolkit>=3.0.0 in /opt/conda/lib/python3.9/site-packages (from fschat==0.2.19) (3.0.36)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: rich>=10.0.0 in /opt/conda/lib/python3.9/site-packages (from fschat==0.2.19) (12.6.0)\u001b[0m\n",
      "\u001b[34mCollecting fastapi\u001b[0m\n",
      "\u001b[34mDownloading fastapi-0.100.0-py3-none-any.whl (65 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 65.7/65.7 kB 18.0 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: torch in /opt/conda/lib/python3.9/site-packages (from fschat==0.2.19) (1.13.1+cu117)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: accelerate in /opt/conda/lib/python3.9/site-packages (from fschat==0.2.19) (0.16.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: tokenizers>=0.12.1 in /opt/conda/lib/python3.9/site-packages (from fschat==0.2.19) (0.13.2)\u001b[0m\n",
      "\u001b[34mCollecting peft\u001b[0m\n",
      "\u001b[34mDownloading peft-0.4.0-py3-none-any.whl (72 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 72.9/72.9 kB 19.7 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting tiktoken\u001b[0m\n",
      "\u001b[34mDownloading tiktoken-0.4.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.7 MB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.7/1.7 MB 72.5 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: sentencepiece in /opt/conda/lib/python3.9/site-packages (from fschat==0.2.19) (0.1.97)\u001b[0m\n",
      "\u001b[34mCollecting markdown2[all]\u001b[0m\n",
      "\u001b[34mDownloading markdown2-2.4.9-py2.py3-none-any.whl (39 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: transformers<4.29.0,>=4.28.0 in /opt/conda/lib/python3.9/site-packages (from fschat==0.2.19) (4.28.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: requests in /opt/conda/lib/python3.9/site-packages (from fschat==0.2.19) (2.28.2)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: numpy in /opt/conda/lib/python3.9/site-packages (from fschat==0.2.19) (1.23.5)\u001b[0m\n",
      "\u001b[34mCollecting uvicorn\u001b[0m\n",
      "\u001b[34mDownloading uvicorn-0.23.1-py3-none-any.whl (59 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 59.5/59.5 kB 13.5 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting nh3\u001b[0m\n",
      "\u001b[35mDownloading wandb-0.15.5-py3-none-any.whl (2.1 MB)\u001b[0m\n",
      "\u001b[35m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 2.1/2.1 MB 80.4 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[35mRequirement already satisfied: requests in /opt/conda/lib/python3.9/site-packages (from fschat==0.2.19) (2.28.2)\u001b[0m\n",
      "\u001b[35mRequirement already satisfied: accelerate in /opt/conda/lib/python3.9/site-packages (from fschat==0.2.19) (0.16.0)\u001b[0m\n",
      "\u001b[35mRequirement already satisfied: prompt-toolkit>=3.0.0 in /opt/conda/lib/python3.9/site-packages (from fschat==0.2.19) (3.0.36)\u001b[0m\n",
      "\u001b[35mCollecting peft\u001b[0m\n",
      "\u001b[35mDownloading peft-0.4.0-py3-none-any.whl (72 kB)\u001b[0m\n",
      "\u001b[35m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 72.9/72.9 kB 17.8 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[35mCollecting markdown2[all]\u001b[0m\n",
      "\u001b[35mDownloading markdown2-2.4.9-py2.py3-none-any.whl (39 kB)\u001b[0m\n",
      "\u001b[35mRequirement already satisfied: rich>=10.0.0 in /opt/conda/lib/python3.9/site-packages (from fschat==0.2.19) (12.6.0)\u001b[0m\n",
      "\u001b[35mCollecting uvicorn\u001b[0m\n",
      "\u001b[35mDownloading uvicorn-0.23.1-py3-none-any.whl (59 kB)\u001b[0m\n",
      "\u001b[35m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 59.5/59.5 kB 16.2 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[35mCollecting gradio\u001b[0m\n",
      "\u001b[35mDownloading gradio-3.38.0-py3-none-any.whl (19.8 MB)\u001b[0m\n",
      "\u001b[35m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 19.8/19.8 MB 57.8 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[35mRequirement already satisfied: tokenizers>=0.12.1 in /opt/conda/lib/python3.9/site-packages (from fschat==0.2.19) (0.13.2)\u001b[0m\n",
      "\u001b[35mRequirement already satisfied: numpy in /opt/conda/lib/python3.9/site-packages (from fschat==0.2.19) (1.23.5)\u001b[0m\n",
      "\u001b[35mCollecting fastapi\u001b[0m\n",
      "\u001b[35mDownloading fastapi-0.100.0-py3-none-any.whl (65 kB)\u001b[0m\n",
      "\u001b[35m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 65.7/65.7 kB 15.1 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[35mRequirement already satisfied: wcwidth in /opt/conda/lib/python3.9/site-packages (from prompt-toolkit>=3.0.0->fschat==0.2.19) (0.2.6)\u001b[0m\n",
      "\u001b[35mRequirement already satisfied: typing-extensions>=4.2.0 in /opt/conda/lib/python3.9/site-packages (from pydantic<=2.0->fschat==0.2.19) (4.4.0)\u001b[0m\n",
      "\u001b[35mRequirement already satisfied: pygments<3.0.0,>=2.6.0 in /opt/conda/lib/python3.9/site-packages (from rich>=10.0.0->fschat==0.2.19) (2.14.0)\u001b[0m\n",
      "\u001b[35mRequirement already satisfied: commonmark<0.10.0,>=0.9.0 in /opt/conda/lib/python3.9/site-packages (from rich>=10.0.0->fschat==0.2.19) (0.9.1)\u001b[0m\n",
      "\u001b[34mDownloading nh3-0.2.14-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.7 MB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.7/1.7 MB 75.6 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pydantic<=2.0 in /opt/conda/lib/python3.9/site-packages (from fschat==0.2.19) (1.10.4)\u001b[0m\n",
      "\u001b[34mCollecting shortuuid\u001b[0m\n",
      "\u001b[34mDownloading shortuuid-1.0.11-py3-none-any.whl (10 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: wcwidth in /opt/conda/lib/python3.9/site-packages (from prompt-toolkit>=3.0.0->fschat==0.2.19) (0.2.6)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: typing-extensions>=4.2.0 in /opt/conda/lib/python3.9/site-packages (from pydantic<=2.0->fschat==0.2.19) (4.4.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pygments<3.0.0,>=2.6.0 in /opt/conda/lib/python3.9/site-packages (from rich>=10.0.0->fschat==0.2.19) (2.14.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: commonmark<0.10.0,>=0.9.0 in /opt/conda/lib/python3.9/site-packages (from rich>=10.0.0->fschat==0.2.19) (0.9.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.9/site-packages (from transformers<4.29.0,>=4.28.0->fschat==0.2.19) (23.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: regex!=2019.12.17 in /opt/conda/lib/python3.9/site-packages (from transformers<4.29.0,>=4.28.0->fschat==0.2.19) (2022.10.31)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: huggingface-hub<1.0,>=0.11.0 in /opt/conda/lib/python3.9/site-packages (from transformers<4.29.0,>=4.28.0->fschat==0.2.19) (0.12.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: tqdm>=4.27 in /opt/conda/lib/python3.9/site-packages (from transformers<4.29.0,>=4.28.0->fschat==0.2.19) (4.64.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.9/site-packages (from transformers<4.29.0,>=4.28.0->fschat==0.2.19) (5.4.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: filelock in /opt/conda/lib/python3.9/site-packages (from transformers<4.29.0,>=4.28.0->fschat==0.2.19) (3.9.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: psutil in /opt/conda/lib/python3.9/site-packages (from accelerate->fschat==0.2.19) (5.9.4)\u001b[0m\n",
      "\u001b[34mCollecting starlette<0.28.0,>=0.27.0\u001b[0m\n",
      "\u001b[34mDownloading starlette-0.27.0-py3-none-any.whl (66 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 67.0/67.0 kB 16.0 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting typing-extensions>=4.2.0\u001b[0m\n",
      "\u001b[34mDownloading typing_extensions-4.7.1-py3-none-any.whl (33 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pandas<3.0,>=1.0 in /opt/conda/lib/python3.9/site-packages (from gradio->fschat==0.2.19) (1.5.3)\u001b[0m\n",
      "\u001b[34mCollecting markdown-it-py[linkify]>=2.0.0\u001b[0m\n",
      "\u001b[34mDownloading markdown_it_py-3.0.0-py3-none-any.whl (87 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 87.5/87.5 kB 20.9 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting aiofiles<24.0,>=22.0\u001b[0m\n",
      "\u001b[34mDownloading aiofiles-23.1.0-py3-none-any.whl (14 kB)\u001b[0m\n",
      "\u001b[34mCollecting semantic-version~=2.0\u001b[0m\n",
      "\u001b[34mDownloading semantic_version-2.10.0-py2.py3-none-any.whl (15 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: aiohttp~=3.0 in /opt/conda/lib/python3.9/site-packages (from gradio->fschat==0.2.19) (3.8.4)\u001b[0m\n",
      "\u001b[34mCollecting mdit-py-plugins<=0.3.3\u001b[0m\n",
      "\u001b[34mDownloading mdit_py_plugins-0.3.3-py3-none-any.whl (50 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 50.5/50.5 kB 14.5 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting pydub\u001b[0m\n",
      "\u001b[34mDownloading pydub-0.25.1-py2.py3-none-any.whl (32 kB)\u001b[0m\n",
      "\u001b[35mRequirement already satisfied: regex!=2019.12.17 in /opt/conda/lib/python3.9/site-packages (from transformers<4.29.0,>=4.28.0->fschat==0.2.19) (2022.10.31)\u001b[0m\n",
      "\u001b[35mRequirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.9/site-packages (from transformers<4.29.0,>=4.28.0->fschat==0.2.19) (23.0)\u001b[0m\n",
      "\u001b[35mRequirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.9/site-packages (from transformers<4.29.0,>=4.28.0->fschat==0.2.19) (5.4.1)\u001b[0m\n",
      "\u001b[35mRequirement already satisfied: huggingface-hub<1.0,>=0.11.0 in /opt/conda/lib/python3.9/site-packages (from transformers<4.29.0,>=4.28.0->fschat==0.2.19) (0.12.0)\u001b[0m\n",
      "\u001b[35mRequirement already satisfied: filelock in /opt/conda/lib/python3.9/site-packages (from transformers<4.29.0,>=4.28.0->fschat==0.2.19) (3.9.0)\u001b[0m\n",
      "\u001b[35mRequirement already satisfied: tqdm>=4.27 in /opt/conda/lib/python3.9/site-packages (from transformers<4.29.0,>=4.28.0->fschat==0.2.19) (4.64.1)\u001b[0m\n",
      "\u001b[35mRequirement already satisfied: psutil in /opt/conda/lib/python3.9/site-packages (from accelerate->fschat==0.2.19) (5.9.4)\u001b[0m\n",
      "\u001b[35mCollecting typing-extensions>=4.2.0\u001b[0m\n",
      "\u001b[35mDownloading typing_extensions-4.7.1-py3-none-any.whl (33 kB)\u001b[0m\n",
      "\u001b[35mCollecting starlette<0.28.0,>=0.27.0\u001b[0m\n",
      "\u001b[35mDownloading starlette-0.27.0-py3-none-any.whl (66 kB)\u001b[0m\n",
      "\u001b[35m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 67.0/67.0 kB 13.7 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[35mRequirement already satisfied: pandas<3.0,>=1.0 in /opt/conda/lib/python3.9/site-packages (from gradio->fschat==0.2.19) (1.5.3)\u001b[0m\n",
      "\u001b[35mRequirement already satisfied: aiohttp~=3.0 in /opt/conda/lib/python3.9/site-packages (from gradio->fschat==0.2.19) (3.8.4)\u001b[0m\n",
      "\u001b[35mCollecting gradio-client>=0.2.10\u001b[0m\n",
      "\u001b[35mDownloading gradio_client-0.2.10-py3-none-any.whl (288 kB)\u001b[0m\n",
      "\u001b[35m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 289.0/289.0 kB 49.7 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[35mCollecting python-multipart\u001b[0m\n",
      "\u001b[35mDownloading python_multipart-0.0.6-py3-none-any.whl (45 kB)\u001b[0m\n",
      "\u001b[35m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 45.7/45.7 kB 12.3 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[35mCollecting ffmpy\u001b[0m\n",
      "\u001b[35mDownloading ffmpy-0.3.1.tar.gz (5.5 kB)\u001b[0m\n",
      "\u001b[35mPreparing metadata (setup.py): started\u001b[0m\n",
      "\u001b[35mPreparing metadata (setup.py): finished with status 'done'\u001b[0m\n",
      "\u001b[35mRequirement already satisfied: jinja2<4.0 in /opt/conda/lib/python3.9/site-packages (from gradio->fschat==0.2.19) (3.1.2)\u001b[0m\n",
      "\u001b[35mRequirement already satisfied: markupsafe~=2.0 in /opt/conda/lib/python3.9/site-packages (from gradio->fschat==0.2.19) (2.1.2)\u001b[0m\n",
      "\u001b[35mCollecting semantic-version~=2.0\u001b[0m\n",
      "\u001b[35mDownloading semantic_version-2.10.0-py2.py3-none-any.whl (15 kB)\u001b[0m\n",
      "\u001b[35mCollecting altair<6.0,>=4.2.0\u001b[0m\n",
      "\u001b[35mDownloading altair-5.0.1-py3-none-any.whl (471 kB)\u001b[0m\n",
      "\u001b[35m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 471.5/471.5 kB 54.8 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[35mCollecting pydub\u001b[0m\n",
      "\u001b[35mDownloading pydub-0.25.1-py2.py3-none-any.whl (32 kB)\u001b[0m\n",
      "\u001b[35mRequirement already satisfied: pillow<11.0,>=8.0 in /opt/conda/lib/python3.9/site-packages (from gradio->fschat==0.2.19) (9.4.0)\u001b[0m\n",
      "\u001b[35mCollecting huggingface-hub<1.0,>=0.11.0\u001b[0m\n",
      "\u001b[35mDownloading huggingface_hub-0.16.4-py3-none-any.whl (268 kB)\u001b[0m\n",
      "\u001b[35m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 268.8/268.8 kB 47.0 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting orjson~=3.0\u001b[0m\n",
      "\u001b[34mDownloading orjson-3.9.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (138 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 138.6/138.6 kB 30.7 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting huggingface-hub<1.0,>=0.11.0\u001b[0m\n",
      "\u001b[34mDownloading huggingface_hub-0.16.4-py3-none-any.whl (268 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 268.8/268.8 kB 44.3 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: jinja2<4.0 in /opt/conda/lib/python3.9/site-packages (from gradio->fschat==0.2.19) (3.1.2)\u001b[0m\n",
      "\u001b[34mCollecting websockets<12.0,>=10.0\u001b[0m\n",
      "\u001b[34mDownloading websockets-11.0.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (129 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 129.7/129.7 kB 30.6 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pillow<11.0,>=8.0 in /opt/conda/lib/python3.9/site-packages (from gradio->fschat==0.2.19) (9.4.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: markupsafe~=2.0 in /opt/conda/lib/python3.9/site-packages (from gradio->fschat==0.2.19) (2.1.2)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: matplotlib~=3.0 in /opt/conda/lib/python3.9/site-packages (from gradio->fschat==0.2.19) (3.6.3)\u001b[0m\n",
      "\u001b[34mCollecting python-multipart\u001b[0m\n",
      "\u001b[34mDownloading python_multipart-0.0.6-py3-none-any.whl (45 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 45.7/45.7 kB 12.1 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting ffmpy\u001b[0m\n",
      "\u001b[34mDownloading ffmpy-0.3.1.tar.gz (5.5 kB)\u001b[0m\n",
      "\u001b[34mPreparing metadata (setup.py): started\u001b[0m\n",
      "\u001b[34mPreparing metadata (setup.py): finished with status 'done'\u001b[0m\n",
      "\u001b[34mCollecting gradio-client>=0.2.10\u001b[0m\n",
      "\u001b[34mDownloading gradio_client-0.2.10-py3-none-any.whl (288 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 289.0/289.0 kB 49.2 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting altair<6.0,>=4.2.0\u001b[0m\n",
      "\u001b[34mDownloading altair-5.0.1-py3-none-any.whl (471 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 471.5/471.5 kB 60.6 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.9/site-packages (from requests->fschat==0.2.19) (2022.12.7)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.9/site-packages (from requests->fschat==0.2.19) (2.1.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.9/site-packages (from requests->fschat==0.2.19) (1.26.14)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.9/site-packages (from requests->fschat==0.2.19) (3.4)\u001b[0m\n",
      "\u001b[34mCollecting h11>=0.8\u001b[0m\n",
      "\u001b[34mDownloading h11-0.14.0-py3-none-any.whl (58 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 58.3/58.3 kB 15.1 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: click>=7.0 in /opt/conda/lib/python3.9/site-packages (from uvicorn->fschat==0.2.19) (8.1.2)\u001b[0m\n",
      "\u001b[34mCollecting httpcore<0.18.0,>=0.15.0\u001b[0m\n",
      "\u001b[34mDownloading httpcore-0.17.3-py3-none-any.whl (74 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 74.5/74.5 kB 20.5 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting sniffio\u001b[0m\n",
      "\u001b[35mCollecting orjson~=3.0\u001b[0m\n",
      "\u001b[35mDownloading orjson-3.9.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (138 kB)\u001b[0m\n",
      "\u001b[35m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 138.6/138.6 kB 30.9 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[35mCollecting markdown-it-py[linkify]>=2.0.0\u001b[0m\n",
      "\u001b[35mDownloading markdown_it_py-3.0.0-py3-none-any.whl (87 kB)\u001b[0m\n",
      "\u001b[35m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 87.5/87.5 kB 20.8 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[35mCollecting aiofiles<24.0,>=22.0\u001b[0m\n",
      "\u001b[35mDownloading aiofiles-23.1.0-py3-none-any.whl (14 kB)\u001b[0m\n",
      "\u001b[35mCollecting websockets<12.0,>=10.0\u001b[0m\n",
      "\u001b[35mDownloading websockets-11.0.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (129 kB)\u001b[0m\n",
      "\u001b[35m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 129.7/129.7 kB 26.1 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[35mRequirement already satisfied: matplotlib~=3.0 in /opt/conda/lib/python3.9/site-packages (from gradio->fschat==0.2.19) (3.6.3)\u001b[0m\n",
      "\u001b[35mCollecting mdit-py-plugins<=0.3.3\u001b[0m\n",
      "\u001b[35mDownloading mdit_py_plugins-0.3.3-py3-none-any.whl (50 kB)\u001b[0m\n",
      "\u001b[35m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 50.5/50.5 kB 14.0 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[35mRequirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.9/site-packages (from requests->fschat==0.2.19) (1.26.14)\u001b[0m\n",
      "\u001b[35mRequirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.9/site-packages (from requests->fschat==0.2.19) (3.4)\u001b[0m\n",
      "\u001b[35mRequirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.9/site-packages (from requests->fschat==0.2.19) (2.1.1)\u001b[0m\n",
      "\u001b[35mRequirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.9/site-packages (from requests->fschat==0.2.19) (2022.12.7)\u001b[0m\n",
      "\u001b[35mRequirement already satisfied: click>=7.0 in /opt/conda/lib/python3.9/site-packages (from uvicorn->fschat==0.2.19) (8.1.2)\u001b[0m\n",
      "\u001b[35mCollecting h11>=0.8\u001b[0m\n",
      "\u001b[35mDownloading h11-0.14.0-py3-none-any.whl (58 kB)\u001b[0m\n",
      "\u001b[35m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 58.3/58.3 kB 16.8 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[35mCollecting sniffio\u001b[0m\n",
      "\u001b[35mDownloading sniffio-1.3.0-py3-none-any.whl (10 kB)\u001b[0m\n",
      "\u001b[35mCollecting httpcore<0.18.0,>=0.15.0\u001b[0m\n",
      "\u001b[35mDownloading httpcore-0.17.3-py3-none-any.whl (74 kB)\u001b[0m\n",
      "\u001b[35m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 74.5/74.5 kB 19.9 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[35mCollecting wavedrom\u001b[0m\n",
      "\u001b[35mDownloading wavedrom-2.0.3.post3.tar.gz (137 kB)\u001b[0m\n",
      "\u001b[35m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 137.7/137.7 kB 29.3 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[35mPreparing metadata (setup.py): started\u001b[0m\n",
      "\u001b[34mDownloading sniffio-1.3.0-py3-none-any.whl (10 kB)\u001b[0m\n",
      "\u001b[34mCollecting wavedrom\u001b[0m\n",
      "\u001b[34mDownloading wavedrom-2.0.3.post3.tar.gz (137 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 137.7/137.7 kB 25.9 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mPreparing metadata (setup.py): started\u001b[0m\n",
      "\u001b[34mPreparing metadata (setup.py): finished with status 'done'\u001b[0m\n",
      "\u001b[34mCollecting safetensors\u001b[0m\n",
      "\u001b[34mDownloading safetensors-0.3.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.3/1.3 MB 83.3 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[35mPreparing metadata (setup.py): finished with status 'done'\u001b[0m\n",
      "\u001b[35mCollecting safetensors\u001b[0m\n",
      "\u001b[35mDownloading safetensors-0.3.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\u001b[0m\n",
      "\u001b[35m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.3/1.3 MB 63.3 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[35mCollecting setproctitle\u001b[0m\n",
      "\u001b[35mDownloading setproctitle-1.3.2-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (30 kB)\u001b[0m\n",
      "\u001b[35mRequirement already satisfied: setuptools in /opt/conda/lib/python3.9/site-packages (from wandb->fschat==0.2.19) (65.6.3)\u001b[0m\n",
      "\u001b[35mCollecting sentry-sdk>=1.0.0\u001b[0m\n",
      "\u001b[35mDownloading sentry_sdk-1.28.1-py2.py3-none-any.whl (214 kB)\u001b[0m\n",
      "\u001b[35m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 214.7/214.7 kB 43.3 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[35mCollecting GitPython!=3.1.29,>=1.0.0\u001b[0m\n",
      "\u001b[35mDownloading GitPython-3.1.32-py3-none-any.whl (188 kB)\u001b[0m\n",
      "\u001b[35m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 188.5/188.5 kB 38.4 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[35mRequirement already satisfied: appdirs>=1.4.3 in /opt/conda/lib/python3.9/site-packages (from wandb->fschat==0.2.19) (1.4.4)\u001b[0m\n",
      "\u001b[35mCollecting pathtools\u001b[0m\n",
      "\u001b[35mDownloading pathtools-0.1.2.tar.gz (11 kB)\u001b[0m\n",
      "\u001b[35mPreparing metadata (setup.py): started\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: protobuf!=4.21.0,<5,>=3.15.0 in /opt/conda/lib/python3.9/site-packages (from wandb->fschat==0.2.19) (3.20.2)\u001b[0m\n",
      "\u001b[34mCollecting pathtools\u001b[0m\n",
      "\u001b[34mDownloading pathtools-0.1.2.tar.gz (11 kB)\u001b[0m\n",
      "\u001b[34mPreparing metadata (setup.py): started\u001b[0m\n",
      "\u001b[34mPreparing metadata (setup.py): finished with status 'done'\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: setuptools in /opt/conda/lib/python3.9/site-packages (from wandb->fschat==0.2.19) (65.6.3)\u001b[0m\n",
      "\u001b[34mCollecting GitPython!=3.1.29,>=1.0.0\u001b[0m\n",
      "\u001b[34mDownloading GitPython-3.1.32-py3-none-any.whl (188 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 188.5/188.5 kB 35.0 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting setproctitle\u001b[0m\n",
      "\u001b[34mDownloading setproctitle-1.3.2-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (30 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: appdirs>=1.4.3 in /opt/conda/lib/python3.9/site-packages (from wandb->fschat==0.2.19) (1.4.4)\u001b[0m\n",
      "\u001b[34mCollecting sentry-sdk>=1.0.0\u001b[0m\n",
      "\u001b[34mDownloading sentry_sdk-1.28.1-py2.py3-none-any.whl (214 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 214.7/214.7 kB 46.4 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting docker-pycreds>=0.4.0\u001b[0m\n",
      "\u001b[34mDownloading docker_pycreds-0.4.0-py2.py3-none-any.whl (9.0 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /opt/conda/lib/python3.9/site-packages (from aiohttp~=3.0->gradio->fschat==0.2.19) (4.0.2)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: attrs>=17.3.0 in /opt/conda/lib/python3.9/site-packages (from aiohttp~=3.0->gradio->fschat==0.2.19) (22.2.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: aiosignal>=1.1.2 in /opt/conda/lib/python3.9/site-packages (from aiohttp~=3.0->gradio->fschat==0.2.19) (1.3.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: frozenlist>=1.1.1 in /opt/conda/lib/python3.9/site-packages (from aiohttp~=3.0->gradio->fschat==0.2.19) (1.3.3)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: yarl<2.0,>=1.0 in /opt/conda/lib/python3.9/site-packages (from aiohttp~=3.0->gradio->fschat==0.2.19) (1.8.2)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: multidict<7.0,>=4.5 in /opt/conda/lib/python3.9/site-packages (from aiohttp~=3.0->gradio->fschat==0.2.19) (6.0.4)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: toolz in /opt/conda/lib/python3.9/site-packages (from altair<6.0,>=4.2.0->gradio->fschat==0.2.19) (0.12.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: jsonschema>=3.0 in /opt/conda/lib/python3.9/site-packages (from altair<6.0,>=4.2.0->gradio->fschat==0.2.19) (4.17.3)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: six>=1.4.0 in /opt/conda/lib/python3.9/site-packages (from docker-pycreds>=0.4.0->wandb->fschat==0.2.19) (1.16.0)\u001b[0m\n",
      "\u001b[34mCollecting gitdb<5,>=4.0.1\u001b[0m\n",
      "\u001b[34mDownloading gitdb-4.0.10-py3-none-any.whl (62 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 62.7/62.7 kB 16.6 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: fsspec in /opt/conda/lib/python3.9/site-packages (from gradio-client>=0.2.10->gradio->fschat==0.2.19) (2023.1.0)\u001b[0m\n",
      "\u001b[34mCollecting anyio<5.0,>=3.0\u001b[0m\n",
      "\u001b[34mDownloading anyio-3.7.1-py3-none-any.whl (80 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 80.9/80.9 kB 22.2 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting mdurl~=0.1\u001b[0m\n",
      "\u001b[34mDownloading mdurl-0.1.2-py3-none-any.whl (10.0 kB)\u001b[0m\n",
      "\u001b[34mCollecting linkify-it-py<3,>=1\u001b[0m\n",
      "\u001b[34mDownloading linkify_it_py-2.0.2-py3-none-any.whl (19 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: python-dateutil>=2.7 in /opt/conda/lib/python3.9/site-packages (from matplotlib~=3.0->gradio->fschat==0.2.19) (2.8.2)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pyparsing>=2.2.1 in /opt/conda/lib/python3.9/site-packages (from matplotlib~=3.0->gradio->fschat==0.2.19) (3.0.9)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: cycler>=0.10 in /opt/conda/lib/python3.9/site-packages (from matplotlib~=3.0->gradio->fschat==0.2.19) (0.11.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: kiwisolver>=1.0.1 in /opt/conda/lib/python3.9/site-packages (from matplotlib~=3.0->gradio->fschat==0.2.19) (1.4.4)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: contourpy>=1.0.1 in /opt/conda/lib/python3.9/site-packages (from matplotlib~=3.0->gradio->fschat==0.2.19) (1.0.7)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: fonttools>=4.22.0 in /opt/conda/lib/python3.9/site-packages (from matplotlib~=3.0->gradio->fschat==0.2.19) (4.38.0)\u001b[0m\n",
      "\u001b[35mPreparing metadata (setup.py): finished with status 'done'\u001b[0m\n",
      "\u001b[35mRequirement already satisfied: protobuf!=4.21.0,<5,>=3.15.0 in /opt/conda/lib/python3.9/site-packages (from wandb->fschat==0.2.19) (3.20.2)\u001b[0m\n",
      "\u001b[35mCollecting docker-pycreds>=0.4.0\u001b[0m\n",
      "\u001b[35mDownloading docker_pycreds-0.4.0-py2.py3-none-any.whl (9.0 kB)\u001b[0m\n",
      "\u001b[35mRequirement already satisfied: multidict<7.0,>=4.5 in /opt/conda/lib/python3.9/site-packages (from aiohttp~=3.0->gradio->fschat==0.2.19) (6.0.4)\u001b[0m\n",
      "\u001b[35mRequirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /opt/conda/lib/python3.9/site-packages (from aiohttp~=3.0->gradio->fschat==0.2.19) (4.0.2)\u001b[0m\n",
      "\u001b[35mRequirement already satisfied: frozenlist>=1.1.1 in /opt/conda/lib/python3.9/site-packages (from aiohttp~=3.0->gradio->fschat==0.2.19) (1.3.3)\u001b[0m\n",
      "\u001b[35mRequirement already satisfied: yarl<2.0,>=1.0 in /opt/conda/lib/python3.9/site-packages (from aiohttp~=3.0->gradio->fschat==0.2.19) (1.8.2)\u001b[0m\n",
      "\u001b[35mRequirement already satisfied: aiosignal>=1.1.2 in /opt/conda/lib/python3.9/site-packages (from aiohttp~=3.0->gradio->fschat==0.2.19) (1.3.1)\u001b[0m\n",
      "\u001b[35mRequirement already satisfied: attrs>=17.3.0 in /opt/conda/lib/python3.9/site-packages (from aiohttp~=3.0->gradio->fschat==0.2.19) (22.2.0)\u001b[0m\n",
      "\u001b[35mRequirement already satisfied: toolz in /opt/conda/lib/python3.9/site-packages (from altair<6.0,>=4.2.0->gradio->fschat==0.2.19) (0.12.0)\u001b[0m\n",
      "\u001b[35mRequirement already satisfied: jsonschema>=3.0 in /opt/conda/lib/python3.9/site-packages (from altair<6.0,>=4.2.0->gradio->fschat==0.2.19) (4.17.3)\u001b[0m\n",
      "\u001b[35mRequirement already satisfied: six>=1.4.0 in /opt/conda/lib/python3.9/site-packages (from docker-pycreds>=0.4.0->wandb->fschat==0.2.19) (1.16.0)\u001b[0m\n",
      "\u001b[35mCollecting gitdb<5,>=4.0.1\u001b[0m\n",
      "\u001b[35mDownloading gitdb-4.0.10-py3-none-any.whl (62 kB)\u001b[0m\n",
      "\u001b[35m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 62.7/62.7 kB 15.8 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[35mRequirement already satisfied: fsspec in /opt/conda/lib/python3.9/site-packages (from gradio-client>=0.2.10->gradio->fschat==0.2.19) (2023.1.0)\u001b[0m\n",
      "\u001b[35mCollecting anyio<5.0,>=3.0\u001b[0m\n",
      "\u001b[35mDownloading anyio-3.7.1-py3-none-any.whl (80 kB)\u001b[0m\n",
      "\u001b[35m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 80.9/80.9 kB 21.8 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[35mCollecting mdurl~=0.1\u001b[0m\n",
      "\u001b[35mDownloading mdurl-0.1.2-py3-none-any.whl (10.0 kB)\u001b[0m\n",
      "\u001b[35mCollecting linkify-it-py<3,>=1\u001b[0m\n",
      "\u001b[35mDownloading linkify_it_py-2.0.2-py3-none-any.whl (19 kB)\u001b[0m\n",
      "\u001b[35mRequirement already satisfied: cycler>=0.10 in /opt/conda/lib/python3.9/site-packages (from matplotlib~=3.0->gradio->fschat==0.2.19) (0.11.0)\u001b[0m\n",
      "\u001b[35mRequirement already satisfied: kiwisolver>=1.0.1 in /opt/conda/lib/python3.9/site-packages (from matplotlib~=3.0->gradio->fschat==0.2.19) (1.4.4)\u001b[0m\n",
      "\u001b[35mRequirement already satisfied: pyparsing>=2.2.1 in /opt/conda/lib/python3.9/site-packages (from matplotlib~=3.0->gradio->fschat==0.2.19) (3.0.9)\u001b[0m\n",
      "\u001b[35mRequirement already satisfied: python-dateutil>=2.7 in /opt/conda/lib/python3.9/site-packages (from matplotlib~=3.0->gradio->fschat==0.2.19) (2.8.2)\u001b[0m\n",
      "\u001b[35mRequirement already satisfied: contourpy>=1.0.1 in /opt/conda/lib/python3.9/site-packages (from matplotlib~=3.0->gradio->fschat==0.2.19) (1.0.7)\u001b[0m\n",
      "\u001b[35mRequirement already satisfied: fonttools>=4.22.0 in /opt/conda/lib/python3.9/site-packages (from matplotlib~=3.0->gradio->fschat==0.2.19) (4.38.0)\u001b[0m\n",
      "\u001b[35mCollecting mdit-py-plugins<=0.3.3\u001b[0m\n",
      "\u001b[35mDownloading mdit_py_plugins-0.3.2-py3-none-any.whl (50 kB)\u001b[0m\n",
      "\u001b[35m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 50.4/50.4 kB 13.2 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[35mDownloading mdit_py_plugins-0.3.1-py3-none-any.whl (46 kB)\u001b[0m\n",
      "\u001b[35m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 46.5/46.5 kB 11.5 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[35mDownloading mdit_py_plugins-0.3.0-py3-none-any.whl (43 kB)\u001b[0m\n",
      "\u001b[35m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 43.7/43.7 kB 8.0 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[35mDownloading mdit_py_plugins-0.2.8-py3-none-any.whl (41 kB)\u001b[0m\n",
      "\u001b[35m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 41.0/41.0 kB 10.3 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[35mDownloading mdit_py_plugins-0.2.7-py3-none-any.whl (41 kB)\u001b[0m\n",
      "\u001b[35m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 41.0/41.0 kB 11.1 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[35mDownloading mdit_py_plugins-0.2.6-py3-none-any.whl (39 kB)\u001b[0m\n",
      "\u001b[35mDownloading mdit_py_plugins-0.2.5-py3-none-any.whl (39 kB)\u001b[0m\n",
      "\u001b[35mDownloading mdit_py_plugins-0.2.4-py3-none-any.whl (39 kB)\u001b[0m\n",
      "\u001b[35mDownloading mdit_py_plugins-0.2.3-py3-none-any.whl (39 kB)\u001b[0m\n",
      "\u001b[35mDownloading mdit_py_plugins-0.2.2-py3-none-any.whl (39 kB)\u001b[0m\n",
      "\u001b[35mDownloading mdit_py_plugins-0.2.1-py3-none-any.whl (38 kB)\u001b[0m\n",
      "\u001b[35mDownloading mdit_py_plugins-0.2.0-py3-none-any.whl (38 kB)\u001b[0m\n",
      "\u001b[35mDownloading mdit_py_plugins-0.1.0-py3-none-any.whl (37 kB)\u001b[0m\n",
      "\u001b[35mINFO: pip is looking at multiple versions of matplotlib to determine which version is compatible with other requirements. This could take a while.\u001b[0m\n",
      "\u001b[34mCollecting mdit-py-plugins<=0.3.3\u001b[0m\n",
      "\u001b[34mDownloading mdit_py_plugins-0.3.2-py3-none-any.whl (50 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 50.4/50.4 kB 14.3 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading mdit_py_plugins-0.3.1-py3-none-any.whl (46 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 46.5/46.5 kB 11.9 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading mdit_py_plugins-0.3.0-py3-none-any.whl (43 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 43.7/43.7 kB 12.8 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading mdit_py_plugins-0.2.8-py3-none-any.whl (41 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 41.0/41.0 kB 9.5 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading mdit_py_plugins-0.2.7-py3-none-any.whl (41 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 41.0/41.0 kB 8.9 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading mdit_py_plugins-0.2.6-py3-none-any.whl (39 kB)\u001b[0m\n",
      "\u001b[34mDownloading mdit_py_plugins-0.2.5-py3-none-any.whl (39 kB)\u001b[0m\n",
      "\u001b[34mDownloading mdit_py_plugins-0.2.4-py3-none-any.whl (39 kB)\u001b[0m\n",
      "\u001b[34mDownloading mdit_py_plugins-0.2.3-py3-none-any.whl (39 kB)\u001b[0m\n",
      "\u001b[34mDownloading mdit_py_plugins-0.2.2-py3-none-any.whl (39 kB)\u001b[0m\n",
      "\u001b[34mDownloading mdit_py_plugins-0.2.1-py3-none-any.whl (38 kB)\u001b[0m\n",
      "\u001b[34mDownloading mdit_py_plugins-0.2.0-py3-none-any.whl (38 kB)\u001b[0m\n",
      "\u001b[34mDownloading mdit_py_plugins-0.1.0-py3-none-any.whl (37 kB)\u001b[0m\n",
      "\u001b[34mINFO: pip is looking at multiple versions of matplotlib to determine which version is compatible with other requirements. This could take a while.\u001b[0m\n",
      "\u001b[34mCollecting matplotlib~=3.0\u001b[0m\n",
      "\u001b[34mDownloading matplotlib-3.7.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (11.6 MB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 11.6/11.6 MB 86.4 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mINFO: pip is looking at multiple versions of markupsafe to determine which version is compatible with other requirements. This could take a while.\u001b[0m\n",
      "\u001b[34mCollecting markupsafe~=2.0\u001b[0m\n",
      "\u001b[34mDownloading MarkupSafe-2.1.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (25 kB)\u001b[0m\n",
      "\u001b[34mINFO: pip is looking at multiple versions of markdown-it-py[linkify] to determine which version is compatible with other requirements. This could take a while.\u001b[0m\n",
      "\u001b[34mCollecting markdown-it-py[linkify]>=2.0.0\u001b[0m\n",
      "\u001b[34mDownloading markdown_it_py-2.2.0-py3-none-any.whl (84 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 84.5/84.5 kB 22.0 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pytz>=2020.1 in /opt/conda/lib/python3.9/site-packages (from pandas<3.0,>=1.0->gradio->fschat==0.2.19) (2022.7.1)\u001b[0m\n",
      "\u001b[35mCollecting matplotlib~=3.0\u001b[0m\n",
      "\u001b[35mDownloading matplotlib-3.7.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (11.6 MB)\u001b[0m\n",
      "\u001b[35m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 11.6/11.6 MB 77.1 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[35mINFO: pip is looking at multiple versions of markupsafe to determine which version is compatible with other requirements. This could take a while.\u001b[0m\n",
      "\u001b[35mCollecting markupsafe~=2.0\u001b[0m\n",
      "\u001b[35mDownloading MarkupSafe-2.1.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (25 kB)\u001b[0m\n",
      "\u001b[35mINFO: pip is looking at multiple versions of markdown-it-py[linkify] to determine which version is compatible with other requirements. This could take a while.\u001b[0m\n",
      "\u001b[35mCollecting markdown-it-py[linkify]>=2.0.0\u001b[0m\n",
      "\u001b[35mDownloading markdown_it_py-2.2.0-py3-none-any.whl (84 kB)\u001b[0m\n",
      "\u001b[35m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 84.5/84.5 kB 22.5 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[35mRequirement already satisfied: pytz>=2020.1 in /opt/conda/lib/python3.9/site-packages (from pandas<3.0,>=1.0->gradio->fschat==0.2.19) (2022.7.1)\u001b[0m\n",
      "\u001b[35mCollecting svgwrite\u001b[0m\n",
      "\u001b[35mDownloading svgwrite-1.4.3-py3-none-any.whl (67 kB)\u001b[0m\n",
      "\u001b[35m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 67.1/67.1 kB 15.7 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[35mRequirement already satisfied: exceptiongroup in /opt/conda/lib/python3.9/site-packages (from anyio<5.0,>=3.0->httpcore<0.18.0,>=0.15.0->httpx->fschat==0.2.19) (1.1.0)\u001b[0m\n",
      "\u001b[35mCollecting smmap<6,>=3.0.1\u001b[0m\n",
      "\u001b[35mDownloading smmap-5.0.0-py3-none-any.whl (24 kB)\u001b[0m\n",
      "\u001b[35mRequirement already satisfied: pyrsistent!=0.17.0,!=0.17.1,!=0.17.2,>=0.14.0 in /opt/conda/lib/python3.9/site-packages (from jsonschema>=3.0->altair<6.0,>=4.2.0->gradio->fschat==0.2.19) (0.19.3)\u001b[0m\n",
      "\u001b[34mCollecting svgwrite\u001b[0m\n",
      "\u001b[34mDownloading svgwrite-1.4.3-py3-none-any.whl (67 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 67.1/67.1 kB 13.2 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: exceptiongroup in /opt/conda/lib/python3.9/site-packages (from anyio<5.0,>=3.0->httpcore<0.18.0,>=0.15.0->httpx->fschat==0.2.19) (1.1.0)\u001b[0m\n",
      "\u001b[34mCollecting smmap<6,>=3.0.1\u001b[0m\n",
      "\u001b[34mDownloading smmap-5.0.0-py3-none-any.whl (24 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pyrsistent!=0.17.0,!=0.17.1,!=0.17.2,>=0.14.0 in /opt/conda/lib/python3.9/site-packages (from jsonschema>=3.0->altair<6.0,>=4.2.0->gradio->fschat==0.2.19) (0.19.3)\u001b[0m\n",
      "\u001b[34mCollecting uc-micro-py\u001b[0m\n",
      "\u001b[34mDownloading uc_micro_py-1.0.2-py3-none-any.whl (6.2 kB)\u001b[0m\n",
      "\u001b[34mBuilding wheels for collected packages: fschat, ffmpy, pathtools, wavedrom\u001b[0m\n",
      "\u001b[34mBuilding editable for fschat (pyproject.toml): started\u001b[0m\n",
      "\u001b[34mBuilding editable for fschat (pyproject.toml): finished with status 'done'\u001b[0m\n",
      "\u001b[34mCreated wheel for fschat: filename=fschat-0.2.19-0.editable-py3-none-any.whl size=12621 sha256=bbd2da61e5d3e0175717fcbd8be631a6c291b8ca2ec534c09c258576578afa73\u001b[0m\n",
      "\u001b[34mStored in directory: /tmp/pip-ephem-wheel-cache-5has4o0d/wheels/40/03/3a/5f39818cea87b3c154b54d046a775b3da4b8ed9b642b8d50e6\u001b[0m\n",
      "\u001b[34mBuilding wheel for ffmpy (setup.py): started\u001b[0m\n",
      "\u001b[35mCollecting uc-micro-py\u001b[0m\n",
      "\u001b[35mDownloading uc_micro_py-1.0.2-py3-none-any.whl (6.2 kB)\u001b[0m\n",
      "\u001b[35mBuilding wheels for collected packages: fschat, ffmpy, pathtools, wavedrom\u001b[0m\n",
      "\u001b[35mBuilding editable for fschat (pyproject.toml): started\u001b[0m\n",
      "\u001b[35mBuilding editable for fschat (pyproject.toml): finished with status 'done'\u001b[0m\n",
      "\u001b[35mCreated wheel for fschat: filename=fschat-0.2.19-0.editable-py3-none-any.whl size=12621 sha256=0e42c2a8c4d3bd56b2666ba6e687140b178c64c8d014999ec86a3cdd505110c1\u001b[0m\n",
      "\u001b[35mStored in directory: /tmp/pip-ephem-wheel-cache-nlbckrqt/wheels/40/03/3a/5f39818cea87b3c154b54d046a775b3da4b8ed9b642b8d50e6\u001b[0m\n",
      "\u001b[35mBuilding wheel for ffmpy (setup.py): started\u001b[0m\n",
      "\u001b[35mBuilding wheel for ffmpy (setup.py): finished with status 'done'\u001b[0m\n",
      "\u001b[35mCreated wheel for ffmpy: filename=ffmpy-0.3.1-py3-none-any.whl size=5580 sha256=a8ada7153e80881f6e6cdaee3e3eb1328d38cab5e4da61f629037d99485f59f0\u001b[0m\n",
      "\u001b[35mStored in directory: /root/.cache/pip/wheels/1f/f1/8d/367922b023b526b7c2ced5db30932def7b18cf39d7ac6e8572\u001b[0m\n",
      "\u001b[35mBuilding wheel for pathtools (setup.py): started\u001b[0m\n",
      "\u001b[34mBuilding wheel for ffmpy (setup.py): finished with status 'done'\u001b[0m\n",
      "\u001b[34mCreated wheel for ffmpy: filename=ffmpy-0.3.1-py3-none-any.whl size=5580 sha256=a8ada7153e80881f6e6cdaee3e3eb1328d38cab5e4da61f629037d99485f59f0\u001b[0m\n",
      "\u001b[34mStored in directory: /root/.cache/pip/wheels/1f/f1/8d/367922b023b526b7c2ced5db30932def7b18cf39d7ac6e8572\u001b[0m\n",
      "\u001b[34mBuilding wheel for pathtools (setup.py): started\u001b[0m\n",
      "\u001b[34mBuilding wheel for pathtools (setup.py): finished with status 'done'\u001b[0m\n",
      "\u001b[34mCreated wheel for pathtools: filename=pathtools-0.1.2-py3-none-any.whl size=8791 sha256=15338a2affbdcdfe7b43cd00af51a6ec3801753844033c6fe463b91a9d87cd5f\u001b[0m\n",
      "\u001b[34mStored in directory: /root/.cache/pip/wheels/b7/0a/67/ada2a22079218c75a88361c0782855cc72aebc4d18d0289d05\u001b[0m\n",
      "\u001b[34mBuilding wheel for wavedrom (setup.py): started\u001b[0m\n",
      "\u001b[35mBuilding wheel for pathtools (setup.py): finished with status 'done'\u001b[0m\n",
      "\u001b[35mCreated wheel for pathtools: filename=pathtools-0.1.2-py3-none-any.whl size=8791 sha256=15338a2affbdcdfe7b43cd00af51a6ec3801753844033c6fe463b91a9d87cd5f\u001b[0m\n",
      "\u001b[35mStored in directory: /root/.cache/pip/wheels/b7/0a/67/ada2a22079218c75a88361c0782855cc72aebc4d18d0289d05\u001b[0m\n",
      "\u001b[35mBuilding wheel for wavedrom (setup.py): started\u001b[0m\n",
      "\u001b[35mBuilding wheel for wavedrom (setup.py): finished with status 'done'\u001b[0m\n",
      "\u001b[35mCreated wheel for wavedrom: filename=wavedrom-2.0.3.post3-py2.py3-none-any.whl size=29933 sha256=bd83b548c7ea4cf1266d61af8a196a2a10d1b64cf314a24b792f7a572e666a77\u001b[0m\n",
      "\u001b[35mStored in directory: /root/.cache/pip/wheels/81/08/ec/3e7bb60504c4ebf08e1d5c88e9abb85b0a3549d9f8d031113f\u001b[0m\n",
      "\u001b[35mSuccessfully built fschat ffmpy pathtools wavedrom\u001b[0m\n",
      "\u001b[34mBuilding wheel for wavedrom (setup.py): finished with status 'done'\u001b[0m\n",
      "\u001b[34mCreated wheel for wavedrom: filename=wavedrom-2.0.3.post3-py2.py3-none-any.whl size=29933 sha256=bd83b548c7ea4cf1266d61af8a196a2a10d1b64cf314a24b792f7a572e666a77\u001b[0m\n",
      "\u001b[34mStored in directory: /root/.cache/pip/wheels/81/08/ec/3e7bb60504c4ebf08e1d5c88e9abb85b0a3549d9f8d031113f\u001b[0m\n",
      "\u001b[34mSuccessfully built fschat ffmpy pathtools wavedrom\u001b[0m\n",
      "\u001b[34mInstalling collected packages: safetensors, pydub, pathtools, nh3, ffmpy, websockets, uc-micro-py, typing-extensions, svgwrite, sniffio, smmap, shortuuid, setproctitle, sentry-sdk, semantic-version, python-multipart, orjson, mdurl, markdown2, h11, docker-pycreds, aiofiles, wavedrom, uvicorn, tiktoken, markdown-it-py, linkify-it-py, huggingface-hub, gitdb, anyio, starlette, mdit-py-plugins, httpcore, GitPython, altair, wandb, peft, httpx, fastapi, gradio-client, gradio, fschat\u001b[0m\n",
      "\u001b[34mAttempting uninstall: typing-extensions\u001b[0m\n",
      "\u001b[34mFound existing installation: typing_extensions 4.4.0\u001b[0m\n",
      "\u001b[34mUninstalling typing_extensions-4.4.0:\u001b[0m\n",
      "\u001b[35mInstalling collected packages: safetensors, pydub, pathtools, nh3, ffmpy, websockets, uc-micro-py, typing-extensions, svgwrite, sniffio, smmap, shortuuid, setproctitle, sentry-sdk, semantic-version, python-multipart, orjson, mdurl, markdown2, h11, docker-pycreds, aiofiles, wavedrom, uvicorn, tiktoken, markdown-it-py, linkify-it-py, huggingface-hub, gitdb, anyio, starlette, mdit-py-plugins, httpcore, GitPython, altair, wandb, peft, httpx, fastapi, gradio-client, gradio, fschat\u001b[0m\n",
      "\u001b[35mAttempting uninstall: typing-extensions\u001b[0m\n",
      "\u001b[35mFound existing installation: typing_extensions 4.4.0\u001b[0m\n",
      "\u001b[35mUninstalling typing_extensions-4.4.0:\u001b[0m\n",
      "\u001b[35mSuccessfully uninstalled typing_extensions-4.4.0\u001b[0m\n",
      "\u001b[34mSuccessfully uninstalled typing_extensions-4.4.0\u001b[0m\n",
      "\u001b[34mAttempting uninstall: huggingface-hub\u001b[0m\n",
      "\u001b[34mFound existing installation: huggingface-hub 0.12.0\u001b[0m\n",
      "\u001b[34mUninstalling huggingface-hub-0.12.0:\u001b[0m\n",
      "\u001b[34mSuccessfully uninstalled huggingface-hub-0.12.0\u001b[0m\n",
      "\u001b[35mAttempting uninstall: huggingface-hub\u001b[0m\n",
      "\u001b[35mFound existing installation: huggingface-hub 0.12.0\u001b[0m\n",
      "\u001b[35mUninstalling huggingface-hub-0.12.0:\u001b[0m\n",
      "\u001b[35mSuccessfully uninstalled huggingface-hub-0.12.0\u001b[0m\n",
      "\u001b[35mSuccessfully installed GitPython-3.1.32 aiofiles-23.1.0 altair-5.0.1 anyio-3.7.1 docker-pycreds-0.4.0 fastapi-0.100.0 ffmpy-0.3.1 fschat-0.2.19 gitdb-4.0.10 gradio-3.38.0 gradio-client-0.2.10 h11-0.14.0 httpcore-0.17.3 httpx-0.24.1 huggingface-hub-0.16.4 linkify-it-py-2.0.2 markdown-it-py-2.2.0 markdown2-2.4.9 mdit-py-plugins-0.3.3 mdurl-0.1.2 nh3-0.2.14 orjson-3.9.2 pathtools-0.1.2 peft-0.4.0 pydub-0.25.1 python-multipart-0.0.6 safetensors-0.3.1 semantic-version-2.10.0 sentry-sdk-1.28.1 setproctitle-1.3.2 shortuuid-1.0.11 smmap-5.0.0 sniffio-1.3.0 starlette-0.27.0 svgwrite-1.4.3 tiktoken-0.4.0 typing-extensions-4.7.1 uc-micro-py-1.0.2 uvicorn-0.23.1 wandb-0.15.5 wavedrom-2.0.3.post3 websockets-11.0.3\u001b[0m\n",
      "\u001b[35mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\n",
      "\u001b[35m[notice] A new release of pip is available: 23.0 -> 23.2\u001b[0m\n",
      "\u001b[35m[notice] To update, run: pip install --upgrade pip\u001b[0m\n",
      "\u001b[35mtorchrun --nproc_per_node 8 --nnodes 2 --node_rank 1 --master_addr algo-1 --master_port 23456 ./fastchat/train/train_mem.py --deepspeed ds.json --model_name_or_path /tmp/llama_pretrain/ --data_path data/dummy_conversation.json --output_dir /tmp/llama_out --num_train_epochs 1 --per_device_train_batch_size 1 --per_device_eval_batch_size 1 --gradient_accumulation_steps 4 --evaluation_strategy no --save_strategy no --save_steps 2000 --save_total_limit 1 --learning_rate 2e-5 --weight_decay 0. --warmup_ratio 0.03 --lr_scheduler_type cosine --logging_steps 1 --cache_dir '/tmp' --model_max_length 2048 --gradient_checkpointing True --lazy_preprocess True --fp16 True --tf32 True --report_to none\u001b[0m\n",
      "\u001b[35mWARNING:torch.distributed.run:\u001b[0m\n",
      "\u001b[35m*****************************************\u001b[0m\n",
      "\u001b[35mSetting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. \u001b[0m\n",
      "\u001b[35m*****************************************\u001b[0m\n",
      "\u001b[34mSuccessfully installed GitPython-3.1.32 aiofiles-23.1.0 altair-5.0.1 anyio-3.7.1 docker-pycreds-0.4.0 fastapi-0.100.0 ffmpy-0.3.1 fschat-0.2.19 gitdb-4.0.10 gradio-3.38.0 gradio-client-0.2.10 h11-0.14.0 httpcore-0.17.3 httpx-0.24.1 huggingface-hub-0.16.4 linkify-it-py-2.0.2 markdown-it-py-2.2.0 markdown2-2.4.9 mdit-py-plugins-0.3.3 mdurl-0.1.2 nh3-0.2.14 orjson-3.9.2 pathtools-0.1.2 peft-0.4.0 pydub-0.25.1 python-multipart-0.0.6 safetensors-0.3.1 semantic-version-2.10.0 sentry-sdk-1.28.1 setproctitle-1.3.2 shortuuid-1.0.11 smmap-5.0.0 sniffio-1.3.0 starlette-0.27.0 svgwrite-1.4.3 tiktoken-0.4.0 typing-extensions-4.7.1 uc-micro-py-1.0.2 uvicorn-0.23.1 wandb-0.15.5 wavedrom-2.0.3.post3 websockets-11.0.3\u001b[0m\n",
      "\u001b[34mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\n",
      "\u001b[34m[notice] A new release of pip is available: 23.0 -> 23.2\u001b[0m\n",
      "\u001b[34m[notice] To update, run: pip install --upgrade pip\u001b[0m\n",
      "\u001b[34mtorchrun --nproc_per_node 8 --nnodes 2 --node_rank 0 --master_addr algo-1 --master_port 23456 ./fastchat/train/train_mem.py --deepspeed ds.json --model_name_or_path /tmp/llama_pretrain/ --data_path data/dummy_conversation.json --output_dir /tmp/llama_out --num_train_epochs 1 --per_device_train_batch_size 1 --per_device_eval_batch_size 1 --gradient_accumulation_steps 4 --evaluation_strategy no --save_strategy no --save_steps 2000 --save_total_limit 1 --learning_rate 2e-5 --weight_decay 0. --warmup_ratio 0.03 --lr_scheduler_type cosine --logging_steps 1 --cache_dir '/tmp' --model_max_length 2048 --gradient_checkpointing True --lazy_preprocess True --fp16 True --tf32 True --report_to none\u001b[0m\n",
      "\u001b[34mWARNING:torch.distributed.run:\u001b[0m\n",
      "\u001b[34m*****************************************\u001b[0m\n",
      "\u001b[34mSetting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. \u001b[0m\n",
      "\u001b[34m*****************************************\u001b[0m\n",
      "\u001b[34m[2023-07-21 09:54:37,975] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)\u001b[0m\n",
      "\u001b[34m[2023-07-21 09:54:37,980] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)\u001b[0m\n",
      "\u001b[34m[2023-07-21 09:54:37,980] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)\u001b[0m\n",
      "\u001b[34m[2023-07-21 09:54:37,980] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)\u001b[0m\n",
      "\u001b[34m[2023-07-21 09:54:37,980] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)\u001b[0m\n",
      "\u001b[34m[2023-07-21 09:54:37,980] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)\u001b[0m\n",
      "\u001b[34m[2023-07-21 09:54:37,981] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)\u001b[0m\n",
      "\u001b[34m[2023-07-21 09:54:37,983] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)\u001b[0m\n",
      "\u001b[35m[2023-07-21 09:54:37,912] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)\u001b[0m\n",
      "\u001b[35m[2023-07-21 09:54:37,912] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)\u001b[0m\n",
      "\u001b[35m[2023-07-21 09:54:37,912] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)\u001b[0m\n",
      "\u001b[35m[2023-07-21 09:54:37,912] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)\u001b[0m\n",
      "\u001b[35m[2023-07-21 09:54:37,912] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)\u001b[0m\n",
      "\u001b[35m[2023-07-21 09:54:37,913] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)\u001b[0m\n",
      "\u001b[35m[2023-07-21 09:54:37,926] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)\u001b[0m\n",
      "\u001b[35m[2023-07-21 09:54:37,963] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)\u001b[0m\n",
      "\u001b[34m[2023-07-21 09:54:41,864] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented\u001b[0m\n",
      "\u001b[34m[2023-07-21 09:54:41,864] [INFO] [comm.py:616:init_distributed] cdb=None\u001b[0m\n",
      "\u001b[34m[2023-07-21 09:54:41,872] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented\u001b[0m\n",
      "\u001b[34m[2023-07-21 09:54:41,872] [INFO] [comm.py:616:init_distributed] cdb=None\u001b[0m\n",
      "\u001b[34m[2023-07-21 09:54:41,872] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented\u001b[0m\n",
      "\u001b[34m[2023-07-21 09:54:41,872] [INFO] [comm.py:616:init_distributed] cdb=None\u001b[0m\n",
      "\u001b[34m[2023-07-21 09:54:41,872] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented\u001b[0m\n",
      "\u001b[34m[2023-07-21 09:54:41,873] [INFO] [comm.py:616:init_distributed] cdb=None\u001b[0m\n",
      "\u001b[34m[2023-07-21 09:54:41,873] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented\u001b[0m\n",
      "\u001b[34m[2023-07-21 09:54:41,873] [INFO] [comm.py:616:init_distributed] cdb=None\u001b[0m\n",
      "\u001b[34m[2023-07-21 09:54:41,894] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented\u001b[0m\n",
      "\u001b[34m[2023-07-21 09:54:41,894] [INFO] [comm.py:616:init_distributed] cdb=None\u001b[0m\n",
      "\u001b[34m[2023-07-21 09:54:41,894] [INFO] [comm.py:643:init_distributed] Initializing TorchBackend in DeepSpeed with backend nccl\u001b[0m\n",
      "\u001b[34m[2023-07-21 09:54:41,902] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented\u001b[0m\n",
      "\u001b[34m[2023-07-21 09:54:41,902] [INFO] [comm.py:616:init_distributed] cdb=None\u001b[0m\n",
      "\u001b[34m[2023-07-21 09:54:41,935] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented\u001b[0m\n",
      "\u001b[34m[2023-07-21 09:54:41,935] [INFO] [comm.py:616:init_distributed] cdb=None\u001b[0m\n",
      "\u001b[35m[2023-07-21 09:54:41,756] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented\u001b[0m\n",
      "\u001b[35m[2023-07-21 09:54:41,756] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented\u001b[0m\n",
      "\u001b[35m[2023-07-21 09:54:41,756] [INFO] [comm.py:616:init_distributed] cdb=None\u001b[0m\n",
      "\u001b[35m[2023-07-21 09:54:41,756] [INFO] [comm.py:616:init_distributed] cdb=None\u001b[0m\n",
      "\u001b[35m[2023-07-21 09:54:41,757] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented\u001b[0m\n",
      "\u001b[35m[2023-07-21 09:54:41,757] [INFO] [comm.py:616:init_distributed] cdb=None\u001b[0m\n",
      "\u001b[35m[2023-07-21 09:54:41,772] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented\u001b[0m\n",
      "\u001b[35m[2023-07-21 09:54:41,772] [INFO] [comm.py:616:init_distributed] cdb=None\u001b[0m\n",
      "\u001b[35m[2023-07-21 09:54:41,797] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented\u001b[0m\n",
      "\u001b[35m[2023-07-21 09:54:41,797] [INFO] [comm.py:616:init_distributed] cdb=None\u001b[0m\n",
      "\u001b[35m[2023-07-21 09:54:41,816] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented\u001b[0m\n",
      "\u001b[35m[2023-07-21 09:54:41,816] [INFO] [comm.py:616:init_distributed] cdb=None\u001b[0m\n",
      "\u001b[35m[2023-07-21 09:54:41,830] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented\u001b[0m\n",
      "\u001b[35m[2023-07-21 09:54:41,830] [INFO] [comm.py:616:init_distributed] cdb=None\u001b[0m\n",
      "\u001b[35m[2023-07-21 09:54:41,853] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented\u001b[0m\n",
      "\u001b[35m[2023-07-21 09:54:41,853] [INFO] [comm.py:616:init_distributed] cdb=None\u001b[0m\n",
      "\u001b[34mNCCL version 2.14.3+cuda11.7\u001b[0m\n",
      "\u001b[34m[2023-07-21 09:54:47,576] [INFO] [partition_parameters.py:326:__exit__] finished initializing model with 6.74B parameters\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[35mLoading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[35mLoading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[35mLoading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]#015Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[35mLoading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[35mLoading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[35mLoading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[35mLoading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  50%|█████     | 1/2 [00:11<00:11, 11.18s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  50%|█████     | 1/2 [00:11<00:11, 11.21s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  50%|█████     | 1/2 [00:11<00:11, 11.23s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  50%|█████     | 1/2 [00:11<00:11, 11.38s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  50%|█████     | 1/2 [00:11<00:11, 11.41s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  50%|█████     | 1/2 [00:11<00:11, 11.41s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  50%|█████     | 1/2 [00:11<00:11, 11.43s/it]\u001b[0m\n",
      "\u001b[35mLoading checkpoint shards:  50%|█████     | 1/2 [00:11<00:11, 11.10s/it]\u001b[0m\n",
      "\u001b[35mLoading checkpoint shards:  50%|█████     | 1/2 [00:11<00:11, 11.16s/it]\u001b[0m\n",
      "\u001b[35mLoading checkpoint shards:  50%|█████     | 1/2 [00:11<00:11, 11.48s/it]\u001b[0m\n",
      "\u001b[35mLoading checkpoint shards:  50%|█████     | 1/2 [00:11<00:11, 11.51s/it]\u001b[0m\n",
      "\u001b[35mLoading checkpoint shards:  50%|█████     | 1/2 [00:11<00:11, 11.53s/it]\u001b[0m\n",
      "\u001b[35mLoading checkpoint shards:  50%|█████     | 1/2 [00:11<00:11, 11.54s/it]\u001b[0m\n",
      "\u001b[35mLoading checkpoint shards:  50%|█████     | 1/2 [00:11<00:11, 11.56s/it]\u001b[0m\n",
      "\u001b[35mLoading checkpoint shards:  50%|█████     | 1/2 [00:11<00:11, 11.58s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  50%|█████     | 1/2 [00:12<00:12, 12.57s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 2/2 [00:15<00:00,  6.89s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 2/2 [00:15<00:00,  7.54s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 2/2 [00:15<00:00,  6.88s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 2/2 [00:15<00:00,  7.54s/it]\u001b[0m\n",
      "\u001b[34mUsing pad_token, but it is not set yet.\u001b[0m\n",
      "\u001b[34mUsing pad_token, but it is not set yet.\u001b[0m\n",
      "\u001b[34m-----------no pad token and add special token PAD----\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 2/2 [00:15<00:00,  6.90s/it]#015Loading checkpoint shards: 100%|██████████| 2/2 [00:15<00:00,  7.55s/it]\u001b[0m\n",
      "\u001b[34mUsing pad_token, but it is not set yet.\u001b[0m\n",
      "\u001b[34mUsing pad_token, but it is not set yet.\u001b[0m\n",
      "\u001b[34m-----------no pad token and add special token PAD----\u001b[0m\n",
      "\u001b[34mUsing pad_token, but it is not set yet.\u001b[0m\n",
      "\u001b[34mUsing pad_token, but it is not set yet.\u001b[0m\n",
      "\u001b[34m-----------no pad token and add special token PAD----\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 2/2 [00:15<00:00,  6.91s/it]#015Loading checkpoint shards: 100%|██████████| 2/2 [00:15<00:00,  7.59s/it]\u001b[0m\n",
      "\u001b[34mUsing pad_token, but it is not set yet.\u001b[0m\n",
      "\u001b[34mUsing pad_token, but it is not set yet.\u001b[0m\n",
      "\u001b[34m-----------no pad token and add special token PAD----\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 2/2 [00:15<00:00,  6.94s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 2/2 [00:15<00:00,  7.60s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 2/2 [00:15<00:00,  6.93s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 2/2 [00:15<00:00,  7.60s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 2/2 [00:15<00:00,  6.93s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 2/2 [00:15<00:00,  7.60s/it]\u001b[0m\n",
      "\u001b[34mUsing pad_token, but it is not set yet.\u001b[0m\n",
      "\u001b[34mUsing pad_token, but it is not set yet.\u001b[0m\n",
      "\u001b[34m-----------no pad token and add special token PAD----\u001b[0m\n",
      "\u001b[34mUsing pad_token, but it is not set yet.\u001b[0m\n",
      "\u001b[34mUsing pad_token, but it is not set yet.\u001b[0m\n",
      "\u001b[34m-----------no pad token and add special token PAD----\u001b[0m\n",
      "\u001b[34mUsing pad_token, but it is not set yet.\u001b[0m\n",
      "\u001b[34mUsing pad_token, but it is not set yet.\u001b[0m\n",
      "\u001b[34m-----------no pad token and add special token PAD----\u001b[0m\n",
      "\u001b[35mLoading checkpoint shards: 100%|██████████| 2/2 [00:15<00:00,  6.89s/it]#015Loading checkpoint shards: 100%|██████████| 2/2 [00:15<00:00,  7.52s/it]\u001b[0m\n",
      "\u001b[35mUsing pad_token, but it is not set yet.\u001b[0m\n",
      "\u001b[35mUsing pad_token, but it is not set yet.\u001b[0m\n",
      "\u001b[35m-----------no pad token and add special token PAD----\u001b[0m\n",
      "\u001b[35mLoading checkpoint shards: 100%|██████████| 2/2 [00:15<00:00,  6.90s/it]\u001b[0m\n",
      "\u001b[35mLoading checkpoint shards: 100%|██████████| 2/2 [00:15<00:00,  7.54s/it]\u001b[0m\n",
      "\u001b[35mUsing pad_token, but it is not set yet.\u001b[0m\n",
      "\u001b[35mUsing pad_token, but it is not set yet.\u001b[0m\n",
      "\u001b[35m-----------no pad token and add special token PAD----\u001b[0m\n",
      "\u001b[35mLoading checkpoint shards: 100%|██████████| 2/2 [00:15<00:00,  6.90s/it]#015Loading checkpoint shards: 100%|██████████| 2/2 [00:15<00:00,  7.59s/it]\u001b[0m\n",
      "\u001b[35mLoading checkpoint shards: 100%|██████████| 2/2 [00:15<00:00,  6.91s/it]\u001b[0m\n",
      "\u001b[35mLoading checkpoint shards: 100%|██████████| 2/2 [00:15<00:00,  7.60s/it]\u001b[0m\n",
      "\u001b[35mUsing pad_token, but it is not set yet.\u001b[0m\n",
      "\u001b[35mUsing pad_token, but it is not set yet.\u001b[0m\n",
      "\u001b[35m-----------no pad token and add special token PAD----\u001b[0m\n",
      "\u001b[35mUsing pad_token, but it is not set yet.\u001b[0m\n",
      "\u001b[35mUsing pad_token, but it is not set yet.\u001b[0m\n",
      "\u001b[35m-----------no pad token and add special token PAD----\u001b[0m\n",
      "\u001b[35mLoading checkpoint shards: 100%|██████████| 2/2 [00:15<00:00,  6.92s/it]\u001b[0m\n",
      "\u001b[35mLoading checkpoint shards: 100%|██████████| 2/2 [00:15<00:00,  7.61s/it]\u001b[0m\n",
      "\u001b[35mLoading checkpoint shards: 100%|██████████| 2/2 [00:15<00:00,  6.92s/it]#015Loading checkpoint shards: 100%|██████████| 2/2 [00:15<00:00,  7.61s/it]\u001b[0m\n",
      "\u001b[35mLoading checkpoint shards: 100%|██████████| 2/2 [00:15<00:00,  6.92s/it]\u001b[0m\n",
      "\u001b[35mLoading checkpoint shards: 100%|██████████| 2/2 [00:15<00:00,  7.62s/it]\u001b[0m\n",
      "\u001b[35mUsing pad_token, but it is not set yet.\u001b[0m\n",
      "\u001b[35mUsing pad_token, but it is not set yet.\u001b[0m\n",
      "\u001b[35m-----------no pad token and add special token PAD----\u001b[0m\n",
      "\u001b[35mLoading checkpoint shards: 100%|██████████| 2/2 [00:15<00:00,  6.92s/it]\u001b[0m\n",
      "\u001b[35mLoading checkpoint shards: 100%|██████████| 2/2 [00:15<00:00,  7.62s/it]\u001b[0m\n",
      "\u001b[35mUsing pad_token, but it is not set yet.\u001b[0m\n",
      "\u001b[35mUsing pad_token, but it is not set yet.\u001b[0m\n",
      "\u001b[35m-----------no pad token and add special token PAD----\u001b[0m\n",
      "\u001b[35mUsing pad_token, but it is not set yet.\u001b[0m\n",
      "\u001b[35mUsing pad_token, but it is not set yet.\u001b[0m\n",
      "\u001b[35m-----------no pad token and add special token PAD----\u001b[0m\n",
      "\u001b[35mUsing pad_token, but it is not set yet.\u001b[0m\n",
      "\u001b[35mUsing pad_token, but it is not set yet.\u001b[0m\n",
      "\u001b[35m-----------no pad token and add special token PAD----\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 2/2 [00:16<00:00,  7.49s/it]#015Loading checkpoint shards: 100%|██████████| 2/2 [00:16<00:00,  8.25s/it]\u001b[0m\n",
      "\u001b[34mUsing pad_token, but it is not set yet.\u001b[0m\n",
      "\u001b[34mUsing pad_token, but it is not set yet.\u001b[0m\n",
      "\u001b[34m-----------no pad token and add special token PAD----\u001b[0m\n",
      "\u001b[34mLoading data...\u001b[0m\n",
      "\u001b[34mFormatting inputs...Skip in lazy mode\u001b[0m\n",
      "\u001b[35mLoading data...\u001b[0m\n",
      "\u001b[35mFormatting inputs...Skip in lazy mode\u001b[0m\n",
      "\u001b[35mUsing /root/.cache/torch_extensions/py39_cu117 as PyTorch extensions root...\u001b[0m\n",
      "\u001b[35mCreating extension directory /root/.cache/torch_extensions/py39_cu117/fused_adam...\u001b[0m\n",
      "\u001b[35mUsing /root/.cache/torch_extensions/py39_cu117 as PyTorch extensions root...\u001b[0m\n",
      "\u001b[35mUsing /root/.cache/torch_extensions/py39_cu117 as PyTorch extensions root...\u001b[0m\n",
      "\u001b[35mUsing /root/.cache/torch_extensions/py39_cu117 as PyTorch extensions root...\u001b[0m\n",
      "\u001b[35mUsing /root/.cache/torch_extensions/py39_cu117 as PyTorch extensions root...\u001b[0m\n",
      "\u001b[35mUsing /root/.cache/torch_extensions/py39_cu117 as PyTorch extensions root...\u001b[0m\n",
      "\u001b[35mUsing /root/.cache/torch_extensions/py39_cu117 as PyTorch extensions root...\u001b[0m\n",
      "\u001b[35mUsing /root/.cache/torch_extensions/py39_cu117 as PyTorch extensions root...\u001b[0m\n",
      "\u001b[35mDetected CUDA files, patching ldflags\u001b[0m\n",
      "\u001b[35mEmitting ninja build file /root/.cache/torch_extensions/py39_cu117/fused_adam/build.ninja...\u001b[0m\n",
      "\u001b[35mBuilding extension module fused_adam...\u001b[0m\n",
      "\u001b[35mAllowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N)\u001b[0m\n",
      "\u001b[34mUsing /root/.cache/torch_extensions/py39_cu117 as PyTorch extensions root...\u001b[0m\n",
      "\u001b[34mCreating extension directory /root/.cache/torch_extensions/py39_cu117/fused_adam...\u001b[0m\n",
      "\u001b[34mUsing /root/.cache/torch_extensions/py39_cu117 as PyTorch extensions root...\u001b[0m\n",
      "\u001b[34mUsing /root/.cache/torch_extensions/py39_cu117 as PyTorch extensions root...\u001b[0m\n",
      "\u001b[34mUsing /root/.cache/torch_extensions/py39_cu117 as PyTorch extensions root...\u001b[0m\n",
      "\u001b[34mUsing /root/.cache/torch_extensions/py39_cu117 as PyTorch extensions root...\u001b[0m\n",
      "\u001b[34mUsing /root/.cache/torch_extensions/py39_cu117 as PyTorch extensions root...\u001b[0m\n",
      "\u001b[34mUsing /root/.cache/torch_extensions/py39_cu117 as PyTorch extensions root...\u001b[0m\n",
      "\u001b[34mUsing /root/.cache/torch_extensions/py39_cu117 as PyTorch extensions root...\u001b[0m\n",
      "\u001b[34mDetected CUDA files, patching ldflags\u001b[0m\n",
      "\u001b[34mEmitting ninja build file /root/.cache/torch_extensions/py39_cu117/fused_adam/build.ninja...\u001b[0m\n",
      "\u001b[34mBuilding extension module fused_adam...\u001b[0m\n",
      "\u001b[34mAllowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N)\u001b[0m\n",
      "\u001b[34m[1/3] /usr/local/cuda/bin/nvcc  -DTORCH_EXTENSION_NAME=fused_adam -DTORCH_API_INCLUDE_EXTENSION_H -DPYBIND11_COMPILER_TYPE=\\\"_gcc\\\" -DPYBIND11_STDLIB=\\\"_libstdcpp\\\" -DPYBIND11_BUILD_ABI=\\\"_cxxabi1011\\\" -I/opt/conda/lib/python3.9/site-packages/deepspeed/ops/csrc/includes -I/opt/conda/lib/python3.9/site-packages/deepspeed/ops/csrc/adam -isystem /opt/conda/lib/python3.9/site-packages/torch/include -isystem /opt/conda/lib/python3.9/site-packages/torch/include/torch/csrc/api/include -isystem /opt/conda/lib/python3.9/site-packages/torch/include/TH -isystem /opt/conda/lib/python3.9/site-packages/torch/include/THC -isystem /usr/local/cuda/include -isystem /opt/conda/include/python3.9 -D_GLIBCXX_USE_CXX11_ABI=0 -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_BFLOAT16_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --expt-relaxed-constexpr -gencode=arch=compute_80,code=compute_80 -gencode=arch=compute_80,code=sm_80 --compiler-options '-fPIC' -O3 -DVERSION_GE_1_1 -DVERSION_GE_1_3 -DVERSION_GE_1_5 -lineinfo --use_fast_math -gencode=arch=compute_80,code=sm_80 -gencode=arch=compute_80,code=compute_80 -DBF16_AVAILABLE -std=c++14 -c /opt/conda/lib/python3.9/site-packages/deepspeed/ops/csrc/adam/multi_tensor_adam.cu -o multi_tensor_adam.cuda.o\u001b[0m\n",
      "\u001b[35m[1/3] /usr/local/cuda/bin/nvcc  -DTORCH_EXTENSION_NAME=fused_adam -DTORCH_API_INCLUDE_EXTENSION_H -DPYBIND11_COMPILER_TYPE=\\\"_gcc\\\" -DPYBIND11_STDLIB=\\\"_libstdcpp\\\" -DPYBIND11_BUILD_ABI=\\\"_cxxabi1011\\\" -I/opt/conda/lib/python3.9/site-packages/deepspeed/ops/csrc/includes -I/opt/conda/lib/python3.9/site-packages/deepspeed/ops/csrc/adam -isystem /opt/conda/lib/python3.9/site-packages/torch/include -isystem /opt/conda/lib/python3.9/site-packages/torch/include/torch/csrc/api/include -isystem /opt/conda/lib/python3.9/site-packages/torch/include/TH -isystem /opt/conda/lib/python3.9/site-packages/torch/include/THC -isystem /usr/local/cuda/include -isystem /opt/conda/include/python3.9 -D_GLIBCXX_USE_CXX11_ABI=0 -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_BFLOAT16_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --expt-relaxed-constexpr -gencode=arch=compute_80,code=compute_80 -gencode=arch=compute_80,code=sm_80 --compiler-options '-fPIC' -O3 -DVERSION_GE_1_1 -DVERSION_GE_1_3 -DVERSION_GE_1_5 -lineinfo --use_fast_math -gencode=arch=compute_80,code=sm_80 -gencode=arch=compute_80,code=compute_80 -DBF16_AVAILABLE -std=c++14 -c /opt/conda/lib/python3.9/site-packages/deepspeed/ops/csrc/adam/multi_tensor_adam.cu -o multi_tensor_adam.cuda.o\u001b[0m\n",
      "\u001b[34m[2/3] c++ -MMD -MF fused_adam_frontend.o.d -DTORCH_EXTENSION_NAME=fused_adam -DTORCH_API_INCLUDE_EXTENSION_H -DPYBIND11_COMPILER_TYPE=\\\"_gcc\\\" -DPYBIND11_STDLIB=\\\"_libstdcpp\\\" -DPYBIND11_BUILD_ABI=\\\"_cxxabi1011\\\" -I/opt/conda/lib/python3.9/site-packages/deepspeed/ops/csrc/includes -I/opt/conda/lib/python3.9/site-packages/deepspeed/ops/csrc/adam -isystem /opt/conda/lib/python3.9/site-packages/torch/include -isystem /opt/conda/lib/python3.9/site-packages/torch/include/torch/csrc/api/include -isystem /opt/conda/lib/python3.9/site-packages/torch/include/TH -isystem /opt/conda/lib/python3.9/site-packages/torch/include/THC -isystem /usr/local/cuda/include -isystem /opt/conda/include/python3.9 -D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -std=c++14 -O3 -std=c++17 -g -Wno-reorder -DVERSION_GE_1_1 -DVERSION_GE_1_3 -DVERSION_GE_1_5 -DBF16_AVAILABLE -c /opt/conda/lib/python3.9/site-packages/deepspeed/ops/csrc/adam/fused_adam_frontend.cpp -o fused_adam_frontend.o\u001b[0m\n",
      "\u001b[35m[2/3] c++ -MMD -MF fused_adam_frontend.o.d -DTORCH_EXTENSION_NAME=fused_adam -DTORCH_API_INCLUDE_EXTENSION_H -DPYBIND11_COMPILER_TYPE=\\\"_gcc\\\" -DPYBIND11_STDLIB=\\\"_libstdcpp\\\" -DPYBIND11_BUILD_ABI=\\\"_cxxabi1011\\\" -I/opt/conda/lib/python3.9/site-packages/deepspeed/ops/csrc/includes -I/opt/conda/lib/python3.9/site-packages/deepspeed/ops/csrc/adam -isystem /opt/conda/lib/python3.9/site-packages/torch/include -isystem /opt/conda/lib/python3.9/site-packages/torch/include/torch/csrc/api/include -isystem /opt/conda/lib/python3.9/site-packages/torch/include/TH -isystem /opt/conda/lib/python3.9/site-packages/torch/include/THC -isystem /usr/local/cuda/include -isystem /opt/conda/include/python3.9 -D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -std=c++14 -O3 -std=c++17 -g -Wno-reorder -DVERSION_GE_1_1 -DVERSION_GE_1_3 -DVERSION_GE_1_5 -DBF16_AVAILABLE -c /opt/conda/lib/python3.9/site-packages/deepspeed/ops/csrc/adam/fused_adam_frontend.cpp -o fused_adam_frontend.o\u001b[0m\n",
      "\u001b[34m[3/3] c++ fused_adam_frontend.o multi_tensor_adam.cuda.o -shared -L/opt/conda/lib/python3.9/site-packages/torch/lib -lc10 -lc10_cuda -ltorch_cpu -ltorch_cuda_cu -ltorch_cuda_cpp -ltorch -ltorch_python -L/usr/local/cuda/lib64 -lcudart -o fused_adam.so\u001b[0m\n",
      "\u001b[34mLoading extension module fused_adam...\u001b[0m\n",
      "\u001b[34mTime to load fused_adam op: 26.398207902908325 seconds\u001b[0m\n",
      "\u001b[34mLoading extension module fused_adam...\u001b[0m\n",
      "\u001b[34mTime to load fused_adam op: 26.357168436050415 seconds\u001b[0m\n",
      "\u001b[34mLoading extension module fused_adam...\u001b[0m\n",
      "\u001b[34mTime to load fused_adam op: 26.357445001602173 seconds\u001b[0m\n",
      "\u001b[34mLoading extension module fused_adam...\u001b[0m\n",
      "\u001b[34mTime to load fused_adam op: 26.356717348098755 seconds\u001b[0m\n",
      "\u001b[34mLoading extension module fused_adam...\u001b[0m\n",
      "\u001b[34mLoading extension module fused_adam...\u001b[0m\n",
      "\u001b[34mTime to load fused_adam op: 26.3575119972229 seconds\u001b[0m\n",
      "\u001b[34mTime to load fused_adam op: 26.355761289596558 seconds\u001b[0m\n",
      "\u001b[34mLoading extension module fused_adam...\u001b[0m\n",
      "\u001b[34mTime to load fused_adam op: 26.356231927871704 seconds\u001b[0m\n",
      "\u001b[34mLoading extension module fused_adam...\u001b[0m\n",
      "\u001b[34mTime to load fused_adam op: 26.452759504318237 seconds\u001b[0m\n",
      "\u001b[34mParameter Offload: Total persistent parameters: 266240 in 65 params\u001b[0m\n",
      "\u001b[35m[3/3] c++ fused_adam_frontend.o multi_tensor_adam.cuda.o -shared -L/opt/conda/lib/python3.9/site-packages/torch/lib -lc10 -lc10_cuda -ltorch_cpu -ltorch_cuda_cu -ltorch_cuda_cpp -ltorch -ltorch_python -L/usr/local/cuda/lib64 -lcudart -o fused_adam.so\u001b[0m\n",
      "\u001b[35mLoading extension module fused_adam...\u001b[0m\n",
      "\u001b[35mTime to load fused_adam op: 26.731706142425537 seconds\u001b[0m\n",
      "\u001b[35mLoading extension module fused_adam...\u001b[0m\n",
      "\u001b[35mTime to load fused_adam op: 26.657614707946777 seconds\u001b[0m\n",
      "\u001b[35mLoading extension module fused_adam...\u001b[0m\n",
      "\u001b[35mTime to load fused_adam op: 26.74975275993347 seconds\u001b[0m\n",
      "\u001b[35mLoading extension module fused_adam...\u001b[0m\n",
      "\u001b[35mTime to load fused_adam op: 26.75052046775818 seconds\u001b[0m\n",
      "\u001b[35mLoading extension module fused_adam...\u001b[0m\n",
      "\u001b[35mLoading extension module fused_adam...\u001b[0m\n",
      "\u001b[35mTime to load fused_adam op: 26.75739550590515 seconds\u001b[0m\n",
      "\u001b[35mTime to load fused_adam op: 26.756890296936035 seconds\u001b[0m\n",
      "\u001b[35mLoading extension module fused_adam...\u001b[0m\n",
      "\u001b[35mTime to load fused_adam op: 26.74615788459778 seconds\u001b[0m\n",
      "\u001b[35mLoading extension module fused_adam...\u001b[0m\n",
      "\u001b[35mTime to load fused_adam op: 26.756934881210327 seconds\u001b[0m\n",
      "\u001b[34m0%|          | 0/14 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[34m[2023-07-21 09:55:40.782: W smdistributed/modelparallel/torch/nn/predefined_hooks.py:78] Found unsupported HuggingFace version 4.28.0 for automated tensor parallelism. HuggingFace modules will not be automatically distributed. You can use smp.tp_register_with_module API to register desired modules for tensor parallelism, or directly instantiate an smp.nn.DistributedModule. Supported HuggingFace transformers versions for automated tensor parallelism: ['4.17.0', '4.20.1', '4.21.0']\u001b[0m\n",
      "\u001b[34m[2023-07-21 09:55:40.782: W smdistributed/modelparallel/torch/nn/predefined_hooks.py:78] Found unsupported HuggingFace version 4.28.0 for automated tensor parallelism. HuggingFace modules will not be automatically distributed. You can use smp.tp_register_with_module API to register desired modules for tensor parallelism, or directly instantiate an smp.nn.DistributedModule. Supported HuggingFace transformers versions for automated tensor parallelism: ['4.17.0', '4.20.1', '4.21.0']\u001b[0m\n",
      "\u001b[34m[2023-07-21 09:55:40.783: W smdistributed/modelparallel/torch/nn/predefined_hooks.py:78] Found unsupported HuggingFace version 4.28.0 for automated tensor parallelism. HuggingFace modules will not be automatically distributed. You can use smp.tp_register_with_module API to register desired modules for tensor parallelism, or directly instantiate an smp.nn.DistributedModule. Supported HuggingFace transformers versions for automated tensor parallelism: ['4.17.0', '4.20.1', '4.21.0']\u001b[0m\n",
      "\u001b[34m[2023-07-21 09:55:40.784: W smdistributed/modelparallel/torch/nn/predefined_hooks.py:78] Found unsupported HuggingFace version 4.28.0 for automated tensor parallelism. HuggingFace modules will not be automatically distributed. You can use smp.tp_register_with_module API to register desired modules for tensor parallelism, or directly instantiate an smp.nn.DistributedModule. Supported HuggingFace transformers versions for automated tensor parallelism: ['4.17.0', '4.20.1', '4.21.0']\u001b[0m\n",
      "\u001b[34m[2023-07-21 09:55:40.784: W smdistributed/modelparallel/torch/nn/predefined_hooks.py:78] Found unsupported HuggingFace version 4.28.0 for automated tensor parallelism. HuggingFace modules will not be automatically distributed. You can use smp.tp_register_with_module API to register desired modules for tensor parallelism, or directly instantiate an smp.nn.DistributedModule. Supported HuggingFace transformers versions for automated tensor parallelism: ['4.17.0', '4.20.1', '4.21.0']\u001b[0m\n",
      "\u001b[34m[2023-07-21 09:55:40.784: W smdistributed/modelparallel/torch/nn/predefined_hooks.py:78] Found unsupported HuggingFace version 4.28.0 for automated tensor parallelism. HuggingFace modules will not be automatically distributed. You can use smp.tp_register_with_module API to register desired modules for tensor parallelism, or directly instantiate an smp.nn.DistributedModule. Supported HuggingFace transformers versions for automated tensor parallelism: ['4.17.0', '4.20.1', '4.21.0']\u001b[0m\n",
      "\u001b[34m[2023-07-21 09:55:40.784: W smdistributed/modelparallel/torch/nn/predefined_hooks.py:78] Found unsupported HuggingFace version 4.28.0 for automated tensor parallelism. HuggingFace modules will not be automatically distributed. You can use smp.tp_register_with_module API to register desired modules for tensor parallelism, or directly instantiate an smp.nn.DistributedModule. Supported HuggingFace transformers versions for automated tensor parallelism: ['4.17.0', '4.20.1', '4.21.0']\u001b[0m\n",
      "\u001b[34m[2023-07-21 09:55:40.784: W smdistributed/modelparallel/torch/nn/predefined_hooks.py:78] Found unsupported HuggingFace version 4.28.0 for automated tensor parallelism. HuggingFace modules will not be automatically distributed. You can use smp.tp_register_with_module API to register desired modules for tensor parallelism, or directly instantiate an smp.nn.DistributedModule. Supported HuggingFace transformers versions for automated tensor parallelism: ['4.17.0', '4.20.1', '4.21.0']\u001b[0m\n",
      "\u001b[34mINFO:root:Using NamedTuple = typing._NamedTuple instead.\u001b[0m\n",
      "\u001b[34mINFO:root:Using NamedTuple = typing._NamedTuple instead.\u001b[0m\n",
      "\u001b[34mINFO:root:Using NamedTuple = typing._NamedTuple instead.\u001b[0m\n",
      "\u001b[34mINFO:root:Using NamedTuple = typing._NamedTuple instead.\u001b[0m\n",
      "\u001b[34mINFO:root:Using NamedTuple = typing._NamedTuple instead.\u001b[0m\n",
      "\u001b[34mINFO:root:Using NamedTuple = typing._NamedTuple instead.\u001b[0m\n",
      "\u001b[34mINFO:root:Using NamedTuple = typing._NamedTuple instead.\u001b[0m\n",
      "\u001b[34mINFO:root:Using NamedTuple = typing._NamedTuple instead.\u001b[0m\n",
      "\u001b[34m[2023-07-21 09:55:40.808 algo-1:281 INFO utils.py:28] RULE_JOB_STOP_SIGNAL_FILENAME: None\u001b[0m\n",
      "\u001b[34m[2023-07-21 09:55:40.808 algo-1:277 INFO utils.py:28] RULE_JOB_STOP_SIGNAL_FILENAME: None\u001b[0m\n",
      "\u001b[34m[2023-07-21 09:55:40.809 algo-1:282 INFO utils.py:28] RULE_JOB_STOP_SIGNAL_FILENAME: None\u001b[0m\n",
      "\u001b[34m[2023-07-21 09:55:40.810 algo-1:278 INFO utils.py:28] RULE_JOB_STOP_SIGNAL_FILENAME: None\u001b[0m\n",
      "\u001b[34m[2023-07-21 09:55:40.810 algo-1:280 INFO utils.py:28] RULE_JOB_STOP_SIGNAL_FILENAME: None\u001b[0m\n",
      "\u001b[34m[2023-07-21 09:55:40.811 algo-1:279 INFO utils.py:28] RULE_JOB_STOP_SIGNAL_FILENAME: None\u001b[0m\n",
      "\u001b[34m[2023-07-21 09:55:40.811 algo-1:283 INFO utils.py:28] RULE_JOB_STOP_SIGNAL_FILENAME: None\u001b[0m\n",
      "\u001b[34m[2023-07-21 09:55:40.811 algo-1:284 INFO utils.py:28] RULE_JOB_STOP_SIGNAL_FILENAME: None\u001b[0m\n",
      "\u001b[34m[2023-07-21 09:55:40.827 algo-1:281 INFO profiler_config_parser.py:111] Unable to find config at /opt/ml/input/config/profilerconfig.json. Profiler is disabled.\u001b[0m\n",
      "\u001b[34m[2023-07-21 09:55:40.827 algo-1:277 INFO profiler_config_parser.py:111] Unable to find config at /opt/ml/input/config/profilerconfig.json. Profiler is disabled.\u001b[0m\n",
      "\u001b[34m[2023-07-21 09:55:40.827 algo-1:282 INFO profiler_config_parser.py:111] Unable to find config at /opt/ml/input/config/profilerconfig.json. Profiler is disabled.\u001b[0m\n",
      "\u001b[34m[2023-07-21 09:55:40.828 algo-1:278 INFO profiler_config_parser.py:111] Unable to find config at /opt/ml/input/config/profilerconfig.json. Profiler is disabled.\u001b[0m\n",
      "\u001b[34m[2023-07-21 09:55:40.829 algo-1:280 INFO profiler_config_parser.py:111] Unable to find config at /opt/ml/input/config/profilerconfig.json. Profiler is disabled.\u001b[0m\n",
      "\u001b[34m[2023-07-21 09:55:40.829 algo-1:279 INFO profiler_config_parser.py:111] Unable to find config at /opt/ml/input/config/profilerconfig.json. Profiler is disabled.\u001b[0m\n",
      "\u001b[34m[2023-07-21 09:55:40.829 algo-1:283 INFO profiler_config_parser.py:111] Unable to find config at /opt/ml/input/config/profilerconfig.json. Profiler is disabled.\u001b[0m\n",
      "\u001b[34m[2023-07-21 09:55:40.830 algo-1:284 INFO profiler_config_parser.py:111] Unable to find config at /opt/ml/input/config/profilerconfig.json. Profiler is disabled.\u001b[0m\n",
      "\u001b[34mWARNING: tokenization mismatch 67 vs. 69\u001b[0m\n",
      "\u001b[34m`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\u001b[0m\n",
      "\u001b[34m`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\u001b[0m\n",
      "\u001b[34m`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\u001b[0m\n",
      "\u001b[34m`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\u001b[0m\n",
      "\u001b[34m`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\u001b[0m\n",
      "\u001b[34m`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\u001b[0m\n",
      "\u001b[34m`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\u001b[0m\n",
      "\u001b[34m`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\u001b[0m\n",
      "\u001b[34m`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\u001b[0m\n",
      "\u001b[34m`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\u001b[0m\n",
      "\u001b[34m`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\u001b[0m\n",
      "\u001b[34m`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\u001b[0m\n",
      "\u001b[34m`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\u001b[0m\n",
      "\u001b[34m`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\u001b[0m\n",
      "\u001b[34m`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\u001b[0m\n",
      "\u001b[34m`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\u001b[0m\n",
      "\u001b[34mWARNING: tokenization mismatch 68 vs. 70\u001b[0m\n",
      "\u001b[34mWARNING: tokenization mismatch 68 vs. 70\u001b[0m\n",
      "\u001b[35m0%|          | 0/14 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[35m[2023-07-21 09:55:40.781: W smdistributed/modelparallel/torch/nn/predefined_hooks.py:78] Found unsupported HuggingFace version 4.28.0 for automated tensor parallelism. HuggingFace modules will not be automatically distributed. You can use smp.tp_register_with_module API to register desired modules for tensor parallelism, or directly instantiate an smp.nn.DistributedModule. Supported HuggingFace transformers versions for automated tensor parallelism: ['4.17.0', '4.20.1', '4.21.0']\u001b[0m\n",
      "\u001b[35m[2023-07-21 09:55:40.782: W smdistributed/modelparallel/torch/nn/predefined_hooks.py:78] Found unsupported HuggingFace version 4.28.0 for automated tensor parallelism. HuggingFace modules will not be automatically distributed. You can use smp.tp_register_with_module API to register desired modules for tensor parallelism, or directly instantiate an smp.nn.DistributedModule. Supported HuggingFace transformers versions for automated tensor parallelism: ['4.17.0', '4.20.1', '4.21.0']\u001b[0m\n",
      "\u001b[35m[2023-07-21 09:55:40.782: W smdistributed/modelparallel/torch/nn/predefined_hooks.py:78] Found unsupported HuggingFace version 4.28.0 for automated tensor parallelism. HuggingFace modules will not be automatically distributed. You can use smp.tp_register_with_module API to register desired modules for tensor parallelism, or directly instantiate an smp.nn.DistributedModule. Supported HuggingFace transformers versions for automated tensor parallelism: ['4.17.0', '4.20.1', '4.21.0']\u001b[0m\n",
      "\u001b[35m[2023-07-21 09:55:40.782: W smdistributed/modelparallel/torch/nn/predefined_hooks.py:78] Found unsupported HuggingFace version 4.28.0 for automated tensor parallelism. HuggingFace modules will not be automatically distributed. You can use smp.tp_register_with_module API to register desired modules for tensor parallelism, or directly instantiate an smp.nn.DistributedModule. Supported HuggingFace transformers versions for automated tensor parallelism: ['4.17.0', '4.20.1', '4.21.0']\u001b[0m\n",
      "\u001b[35m[2023-07-21 09:55:40.782: W smdistributed/modelparallel/torch/nn/predefined_hooks.py:78] Found unsupported HuggingFace version 4.28.0 for automated tensor parallelism. HuggingFace modules will not be automatically distributed. You can use smp.tp_register_with_module API to register desired modules for tensor parallelism, or directly instantiate an smp.nn.DistributedModule. Supported HuggingFace transformers versions for automated tensor parallelism: ['4.17.0', '4.20.1', '4.21.0']\u001b[0m\n",
      "\u001b[35m[2023-07-21 09:55:40.782: W smdistributed/modelparallel/torch/nn/predefined_hooks.py:78] Found unsupported HuggingFace version 4.28.0 for automated tensor parallelism. HuggingFace modules will not be automatically distributed. You can use smp.tp_register_with_module API to register desired modules for tensor parallelism, or directly instantiate an smp.nn.DistributedModule. Supported HuggingFace transformers versions for automated tensor parallelism: ['4.17.0', '4.20.1', '4.21.0']\u001b[0m\n",
      "\u001b[35m[2023-07-21 09:55:40.782: W smdistributed/modelparallel/torch/nn/predefined_hooks.py:78] Found unsupported HuggingFace version 4.28.0 for automated tensor parallelism. HuggingFace modules will not be automatically distributed. You can use smp.tp_register_with_module API to register desired modules for tensor parallelism, or directly instantiate an smp.nn.DistributedModule. Supported HuggingFace transformers versions for automated tensor parallelism: ['4.17.0', '4.20.1', '4.21.0']\u001b[0m\n",
      "\u001b[35m[2023-07-21 09:55:40.783: W smdistributed/modelparallel/torch/nn/predefined_hooks.py:78] Found unsupported HuggingFace version 4.28.0 for automated tensor parallelism. HuggingFace modules will not be automatically distributed. You can use smp.tp_register_with_module API to register desired modules for tensor parallelism, or directly instantiate an smp.nn.DistributedModule. Supported HuggingFace transformers versions for automated tensor parallelism: ['4.17.0', '4.20.1', '4.21.0']\u001b[0m\n",
      "\u001b[35mINFO:root:Using NamedTuple = typing._NamedTuple instead.\u001b[0m\n",
      "\u001b[35mINFO:root:Using NamedTuple = typing._NamedTuple instead.\u001b[0m\n",
      "\u001b[35mINFO:root:Using NamedTuple = typing._NamedTuple instead.\u001b[0m\n",
      "\u001b[35mINFO:root:Using NamedTuple = typing._NamedTuple instead.\u001b[0m\n",
      "\u001b[35mINFO:root:Using NamedTuple = typing._NamedTuple instead.\u001b[0m\n",
      "\u001b[35mINFO:root:Using NamedTuple = typing._NamedTuple instead.\u001b[0m\n",
      "\u001b[35mINFO:root:Using NamedTuple = typing._NamedTuple instead.\u001b[0m\n",
      "\u001b[35mINFO:root:Using NamedTuple = typing._NamedTuple instead.\u001b[0m\n",
      "\u001b[35m[2023-07-21 09:55:40.807 algo-2:277 INFO utils.py:28] RULE_JOB_STOP_SIGNAL_FILENAME: None\u001b[0m\n",
      "\u001b[35m[2023-07-21 09:55:40.807 algo-2:275 INFO utils.py:28] RULE_JOB_STOP_SIGNAL_FILENAME: None\u001b[0m\n",
      "\u001b[35m[2023-07-21 09:55:40.808 algo-2:280 INFO utils.py:28] RULE_JOB_STOP_SIGNAL_FILENAME: None\u001b[0m\n",
      "\u001b[35m[2023-07-21 09:55:40.808 algo-2:279 INFO utils.py:28] RULE_JOB_STOP_SIGNAL_FILENAME: None\u001b[0m\n",
      "\u001b[35m[2023-07-21 09:55:40.808 algo-2:276 INFO utils.py:28] RULE_JOB_STOP_SIGNAL_FILENAME: None\u001b[0m\n",
      "\u001b[35m[2023-07-21 09:55:40.808 algo-2:273 INFO utils.py:28] RULE_JOB_STOP_SIGNAL_FILENAME: None\u001b[0m\n",
      "\u001b[35m[2023-07-21 09:55:40.809 algo-2:278 INFO utils.py:28] RULE_JOB_STOP_SIGNAL_FILENAME: None\u001b[0m\n",
      "\u001b[35m[2023-07-21 09:55:40.810 algo-2:274 INFO utils.py:28] RULE_JOB_STOP_SIGNAL_FILENAME: None\u001b[0m\n",
      "\u001b[35m[2023-07-21 09:55:40.826 algo-2:277 INFO profiler_config_parser.py:111] Unable to find config at /opt/ml/input/config/profilerconfig.json. Profiler is disabled.\u001b[0m\n",
      "\u001b[35m[2023-07-21 09:55:40.826 algo-2:275 INFO profiler_config_parser.py:111] Unable to find config at /opt/ml/input/config/profilerconfig.json. Profiler is disabled.\u001b[0m\n",
      "\u001b[35m[2023-07-21 09:55:40.826 algo-2:280 INFO profiler_config_parser.py:111] Unable to find config at /opt/ml/input/config/profilerconfig.json. Profiler is disabled.\u001b[0m\n",
      "\u001b[35m[2023-07-21 09:55:40.826 algo-2:276 INFO profiler_config_parser.py:111] Unable to find config at /opt/ml/input/config/profilerconfig.json. Profiler is disabled.\u001b[0m\n",
      "\u001b[35m[2023-07-21 09:55:40.826 algo-2:273 INFO profiler_config_parser.py:111] Unable to find config at /opt/ml/input/config/profilerconfig.json. Profiler is disabled.\u001b[0m\n",
      "\u001b[35m[2023-07-21 09:55:40.827 algo-2:279 INFO profiler_config_parser.py:111] Unable to find config at /opt/ml/input/config/profilerconfig.json. Profiler is disabled.\u001b[0m\n",
      "\u001b[35m[2023-07-21 09:55:40.827 algo-2:278 INFO profiler_config_parser.py:111] Unable to find config at /opt/ml/input/config/profilerconfig.json. Profiler is disabled.\u001b[0m\n",
      "\u001b[35m[2023-07-21 09:55:40.828 algo-2:274 INFO profiler_config_parser.py:111] Unable to find config at /opt/ml/input/config/profilerconfig.json. Profiler is disabled.\u001b[0m\n",
      "\u001b[35mWARNING: tokenization mismatch 60 vs. 62\u001b[0m\n",
      "\u001b[35m`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\u001b[0m\n",
      "\u001b[35m`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\u001b[0m\n",
      "\u001b[35m`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\u001b[0m\n",
      "\u001b[35m`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\u001b[0m\n",
      "\u001b[35m`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\u001b[0m\n",
      "\u001b[35m`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\u001b[0m\n",
      "\u001b[35m`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\u001b[0m\n",
      "\u001b[35m`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\u001b[0m\n",
      "\u001b[35m`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\u001b[0m\n",
      "\u001b[35m`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\u001b[0m\n",
      "\u001b[35m`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\u001b[0m\n",
      "\u001b[35m`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\u001b[0m\n",
      "\u001b[35m`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\u001b[0m\n",
      "\u001b[35m`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\u001b[0m\n",
      "\u001b[35m`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\u001b[0m\n",
      "\u001b[35m`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\u001b[0m\n",
      "\u001b[35mWARNING: tokenization mismatch 69 vs. 71\u001b[0m\n",
      "\u001b[35mWARNING: tokenization mismatch 72 vs. 74\u001b[0m\n",
      "\u001b[34mWARNING: tokenization mismatch 70 vs. 72\u001b[0m\n",
      "\u001b[35mWARNING: tokenization mismatch 69 vs. 71\u001b[0m\n",
      "\u001b[34m7%|▋         | 1/14 [00:33<07:15, 33.50s/it]\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34m{'loss': 3.7076, 'learning_rate': 0, 'epoch': 0.07}\u001b[0m\n",
      "\u001b[34m7%|▋         | 1/14 [00:33<07:15, 33.50s/it]\u001b[0m\n",
      "\u001b[34mWARNING: tokenization mismatch 71 vs. 73\u001b[0m\n",
      "\u001b[35m7%|▋         | 1/14 [00:33<07:15, 33.50s/it]\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35m{'loss': 3.7076, 'learning_rate': 0, 'epoch': 0.07}\u001b[0m\n",
      "\u001b[35m7%|▋         | 1/14 [00:33<07:15, 33.50s/it]\u001b[0m\n",
      "\u001b[35mWARNING: tokenization mismatch 70 vs. 72\u001b[0m\n",
      "\u001b[34mWARNING: tokenization mismatch 67 vs. 69\u001b[0m\n",
      "\u001b[35mWARNING: tokenization mismatch 70 vs. 72\u001b[0m\n",
      "\u001b[35mWARNING: tokenization mismatch 74 vs. 76\u001b[0m\n",
      "\u001b[34mWARNING: tokenization mismatch 66 vs. 68\u001b[0m\n",
      "\u001b[34mWARNING: tokenization mismatch 75 vs. 77\u001b[0m\n",
      "\u001b[35mWARNING: tokenization mismatch 70 vs. 72\u001b[0m\n",
      "\u001b[34m14%|█▍        | 2/14 [01:02<06:11, 30.92s/it]\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34m{'loss': 3.7453, 'learning_rate': 0, 'epoch': 0.14}\u001b[0m\n",
      "\u001b[34m14%|█▍        | 2/14 [01:02<06:11, 30.92s/it]\u001b[0m\n",
      "\u001b[34mWARNING: tokenization mismatch 70 vs. 72\u001b[0m\n",
      "\u001b[35m14%|█▍        | 2/14 [01:02<06:11, 30.92s/it]\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35m{'loss': 3.7453, 'learning_rate': 0, 'epoch': 0.14}\u001b[0m\n",
      "\u001b[35m14%|█▍        | 2/14 [01:02<06:11, 30.92s/it]\u001b[0m\n",
      "\u001b[35mWARNING: tokenization mismatch 70 vs. 72\u001b[0m\n",
      "\u001b[34mWARNING: tokenization mismatch 70 vs. 72\u001b[0m\n",
      "\u001b[35mWARNING: tokenization mismatch 69 vs. 71\u001b[0m\n",
      "\u001b[34mWARNING: tokenization mismatch 74 vs. 76\u001b[0m\n",
      "\u001b[35mWARNING: tokenization mismatch 67 vs. 69\u001b[0m\n",
      "\u001b[34mWARNING: tokenization mismatch 71 vs. 73\u001b[0m\n",
      "\u001b[35mWARNING: tokenization mismatch 68 vs. 70\u001b[0m\n",
      "\u001b[34m21%|██▏       | 3/14 [01:31<05:31, 30.11s/it]\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34m{'loss': 3.7019, 'learning_rate': 0, 'epoch': 0.21}\u001b[0m\n",
      "\u001b[34m21%|██▏       | 3/14 [01:31<05:31, 30.11s/it]\u001b[0m\n",
      "\u001b[34mWARNING: tokenization mismatch 73 vs. 75\u001b[0m\n",
      "\u001b[35m21%|██▏       | 3/14 [01:31<05:31, 30.11s/it]\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35m{'loss': 3.7019, 'learning_rate': 0, 'epoch': 0.21}\u001b[0m\n",
      "\u001b[35m21%|██▏       | 3/14 [01:31<05:31, 30.11s/it]\u001b[0m\n",
      "\u001b[35mWARNING: tokenization mismatch 75 vs. 77\u001b[0m\n",
      "\u001b[35mWARNING: tokenization mismatch 74 vs. 75\u001b[0m\n",
      "\u001b[34mWARNING: tokenization mismatch 67 vs. 69\u001b[0m\n",
      "\u001b[34mWARNING: tokenization mismatch 67 vs. 69\u001b[0m\n",
      "\u001b[35mWARNING: tokenization mismatch 70 vs. 72\u001b[0m\n",
      "\u001b[34mWARNING: tokenization mismatch 71 vs. 73\u001b[0m\n",
      "\u001b[35mWARNING: tokenization mismatch 71 vs. 73\u001b[0m\n",
      "\u001b[34m29%|██▊       | 4/14 [02:00<04:57, 29.73s/it]\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34m{'loss': 3.7553, 'learning_rate': 0, 'epoch': 0.28}\u001b[0m\n",
      "\u001b[34m29%|██▊       | 4/14 [02:00<04:57, 29.73s/it]\u001b[0m\n",
      "\u001b[34mWARNING: tokenization mismatch 69 vs. 71\u001b[0m\n",
      "\u001b[35m29%|██▊       | 4/14 [02:00<04:57, 29.73s/it]\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35m{'loss': 3.7553, 'learning_rate': 0, 'epoch': 0.28}\u001b[0m\n",
      "\u001b[35m29%|██▊       | 4/14 [02:00<04:57, 29.73s/it]\u001b[0m\n",
      "\u001b[35mWARNING: tokenization mismatch 70 vs. 72\u001b[0m\n",
      "\u001b[34mWARNING: tokenization mismatch 73 vs. 75\u001b[0m\n",
      "\u001b[35mWARNING: tokenization mismatch 69 vs. 71\u001b[0m\n",
      "\u001b[34mWARNING: tokenization mismatch 73 vs. 75\u001b[0m\n",
      "\u001b[35mWARNING: tokenization mismatch 70 vs. 72\u001b[0m\n",
      "\u001b[34mWARNING: tokenization mismatch 68 vs. 70\u001b[0m\n",
      "\u001b[35mWARNING: tokenization mismatch 75 vs. 77\u001b[0m\n",
      "\u001b[34m36%|███▌      | 5/14 [02:30<04:25, 29.50s/it]\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34m{'loss': 3.7032, 'learning_rate': 0, 'epoch': 0.35}\u001b[0m\n",
      "\u001b[34m36%|███▌      | 5/14 [02:30<04:25, 29.50s/it]\u001b[0m\n",
      "\u001b[34mWARNING: tokenization mismatch 67 vs. 69\u001b[0m\n",
      "\u001b[35m36%|███▌      | 5/14 [02:30<04:25, 29.50s/it]\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35m{'loss': 3.7032, 'learning_rate': 0, 'epoch': 0.35}\u001b[0m\n",
      "\u001b[35m36%|███▌      | 5/14 [02:30<04:25, 29.50s/it]\u001b[0m\n",
      "\u001b[35mWARNING: tokenization mismatch 70 vs. 72\u001b[0m\n",
      "\u001b[34mWARNING: tokenization mismatch 71 vs. 73\u001b[0m\n",
      "\u001b[35mWARNING: tokenization mismatch 70 vs. 72\u001b[0m\n",
      "\u001b[34mWARNING: tokenization mismatch 70 vs. 72\u001b[0m\n",
      "\u001b[35mWARNING: tokenization mismatch 71 vs. 73\u001b[0m\n",
      "\u001b[34mWARNING: tokenization mismatch 70 vs. 72\u001b[0m\n",
      "\u001b[35mWARNING: tokenization mismatch 75 vs. 77\u001b[0m\n",
      "\u001b[35m43%|████▎     | 6/14 [02:59<03:54, 29.35s/it]\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35m{'loss': 3.7077, 'learning_rate': 0, 'epoch': 0.42}\u001b[0m\n",
      "\u001b[35m43%|████▎     | 6/14 [02:59<03:54, 29.35s/it]\u001b[0m\n",
      "\u001b[35mWARNING: tokenization mismatch 72 vs. 74\u001b[0m\n",
      "\u001b[34m43%|████▎     | 6/14 [02:59<03:54, 29.35s/it]\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34m{'loss': 3.7077, 'learning_rate': 0, 'epoch': 0.42}\u001b[0m\n",
      "\u001b[34m43%|████▎     | 6/14 [02:59<03:54, 29.35s/it]\u001b[0m\n",
      "\u001b[34mWARNING: tokenization mismatch 75 vs. 77\u001b[0m\n",
      "\u001b[34mWARNING: tokenization mismatch 70 vs. 72\u001b[0m\n",
      "\u001b[35mWARNING: tokenization mismatch 67 vs. 69\u001b[0m\n",
      "\u001b[34mWARNING: tokenization mismatch 74 vs. 76\u001b[0m\n",
      "\u001b[35mWARNING: tokenization mismatch 69 vs. 71\u001b[0m\n",
      "\u001b[34mWARNING: tokenization mismatch 70 vs. 72\u001b[0m\n",
      "\u001b[35mWARNING: tokenization mismatch 71 vs. 73\u001b[0m\n",
      "\u001b[34m50%|█████     | 7/14 [03:28<03:24, 29.27s/it]\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34m{'loss': 3.6641, 'learning_rate': 0, 'epoch': 0.49}\u001b[0m\n",
      "\u001b[34m50%|█████     | 7/14 [03:28<03:24, 29.27s/it]\u001b[0m\n",
      "\u001b[34mWARNING: tokenization mismatch 61 vs. 62\u001b[0m\n",
      "\u001b[35m50%|█████     | 7/14 [03:28<03:24, 29.27s/it]\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35m{'loss': 3.6641, 'learning_rate': 0, 'epoch': 0.49}\u001b[0m\n",
      "\u001b[35m50%|█████     | 7/14 [03:28<03:24, 29.27s/it]\u001b[0m\n",
      "\u001b[35mWARNING: tokenization mismatch 72 vs. 74\u001b[0m\n",
      "\u001b[34mWARNING: tokenization mismatch 75 vs. 77\u001b[0m\n",
      "\u001b[35mWARNING: tokenization mismatch 72 vs. 74\u001b[0m\n",
      "\u001b[34mWARNING: tokenization mismatch 73 vs. 75\u001b[0m\n",
      "\u001b[35mWARNING: tokenization mismatch 74 vs. 76\u001b[0m\n",
      "\u001b[34mWARNING: tokenization mismatch 71 vs. 73\u001b[0m\n",
      "\u001b[35mWARNING: tokenization mismatch 71 vs. 73\u001b[0m\n",
      "\u001b[34m57%|█████▋    | 8/14 [03:57<02:55, 29.22s/it]\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34m{'loss': 3.6504, 'learning_rate': 0, 'epoch': 0.56}\u001b[0m\n",
      "\u001b[34m57%|█████▋    | 8/14 [03:57<02:55, 29.22s/it]\u001b[0m\n",
      "\u001b[34mWARNING: tokenization mismatch 67 vs. 69\u001b[0m\n",
      "\u001b[35m57%|█████▋    | 8/14 [03:57<02:55, 29.22s/it]\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35m{'loss': 3.6504, 'learning_rate': 0, 'epoch': 0.56}\u001b[0m\n",
      "\u001b[35m57%|█████▋    | 8/14 [03:57<02:55, 29.22s/it]\u001b[0m\n",
      "\u001b[35mWARNING: tokenization mismatch 71 vs. 73\u001b[0m\n",
      "\u001b[34mWARNING: tokenization mismatch 71 vs. 73\u001b[0m\n",
      "\u001b[35mWARNING: tokenization mismatch 70 vs. 72\u001b[0m\n",
      "\u001b[34mWARNING: tokenization mismatch 69 vs. 71\u001b[0m\n",
      "\u001b[35mWARNING: tokenization mismatch 69 vs. 71\u001b[0m\n",
      "\u001b[34mWARNING: tokenization mismatch 70 vs. 72\u001b[0m\n",
      "\u001b[35mWARNING: tokenization mismatch 70 vs. 72\u001b[0m\n",
      "\u001b[34m64%|██████▍   | 9/14 [04:26<02:25, 29.18s/it]\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34m{'loss': 3.747, 'learning_rate': 0, 'epoch': 0.63}\u001b[0m\n",
      "\u001b[34m64%|██████▍   | 9/14 [04:26<02:25, 29.18s/it]\u001b[0m\n",
      "\u001b[34mWARNING: tokenization mismatch 70 vs. 72\u001b[0m\n",
      "\u001b[35m64%|██████▍   | 9/14 [04:26<02:25, 29.18s/it]\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35m{'loss': 3.747, 'learning_rate': 0, 'epoch': 0.63}\u001b[0m\n",
      "\u001b[35m64%|██████▍   | 9/14 [04:26<02:25, 29.18s/it]\u001b[0m\n",
      "\u001b[35mWARNING: tokenization mismatch 65 vs. 67\u001b[0m\n",
      "\u001b[35mWARNING: tokenization mismatch 70 vs. 72\u001b[0m\n",
      "\u001b[34mWARNING: tokenization mismatch 62 vs. 64\u001b[0m\n",
      "\u001b[34mWARNING: tokenization mismatch 60 vs. 62\u001b[0m\n",
      "\u001b[35mWARNING: tokenization mismatch 70 vs. 72\u001b[0m\n",
      "\u001b[34mWARNING: tokenization mismatch 74 vs. 76\u001b[0m\n",
      "\u001b[35mWARNING: tokenization mismatch 71 vs. 73\u001b[0m\n",
      "\u001b[34m71%|███████▏  | 10/14 [04:55<01:56, 29.17s/it]\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[34m{'loss': 3.6643, 'learning_rate': 0, 'epoch': 0.7}\u001b[0m\n",
      "\u001b[34m71%|███████▏  | 10/14 [04:55<01:56, 29.17s/it]\u001b[0m\n",
      "\u001b[34mWARNING: tokenization mismatch 67 vs. 69\u001b[0m\n",
      "\u001b[35m71%|███████▏  | 10/14 [04:55<01:56, 29.17s/it]\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35mtried to get lr value before scheduler/optimizer started stepping, returning lr=0\u001b[0m\n",
      "\u001b[35m{'loss': 3.6643, 'learning_rate': 0, 'epoch': 0.7}\u001b[0m\n",
      "\u001b[35m71%|███████▏  | 10/14 [04:55<01:56, 29.17s/it]\u001b[0m\n",
      "\u001b[35mWARNING: tokenization mismatch 69 vs. 71\u001b[0m\n",
      "\u001b[34mWARNING: tokenization mismatch 75 vs. 77\u001b[0m\n",
      "\u001b[35mWARNING: tokenization mismatch 73 vs. 75\u001b[0m\n",
      "\u001b[34mWARNING: tokenization mismatch 67 vs. 69\u001b[0m\n",
      "\u001b[35mWARNING: tokenization mismatch 70 vs. 72\u001b[0m\n",
      "\u001b[34mWARNING: tokenization mismatch 68 vs. 70\u001b[0m\n",
      "\u001b[35mWARNING: tokenization mismatch 75 vs. 76\u001b[0m\n",
      "\u001b[34m79%|███████▊  | 11/14 [05:24<01:27, 29.16s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 3.705, 'learning_rate': 0.0, 'epoch': 0.77}\u001b[0m\n",
      "\u001b[34m79%|███████▊  | 11/14 [05:24<01:27, 29.16s/it]\u001b[0m\n",
      "\u001b[34mWARNING: tokenization mismatch 70 vs. 72\u001b[0m\n",
      "\u001b[35m79%|███████▊  | 11/14 [05:24<01:27, 29.16s/it]\u001b[0m\n",
      "\u001b[35m{'loss': 3.705, 'learning_rate': 0.0, 'epoch': 0.77}\u001b[0m\n",
      "\u001b[35m79%|███████▊  | 11/14 [05:24<01:27, 29.16s/it]\u001b[0m\n",
      "\u001b[35mWARNING: tokenization mismatch 70 vs. 72\u001b[0m\n",
      "\u001b[35mWARNING: tokenization mismatch 71 vs. 73\u001b[0m\n",
      "\u001b[34mWARNING: tokenization mismatch 74 vs. 76\u001b[0m\n",
      "\u001b[34mWARNING: tokenization mismatch 70 vs. 72\u001b[0m\n",
      "\u001b[35mWARNING: tokenization mismatch 66 vs. 68\u001b[0m\n",
      "\u001b[34mWARNING: tokenization mismatch 76 vs. 78\u001b[0m\n",
      "\u001b[35mWARNING: tokenization mismatch 70 vs. 72\u001b[0m\n",
      "\u001b[34m86%|████████▌ | 12/14 [05:53<00:58, 29.13s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 0.8411, 'learning_rate': 2e-05, 'epoch': 0.84}\u001b[0m\n",
      "\u001b[34m86%|████████▌ | 12/14 [05:53<00:58, 29.13s/it]\u001b[0m\n",
      "\u001b[34mWARNING: tokenization mismatch 68 vs. 70\u001b[0m\n",
      "\u001b[35m86%|████████▌ | 12/14 [05:53<00:58, 29.13s/it]\u001b[0m\n",
      "\u001b[35m{'loss': 0.8411, 'learning_rate': 2e-05, 'epoch': 0.84}\u001b[0m\n",
      "\u001b[35m86%|████████▌ | 12/14 [05:53<00:58, 29.13s/it]\u001b[0m\n",
      "\u001b[35mWARNING: tokenization mismatch 70 vs. 72\u001b[0m\n",
      "\u001b[34mWARNING: tokenization mismatch 67 vs. 69\u001b[0m\n",
      "\u001b[35mWARNING: tokenization mismatch 73 vs. 75\u001b[0m\n",
      "\u001b[34mWARNING: tokenization mismatch 75 vs. 77\u001b[0m\n",
      "\u001b[35mWARNING: tokenization mismatch 71 vs. 73\u001b[0m\n",
      "\u001b[34mWARNING: tokenization mismatch 67 vs. 69\u001b[0m\n",
      "\u001b[35mWARNING: tokenization mismatch 70 vs. 72\u001b[0m\n",
      "\u001b[34m93%|█████████▎| 13/14 [06:22<00:29, 29.09s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 0.8146, 'learning_rate': 2e-05, 'epoch': 0.91}\u001b[0m\n",
      "\u001b[34m93%|█████████▎| 13/14 [06:22<00:29, 29.09s/it]\u001b[0m\n",
      "\u001b[34mWARNING: tokenization mismatch 75 vs. 77\u001b[0m\n",
      "\u001b[35m93%|█████████▎| 13/14 [06:22<00:29, 29.09s/it]\u001b[0m\n",
      "\u001b[35m{'loss': 0.8146, 'learning_rate': 2e-05, 'epoch': 0.91}\u001b[0m\n",
      "\u001b[35m93%|█████████▎| 13/14 [06:22<00:29, 29.09s/it]\u001b[0m\n",
      "\u001b[35mWARNING: tokenization mismatch 66 vs. 68\u001b[0m\n",
      "\u001b[34mWARNING: tokenization mismatch 69 vs. 71\u001b[0m\n",
      "\u001b[35mWARNING: tokenization mismatch 73 vs. 75\u001b[0m\n",
      "\u001b[34mWARNING: tokenization mismatch 72 vs. 74\u001b[0m\n",
      "\u001b[35mWARNING: tokenization mismatch 71 vs. 73\u001b[0m\n",
      "\u001b[34mWARNING: tokenization mismatch 74 vs. 76\u001b[0m\n",
      "\u001b[35mWARNING: tokenization mismatch 68 vs. 70\u001b[0m\n",
      "\u001b[34m100%|██████████| 14/14 [06:51<00:00, 29.08s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 0.4294, 'learning_rate': 2e-05, 'epoch': 0.98}\u001b[0m\n",
      "\u001b[34m100%|██████████| 14/14 [06:51<00:00, 29.08s/it]\u001b[0m\n",
      "\u001b[34m{'train_runtime': 412.175, 'train_samples_per_second': 2.208, 'train_steps_per_second': 0.034, 'train_loss': 3.0597771235874722, 'epoch': 0.98}\u001b[0m\n",
      "\u001b[34m100%|██████████| 14/14 [06:51<00:00, 29.08s/it]\u001b[0m\n",
      "\u001b[34m100%|██████████| 14/14 [06:51<00:00, 29.41s/it]\u001b[0m\n",
      "\u001b[35m100%|██████████| 14/14 [06:51<00:00, 29.08s/it]\u001b[0m\n",
      "\u001b[35m{'loss': 0.4294, 'learning_rate': 2e-05, 'epoch': 0.98}\u001b[0m\n",
      "\u001b[35m100%|██████████| 14/14 [06:51<00:00, 29.08s/it]\u001b[0m\n",
      "\u001b[35m{'train_runtime': 412.33, 'train_samples_per_second': 2.207, 'train_steps_per_second': 0.034, 'train_loss': 3.0597771235874722, 'epoch': 0.98}\u001b[0m\n",
      "\u001b[35m100%|██████████| 14/14 [06:51<00:00, 29.08s/it]\u001b[0m\n",
      "\u001b[35m100%|██████████| 14/14 [06:51<00:00, 29.41s/it]\u001b[0m\n",
      "\u001b[34mbegin to upload trained model\u001b[0m\n",
      "\u001b[34mcurrent host==algo-1\u001b[0m\n",
      "\u001b[34mmaster host==\u001b[0m\n",
      "\u001b[34mcp /tmp/llama_out/generation_config.json s3://sagemaker-us-west-2-687912291502/llama/output/2023-07-21-10-03-02/llama_out/generation_config.json\u001b[0m\n",
      "\u001b[34mcp /tmp/llama_out/training_args.bin s3://sagemaker-us-west-2-687912291502/llama/output/2023-07-21-10-03-02/llama_out/training_args.bin\u001b[0m\n",
      "\u001b[34mcp /tmp/llama_out/added_tokens.json s3://sagemaker-us-west-2-687912291502/llama/output/2023-07-21-10-03-02/llama_out/added_tokens.json\u001b[0m\n",
      "\u001b[34mcp /tmp/llama_out/config.json s3://sagemaker-us-west-2-687912291502/llama/output/2023-07-21-10-03-02/llama_out/config.json\u001b[0m\n",
      "\u001b[34mcp /tmp/llama_out/special_tokens_map.json s3://sagemaker-us-west-2-687912291502/llama/output/2023-07-21-10-03-02/llama_out/special_tokens_map.json\u001b[0m\n",
      "\u001b[34mcp /tmp/llama_out/tokenizer_config.json s3://sagemaker-us-west-2-687912291502/llama/output/2023-07-21-10-03-02/llama_out/tokenizer_config.json\u001b[0m\n",
      "\u001b[34mcp /tmp/llama_out/tokenizer.model s3://sagemaker-us-west-2-687912291502/llama/output/2023-07-21-10-03-02/llama_out/tokenizer.model\u001b[0m\n",
      "\u001b[35mbegin to upload trained model\u001b[0m\n",
      "\u001b[35mcurrent host==algo-2\u001b[0m\n",
      "\u001b[35mmaster host==\u001b[0m\n",
      "\u001b[35m2023-07-21 10:03:02,094 sagemaker-training-toolkit INFO     Waiting for the process to finish and give a return code.\u001b[0m\n",
      "\u001b[35m2023-07-21 10:03:02,094 sagemaker-training-toolkit INFO     Done waiting for a return code. Received 0 from exiting process.\u001b[0m\n",
      "\u001b[35m2023-07-21 10:03:02,095 sagemaker-training-toolkit INFO     Reporting training SUCCESS\u001b[0m\n",
      "\u001b[34mcp /tmp/llama_out/pytorch_model.bin s3://sagemaker-us-west-2-687912291502/llama/output/2023-07-21-10-03-02/llama_out/pytorch_model.bin\u001b[0m\n",
      "\u001b[34m2023-07-21 10:03:44,650 sagemaker-training-toolkit INFO     Waiting for the process to finish and give a return code.\u001b[0m\n",
      "\u001b[34m2023-07-21 10:03:44,650 sagemaker-training-toolkit INFO     Done waiting for a return code. Received 0 from exiting process.\u001b[0m\n",
      "\u001b[34m2023-07-21 10:03:44,650 sagemaker-training-toolkit INFO     Reporting training SUCCESS\u001b[0m\n",
      "\n",
      "2023-07-21 10:04:55 Uploading - Uploading generated training model\n",
      "2023-07-21 10:04:55 Completed - Training job completed\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "from sagemaker.estimator import Estimator\n",
    "\n",
    "environment = {\n",
    "              'MODEL_S3_BUCKET': sagemaker_default_bucket # The bucket to store pretrained model and fine-tune model\n",
    "}\n",
    "\n",
    "base_job_name = 'vicuna-demo'         \n",
    "\n",
    "instance_type = 'ml.p4d.24xlarge'\n",
    "\n",
    "estimator = Estimator(role=role,\n",
    "                      entry_point='ds-train-distribute.sh',\n",
    "                      source_dir='./FastChat/',\n",
    "                      base_job_name=base_job_name,\n",
    "                      instance_count=2,\n",
    "                      instance_type=instance_type,\n",
    "                      image_uri=image_uri,\n",
    "                      environment=environment,\n",
    "                      disable_profiler=True,\n",
    "                      debugger_hook_config=False,\n",
    "                      max_run=24*60*60*2)\n",
    "\n",
    "estimator.fit()\n",
    "#estimator.fit(inputs)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7a3e673c",
   "metadata": {},
   "source": [
    "You could find the model path in S3 from above logs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "id": "e2b535a9-a683-473b-a769-8d3b3354d89a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2023-07-19 15:07:45         21 added_tokens.json\n",
      "2023-07-19 15:07:45        545 config.json\n",
      "2023-07-19 15:07:45        132 generation_config.json\n",
      "2023-07-19 15:07:45 13476958625 pytorch_model.bin\n",
      "2023-07-19 15:07:45        423 special_tokens_map.json\n",
      "2023-07-19 15:07:45     499723 tokenizer.model\n",
      "2023-07-19 15:07:45        736 tokenizer_config.json\n",
      "2023-07-19 15:07:45       4795 training_args.bin\n"
     ]
    }
   ],
   "source": [
    "!aws s3 ls s3://sagemaker-us-west-2-687912291502/llama/output/2023-07-19-15-07-44/llama_out/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "07ff2be0-bd27-478f-89a5-e95c6f9b8b06",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_pytorch_p39",
   "language": "python",
   "name": "conda_pytorch_p39"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}