baichuan_fine_tune_lora.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "252de0de",
   "metadata": {
    "tags": []
   },
   "source": [
    "### SageMaker fine tune baichuan\n",
    "\n",
    "#### 准备\n",
    "1. 升级boto3, sagemaker python sdk  \n",
    "2. 准备requirements.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b8f2c403",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!pip install --upgrade boto3\n",
    "!pip install --upgrade sagemaker"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a4a30f3a",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ec2-user/anaconda3/envs/amazonei_pytorch_latest_p37/lib/python3.7/site-packages/boto3/compat.py:82: PythonDeprecationWarning: Boto3 will no longer support Python 3.7 starting December 13, 2023. To continue receiving service updates, bug fixes, and security updates please upgrade to Python 3.8 or later. More information can be found here: https://aws.amazon.com/blogs/developer/python-support-policy-updates-for-aws-sdks-and-tools/\n",
      "  warnings.warn(warning, PythonDeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "arn:aws:iam::687912291502:role/service-role/AmazonSageMaker-ExecutionRole-20211013T113123\n",
      "sagemaker-us-west-2-687912291502\n"
     ]
    }
   ],
   "source": [
    "import boto3\n",
    "import sagemaker\n",
    "\n",
    "account = boto3.client('sts').get_caller_identity().get('Account')\n",
    "region = boto3.session.Session().region_name\n",
    "\n",
    "sagemaker_session = sagemaker.Session()\n",
    "bucket = sagemaker_session.default_bucket()\n",
    "role = sagemaker.get_execution_role()\n",
    "\n",
    "print(role)\n",
    "print(bucket)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f6f59e3f",
   "metadata": {
    "tags": []
   },
   "source": [
    "### baichuan fine tune \n",
    " deepspeed+QLoRA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "9db2f288-0610-4b18-9409-c8ef5ee91ca0",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cloning into 'LLaMA-Efficient-Tuning'...\n",
      "remote: Enumerating objects: 1100, done.\u001b[K\n",
      "remote: Counting objects: 100% (570/570), done.\u001b[K\n",
      "remote: Compressing objects: 100% (171/171), done.\u001b[K\n",
      "remote: Total 1100 (delta 471), reused 455 (delta 398), pack-reused 530\u001b[K\n",
      "Receiving objects: 100% (1100/1100), 72.44 MiB | 13.17 MiB/s, done.\n",
      "Resolving deltas: 100% (730/730), done.\n",
      "Updating files: 100% (98/98), done.\n"
     ]
    }
   ],
   "source": [
    "# 我们在原始的LLaMA-Efficient-Tuning.git 基础上做了一些魔改 1.每次save同步到S3，2.调整训练参数\n",
    "!rm -rf ./LLaMA-Efficient-Tuning\n",
    "!git clone https://github.com/hiyouga/LLaMA-Efficient-Tuning.git\n",
    "!cp ./s5cmd ./LLaMA-Efficient-Tuning/"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b42e2794-2ed4-49f9-b1b0-054f1a0af54f",
   "metadata": {
    "tags": []
   },
   "source": [
    "## prepare docker images，如果准备好就跳过"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8aceb270-324f-445e-ae84-0da1d1b98adf",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting Dockerfile\n"
     ]
    }
   ],
   "source": [
    "%%writefile Dockerfile\n",
    "## You should change below region code to the region you used, here sample is use us-west-2\n",
    "From 763104351884.dkr.ecr.us-west-2.amazonaws.com/huggingface-pytorch-training:1.13.1-transformers4.26.0-gpu-py39-cu117-ubuntu20.04 \n",
    "#From pytorch/pytorch:1.5-cuda10.1-cudnn7-runtime\n",
    "\n",
    "ENV LANG=C.UTF-8\n",
    "ENV PYTHONUNBUFFERED=TRUE\n",
    "ENV PYTHONDONTWRITEBYTECODE=TRUE\n",
    "RUN pip3 uninstall -y deepspeed && pip3 install deepspeed\n",
    "#RUN pip install -U git+https://github.com/ssbuild/deep_training.git\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "3b8ee553",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING! Your password will be stored unencrypted in /home/ec2-user/.docker/config.json.\n",
      "Configure a credential helper to remove this warning. See\n",
      "https://docs.docker.com/engine/reference/commandline/login/#credentials-store\n",
      "\n",
      "Login Succeeded\n"
     ]
    }
   ],
   "source": [
    "## You should change below region code to the region you used, here sample is use us-west-2\n",
    "!aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin 763104351884.dkr.ecr.us-west-2.amazonaws.com"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c00b23c6-6e88-48f2-96dd-99140c147be5",
   "metadata": {},
   "source": [
    "**Build image and push to ECR.**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "fa970133-14f1-40d4-963f-895154a43f94",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "## define repo name, should contain *sagemaker* in the name\n",
    "repo_name = \"sagemaker-baichuan_finetuning\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "51c2d43e-ff0f-4718-b772-555c95d6aaaf",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Login Succeeded\n",
      "Sending build context to Docker daemon  81.59GB\n",
      "Step 1/5 : From 763104351884.dkr.ecr.us-west-2.amazonaws.com/huggingface-pytorch-training:1.13.1-transformers4.26.0-gpu-py39-cu117-ubuntu20.04\n",
      " ---> c5a6ef695006\n",
      "Step 2/5 : ENV LANG=C.UTF-8\n",
      " ---> Using cache\n",
      " ---> af49cfa7feae\n",
      "Step 3/5 : ENV PYTHONUNBUFFERED=TRUE\n",
      " ---> Using cache\n",
      " ---> 287106637dc6\n",
      "Step 4/5 : ENV PYTHONDONTWRITEBYTECODE=TRUE\n",
      " ---> Using cache\n",
      " ---> 773b4cf30c90\n",
      "Step 5/5 : RUN pip3 uninstall -y deepspeed && pip3 install deepspeed\n",
      " ---> Using cache\n",
      " ---> ce72201e73cd\n",
      "Successfully built ce72201e73cd\n",
      "Successfully tagged sagemaker-baichuan_finetuning:latest\n",
      "The push refers to repository [687912291502.dkr.ecr.us-west-2.amazonaws.com/sagemaker-baichuan_finetuning]\n",
      "02a87473f68b: Preparing\n",
      "f8dae5c3df1e: Preparing\n",
      "e3221f18601a: Preparing\n",
      "b6f286626882: Preparing\n",
      "76fe97d80cdb: Preparing\n",
      "f5f76489fff8: Preparing\n",
      "621c3f07daa7: Preparing\n",
      "9b484bb42e11: Preparing\n",
      "54c7c0b58471: Preparing\n",
      "c34adc3ab668: Preparing\n",
      "bbf651e48b84: Preparing\n",
      "f61045791108: Preparing\n",
      "4e2ac0cda74a: Preparing\n",
      "658a33d555eb: Preparing\n",
      "bd16d9a61a98: Preparing\n",
      "f0c0cd2accfa: Preparing\n",
      "1275469c066c: Preparing\n",
      "b802dd3babf4: Preparing\n",
      "a3834ec63558: Preparing\n",
      "63edcef6dedf: Preparing\n",
      "0154e84cc2dd: Preparing\n",
      "7085d1c151f6: Preparing\n",
      "a77a2104cfb6: Preparing\n",
      "6808e7f9da2f: Preparing\n",
      "9b484bb42e11: Waiting\n",
      "54c7c0b58471: Waiting\n",
      "c34adc3ab668: Waiting\n",
      "3bc059a9dec6: Preparing\n",
      "de783f3fec23: Preparing\n",
      "18ca52d74b2f: Preparing\n",
      "bbf651e48b84: Waiting\n",
      "f5f76489fff8: Waiting\n",
      "f61045791108: Waiting\n",
      "4e2ac0cda74a: Waiting\n",
      "621c3f07daa7: Waiting\n",
      "658a33d555eb: Waiting\n",
      "a3834ec63558: Waiting\n",
      "bd16d9a61a98: Waiting\n",
      "63edcef6dedf: Waiting\n",
      "de783f3fec23: Waiting\n",
      "6808e7f9da2f: Waiting\n",
      "1275469c066c: Waiting\n",
      "f0c0cd2accfa: Waiting\n",
      "0154e84cc2dd: Waiting\n",
      "b802dd3babf4: Waiting\n",
      "a77a2104cfb6: Waiting\n",
      "73df6ccd636c: Preparing\n",
      "18ca52d74b2f: Waiting\n",
      "6738b73ff7a8: Preparing\n",
      "2a8292d9bfcc: Preparing\n",
      "73df6ccd636c: Waiting\n",
      "5b75a5ef32a7: Preparing\n",
      "25a5f55a11f0: Preparing\n",
      "707f484816ae: Preparing\n",
      "6738b73ff7a8: Waiting\n",
      "0430aa1e47d4: Preparing\n",
      "65448e793131: Preparing\n",
      "2a8292d9bfcc: Waiting\n",
      "15af6e2d42ba: Preparing\n",
      "5b75a5ef32a7: Waiting\n",
      "b46caef92993: Preparing\n",
      "53ce33a12646: Preparing\n",
      "aad68760f4ce: Preparing\n",
      "323d67ab1719: Preparing\n",
      "e72743a0fdfe: Preparing\n",
      "3996353f5820: Preparing\n",
      "ea87e0b9c30f: Preparing\n",
      "af18356cdf10: Preparing\n",
      "f6e30dd4497e: Preparing\n",
      "99832d04a153: Preparing\n",
      "a5981ed7a378: Preparing\n",
      "250519a2f830: Preparing\n",
      "6cadbde53f94: Preparing\n",
      "0002c93bdb37: Preparing\n",
      "25a5f55a11f0: Waiting\n",
      "707f484816ae: Waiting\n",
      "0430aa1e47d4: Waiting\n",
      "323d67ab1719: Waiting\n",
      "65448e793131: Waiting\n",
      "e72743a0fdfe: Waiting\n",
      "15af6e2d42ba: Waiting\n",
      "3996353f5820: Waiting\n",
      "53ce33a12646: Waiting\n",
      "ea87e0b9c30f: Waiting\n",
      "af18356cdf10: Waiting\n",
      "aad68760f4ce: Waiting\n",
      "a5981ed7a378: Waiting\n",
      "0002c93bdb37: Waiting\n",
      "250519a2f830: Waiting\n",
      "99832d04a153: Waiting\n",
      "6cadbde53f94: Waiting\n",
      "f6e30dd4497e: Waiting\n",
      "76fe97d80cdb: Layer already exists\n",
      "e3221f18601a: Layer already exists\n",
      "f8dae5c3df1e: Layer already exists\n",
      "b6f286626882: Layer already exists\n",
      "f5f76489fff8: Layer already exists\n",
      "621c3f07daa7: Layer already exists\n",
      "9b484bb42e11: Layer already exists\n",
      "54c7c0b58471: Layer already exists\n",
      "c34adc3ab668: Layer already exists\n",
      "bbf651e48b84: Layer already exists\n",
      "f61045791108: Layer already exists\n",
      "4e2ac0cda74a: Layer already exists\n",
      "658a33d555eb: Layer already exists\n",
      "f0c0cd2accfa: Layer already exists\n",
      "bd16d9a61a98: Layer already exists\n",
      "1275469c066c: Layer already exists\n",
      "b802dd3babf4: Layer already exists\n",
      "a3834ec63558: Layer already exists\n",
      "63edcef6dedf: Layer already exists\n",
      "0154e84cc2dd: Layer already exists\n",
      "7085d1c151f6: Layer already exists\n",
      "a77a2104cfb6: Layer already exists\n",
      "6808e7f9da2f: Layer already exists\n",
      "de783f3fec23: Layer already exists\n",
      "18ca52d74b2f: Layer already exists\n",
      "3bc059a9dec6: Layer already exists\n",
      "73df6ccd636c: Layer already exists\n",
      "6738b73ff7a8: Layer already exists\n",
      "2a8292d9bfcc: Layer already exists\n",
      "5b75a5ef32a7: Layer already exists\n",
      "25a5f55a11f0: Layer already exists\n",
      "707f484816ae: Layer already exists\n",
      "65448e793131: Layer already exists\n",
      "0430aa1e47d4: Layer already exists\n",
      "15af6e2d42ba: Layer already exists\n",
      "b46caef92993: Layer already exists\n",
      "53ce33a12646: Layer already exists\n",
      "aad68760f4ce: Layer already exists\n",
      "323d67ab1719: Layer already exists\n",
      "e72743a0fdfe: Layer already exists\n",
      "3996353f5820: Layer already exists\n",
      "ea87e0b9c30f: Layer already exists\n",
      "af18356cdf10: Layer already exists\n",
      "f6e30dd4497e: Layer already exists\n",
      "a5981ed7a378: Layer already exists\n",
      "250519a2f830: Layer already exists\n",
      "6cadbde53f94: Layer already exists\n",
      "99832d04a153: Layer already exists\n",
      "0002c93bdb37: Layer already exists\n",
      "02a87473f68b: Pushed\n",
      "latest: digest: sha256:f826bf57139eebafb3c87041f2c7931bdec61ca24068ad5d13af16d7b4fec5f0 size: 10832\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING! Your password will be stored unencrypted in /home/ec2-user/.docker/config.json.\n",
      "Configure a credential helper to remove this warning. See\n",
      "https://docs.docker.com/engine/reference/commandline/login/#credentials-store\n",
      "\n"
     ]
    }
   ],
   "source": [
    "%%script env repo_name=$repo_name bash\n",
    "\n",
    "#!/usr/bin/env bash\n",
    "\n",
    "# This script shows how to build the Docker image and push it to ECR to be ready for use\n",
    "# by SageMaker.\n",
    "\n",
    "# The argument to this script is the image name. This will be used as the image on the local\n",
    "# machine and combined with the account and region to form the repository name for ECR.\n",
    "# The name of our algorithm\n",
    "algorithm_name=${repo_name}\n",
    "\n",
    "account=$(aws sts get-caller-identity --query Account --output text)\n",
    "\n",
    "# Get the region defined in the current configuration (default to us-west-2 if none defined)\n",
    "region=$(aws configure get region)\n",
    "region=${region:-us-west-2}\n",
    "\n",
    "fullname=\"${account}.dkr.ecr.${region}.amazonaws.com/${algorithm_name}:latest\"\n",
    "\n",
    "# If the repository doesn't exist in ECR, create it.\n",
    "aws ecr describe-repositories --repository-names \"${algorithm_name}\" > /dev/null 2>&1\n",
    "\n",
    "if [ $? -ne 0 ]\n",
    "then\n",
    "    aws ecr create-repository --repository-name \"${algorithm_name}\" > /dev/null\n",
    "fi\n",
    "\n",
    "# Get the login command from ECR and execute it directly\n",
    "aws ecr get-login-password --region ${region}|docker login --username AWS --password-stdin ${fullname}\n",
    "\n",
    "# Build the docker image locally with the image name and then push it to ECR\n",
    "# with the full name.\n",
    "\n",
    "docker build -t ${algorithm_name} .\n",
    "docker tag ${algorithm_name} ${fullname}\n",
    "\n",
    "docker push ${fullname}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "97efab5a-0fd4-4bb6-bf08-809c1a699050",
   "metadata": {
    "tags": []
   },
   "source": [
    "## train GPT4 baichuan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "82deec4d-c736-4d32-a907-2b7a58916cd2",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!pip install huggingface_hub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "2204cdfb-2dcf-4b1e-8f43-42ff90e3c46d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from huggingface_hub import snapshot_download\n",
    "from pathlib import Path\n",
    "\n",
    "local_model_path = Path(\"./LLM_baichuan_model_13b\")\n",
    "local_model_path.mkdir(exist_ok=True)\n",
    "\n",
    "#commit_hash = \"ba9db8ed916eb8c4d4349d40ef7a0b6b68a0b930\"\n",
    "model_name = 'baichuan-inc/Baichuan-13B-Chat'\n",
    "model_cache_path = local_model_path\n",
    "#snapshot_download(repo_id=model_name, revision=commit_hash,cache_dir=local_model_path)\n",
    "#snapshot_download(repo_id=model_name,cache_dir=model_cache_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "64f6346e-c1de-4956-8d9c-665a75db9ff0",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "s3_code_prefix: llm/models/LLM_baichuan_deploy_code\n",
      "model_snapshot_path: LLM_baichuan_model_13b/models--baichuan-inc--Baichuan-13B-Chat/snapshots/ff1fbc5e10eb514c3ee54aeff36a4e703b8d9e9a\n"
     ]
    }
   ],
   "source": [
    "s3_model_prefix = \"llm/models/LLM_baichuan_model_13b\"  # folder where model checkpoint will go\n",
    "model_snapshot_path = list(local_model_path.glob(\"**/snapshots/*\"))[0]\n",
    "s3_code_prefix = \"llm/models/LLM_baichuan_deploy_code\"\n",
    "print(f\"s3_code_prefix: {s3_code_prefix}\")\n",
    "print(f\"model_snapshot_path: {model_snapshot_path}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b7e41aa3-be6c-47f2-abd9-28b9f39fb0cc",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!aws s3 rm s3://{bucket}/{s3_model_prefix} --recursive\n",
    "!aws s3 cp --recursive {model_snapshot_path} s3://{bucket}/{s3_model_prefix}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "46f0ea3e-bce4-499b-b665-c3c013f9eda5",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "####下载的baichuan model s3路径 ###########\n",
    "# 这里修改成自己的S3地址\n",
    "model_s3_path=\"s3://sagemaker-us-west-2-687912291502/llm/models/LLM_baichuan_model_13b/\"\n",
    "\n",
    "# base_model_path = 's3://sagemaker-us-west-2-960661357527/llm/models/LLM_baichuan_model/'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a3e6c0c7-f2dd-4fe2-87f0-474be4e4193a",
   "metadata": {},
   "source": [
    "## 准备好镜像"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "913a9650",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "## define repo name, should contain *sagemaker* in the name\n",
    "repo_name = \"sagemaker-baichuan_finetuning\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "bb3a0c64-1e8d-402f-8d58-1b8327b970b2",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'687912291502.dkr.ecr.us-west-2.amazonaws.com/sagemaker-baichuan_finetuning:latest'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## The image uri which is build and pushed above\n",
    "image_uri = \"{}.dkr.ecr.{}.amazonaws.com/{}:latest\".format(account, region, repo_name)\n",
    "image_uri"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "d43edef7-e129-4755-b2a0-185e416755f8",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting ./LLaMA-Efficient-Tuning/requirements.txt\n"
     ]
    }
   ],
   "source": [
    "%%writefile ./LLaMA-Efficient-Tuning/requirements.txt\n",
    "#transformers>=4.29.1\n",
    "#datasets>=2.12.0\n",
    "#accelerate>=0.19.0\n",
    "#peft>=0.3.0\n",
    "#trl>=0.4.4\n",
    "#sentencepiece\n",
    "#jieba\n",
    "#rouge-chinese\n",
    "#nltk\n",
    "#gradio\n",
    "#mdtex2html\n",
    "#uvicorn\n",
    "#fastapi\n",
    "#sse-starlette\n",
    "#transformers_stream_generator\n",
    "#deepspeed\n",
    "#xformers\n",
    "#wandb\n",
    "transformers>=4.29.2\n",
    "datasets>=2.12.0\n",
    "accelerate>=0.19.0\n",
    "peft>=0.3.0\n",
    "trl>=0.4.4\n",
    "sentencepiece\n",
    "jieba\n",
    "rouge-chinese\n",
    "nltk\n",
    "gradio\n",
    "mdtex2html\n",
    "uvicorn\n",
    "fastapi\n",
    "sse-starlette\n",
    "transformers_stream_generator\n",
    "wandb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "1c00bd60-67c4-413a-a68a-f25c02ca1e58",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting ./LLaMA-Efficient-Tuning/train.sh\n"
     ]
    }
   ],
   "source": [
    "%%writefile ./LLaMA-Efficient-Tuning/train.sh\n",
    "#!/bin/bash\n",
    "\n",
    "chmod +x ./s5cmd\n",
    "#chmod +x ./monitor.sh\n",
    "\n",
    "pip install -U -r requirements.txt\n",
    "\n",
    "apt install -y inotify-tools\n",
    "cp ./s5cmd /usr/bin\n",
    "\n",
    "\n",
    "\n",
    "./s5cmd sync $MODEL_S3_PATH* /tmp/baichun-13b/\n",
    "cp tests/modeling_baichuan.py /tmp/baichuan-13b/\n",
    "\n",
    "# This is secret and shouldn't be checked into version control\n",
    "export  WANDB_API_KEY=\"64f29a79439a4153b2b9f42f05eba7c3c5ca7b95\"\n",
    "# Name and notes optional\n",
    "export WANDB_PROJECT=\"sm-baichuan7b-sft\"\n",
    "\n",
    "\n",
    "export CUDA_VISIBLE_DEVICES='0,1,2,3,4,5,6,7'\n",
    "accelerate launch --config_file ds_zero2.yaml  src/train_bash.py \\\n",
    "    --model_name_or_path \"/tmp/baichun-13b/\" \\\n",
    "    --do_train \\\n",
    "    --dataset zhetian_sft \\\n",
    "    --finetuning_type lora \\\n",
    "    --output_dir /tmp/ouput/ \\\n",
    "    --overwrite_cache \\\n",
    "    --per_device_train_batch_size 4 \\\n",
    "    --gradient_accumulation_steps 2 \\\n",
    "    --lr_scheduler_type cosine \\\n",
    "    --logging_steps 10 \\\n",
    "    --save_steps 100 \\\n",
    "    --learning_rate 3e-4 \\\n",
    "    --num_train_epochs 2 \\\n",
    "    --plot_loss \\\n",
    "    --fp16 \\\n",
    "    --report_to wandb \\\n",
    "    --padding_side right \\\n",
    "    --prompt_template baichuan \\\n",
    "    --lora_rank 64 \\\n",
    "    --lora_alpha 128 \\\n",
    "    --max_source_length 1024 \\\n",
    "    --lora_target \"W_pack,o_proj,gate_proj,up_proj,down_proj\"\n",
    "\n",
    "#./s5cmd sync /tmp/ouput/ $MODEL_S3_PATH/models/baichuan_finetuning/output/$cur_date/\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "9ff767c5-d65e-4697-b287-63168724f0c7",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'s3://sagemaker-us-west-2-687912291502/llm/models/LLM_baichuan_model_13b/'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_s3_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "3c9f0cc5-6b01-4c10-b083-6f2ddf7d6f7a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "use_spot_instances = False\n",
    "max_run = 3600*24\n",
    "max_wait = 3600*24*2 if use_spot_instances else None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "05074911-7b33-4d96-b7e6-5cfa655b8672",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using provided s3_resource\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:sagemaker:Creating training-job with name: baichuan-finetuning-2023-07-20-05-51-37-413\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2023-07-20 05:51:59 Starting - Starting the training job......\n",
      "2023-07-20 05:52:37 Starting - Preparing the instances for training.....................\n",
      "2023-07-20 05:56:30 Downloading - Downloading input data\n",
      "2023-07-20 05:56:30 Training - Downloading the training image.....................\n",
      "2023-07-20 06:00:01 Training - Training image download completed. Training in progress........\u001b[34mbash: cannot set terminal process group (-1): Inappropriate ioctl for device\u001b[0m\n",
      "\u001b[34mbash: no job control in this shell\u001b[0m\n",
      "\u001b[34m2023-07-20 06:00:59,057 sagemaker-training-toolkit INFO     Imported framework sagemaker_pytorch_container.training\u001b[0m\n",
      "\u001b[34m2023-07-20 06:00:59,122 sagemaker-training-toolkit INFO     No Neurons detected (normal if no neurons installed)\u001b[0m\n",
      "\u001b[34m2023-07-20 06:00:59,131 sagemaker_pytorch_container.training INFO     Block until all host DNS lookups succeed.\u001b[0m\n",
      "\u001b[34m2023-07-20 06:00:59,133 sagemaker_pytorch_container.training INFO     Invoking user training script.\u001b[0m\n",
      "\u001b[34m2023-07-20 06:01:02,391 sagemaker-training-toolkit INFO     No Neurons detected (normal if no neurons installed)\u001b[0m\n",
      "\u001b[34m2023-07-20 06:01:02,464 sagemaker-training-toolkit INFO     No Neurons detected (normal if no neurons installed)\u001b[0m\n",
      "\u001b[34m2023-07-20 06:01:02,538 sagemaker-training-toolkit INFO     No Neurons detected (normal if no neurons installed)\u001b[0m\n",
      "\u001b[34m2023-07-20 06:01:02,549 sagemaker-training-toolkit INFO     Invoking user script\u001b[0m\n",
      "\u001b[34mTraining Env:\u001b[0m\n",
      "\u001b[34m{\n",
      "    \"additional_framework_parameters\": {},\n",
      "    \"channel_input_dirs\": {},\n",
      "    \"current_host\": \"algo-1\",\n",
      "    \"current_instance_group\": \"homogeneousCluster\",\n",
      "    \"current_instance_group_hosts\": [\n",
      "        \"algo-1\"\n",
      "    ],\n",
      "    \"current_instance_type\": \"ml.p4d.24xlarge\",\n",
      "    \"distribution_hosts\": [],\n",
      "    \"distribution_instance_groups\": [],\n",
      "    \"framework_module\": \"sagemaker_pytorch_container.training:main\",\n",
      "    \"hosts\": [\n",
      "        \"algo-1\"\n",
      "    ],\n",
      "    \"hyperparameters\": {},\n",
      "    \"input_config_dir\": \"/opt/ml/input/config\",\n",
      "    \"input_data_config\": {},\n",
      "    \"input_dir\": \"/opt/ml/input\",\n",
      "    \"instance_groups\": [\n",
      "        \"homogeneousCluster\"\n",
      "    ],\n",
      "    \"instance_groups_dict\": {\n",
      "        \"homogeneousCluster\": {\n",
      "            \"instance_group_name\": \"homogeneousCluster\",\n",
      "            \"instance_type\": \"ml.p4d.24xlarge\",\n",
      "            \"hosts\": [\n",
      "                \"algo-1\"\n",
      "            ]\n",
      "        }\n",
      "    },\n",
      "    \"is_hetero\": false,\n",
      "    \"is_master\": true,\n",
      "    \"is_modelparallel_enabled\": null,\n",
      "    \"is_smddpmprun_installed\": true,\n",
      "    \"job_name\": \"baichuan-finetuning-2023-07-20-05-51-37-413\",\n",
      "    \"log_level\": 20,\n",
      "    \"master_hostname\": \"algo-1\",\n",
      "    \"model_dir\": \"/opt/ml/model\",\n",
      "    \"module_dir\": \"s3://sagemaker-us-west-2-687912291502/baichuan-finetuning-2023-07-20-05-51-37-413/source/sourcedir.tar.gz\",\n",
      "    \"module_name\": \"train.sh\",\n",
      "    \"network_interface_name\": \"eth0\",\n",
      "    \"num_cpus\": 96,\n",
      "    \"num_gpus\": 8,\n",
      "    \"num_neurons\": 0,\n",
      "    \"output_data_dir\": \"/opt/ml/output/data\",\n",
      "    \"output_dir\": \"/opt/ml/output\",\n",
      "    \"output_intermediate_dir\": \"/opt/ml/output/intermediate\",\n",
      "    \"resource_config\": {\n",
      "        \"current_host\": \"algo-1\",\n",
      "        \"current_instance_type\": \"ml.p4d.24xlarge\",\n",
      "        \"current_group_name\": \"homogeneousCluster\",\n",
      "        \"hosts\": [\n",
      "            \"algo-1\"\n",
      "        ],\n",
      "        \"instance_groups\": [\n",
      "            {\n",
      "                \"instance_group_name\": \"homogeneousCluster\",\n",
      "                \"instance_type\": \"ml.p4d.24xlarge\",\n",
      "                \"hosts\": [\n",
      "                    \"algo-1\"\n",
      "                ]\n",
      "            }\n",
      "        ],\n",
      "        \"network_interface_name\": \"eth0\"\n",
      "    },\n",
      "    \"user_entry_point\": \"train.sh\"\u001b[0m\n",
      "\u001b[34m}\u001b[0m\n",
      "\u001b[34mEnvironment variables:\u001b[0m\n",
      "\u001b[34mSM_HOSTS=[\"algo-1\"]\u001b[0m\n",
      "\u001b[34mSM_NETWORK_INTERFACE_NAME=eth0\u001b[0m\n",
      "\u001b[34mSM_HPS={}\u001b[0m\n",
      "\u001b[34mSM_USER_ENTRY_POINT=train.sh\u001b[0m\n",
      "\u001b[34mSM_FRAMEWORK_PARAMS={}\u001b[0m\n",
      "\u001b[34mSM_RESOURCE_CONFIG={\"current_group_name\":\"homogeneousCluster\",\"current_host\":\"algo-1\",\"current_instance_type\":\"ml.p4d.24xlarge\",\"hosts\":[\"algo-1\"],\"instance_groups\":[{\"hosts\":[\"algo-1\"],\"instance_group_name\":\"homogeneousCluster\",\"instance_type\":\"ml.p4d.24xlarge\"}],\"network_interface_name\":\"eth0\"}\u001b[0m\n",
      "\u001b[34mSM_INPUT_DATA_CONFIG={}\u001b[0m\n",
      "\u001b[34mSM_OUTPUT_DATA_DIR=/opt/ml/output/data\u001b[0m\n",
      "\u001b[34mSM_CHANNELS=[]\u001b[0m\n",
      "\u001b[34mSM_CURRENT_HOST=algo-1\u001b[0m\n",
      "\u001b[34mSM_CURRENT_INSTANCE_TYPE=ml.p4d.24xlarge\u001b[0m\n",
      "\u001b[34mSM_CURRENT_INSTANCE_GROUP=homogeneousCluster\u001b[0m\n",
      "\u001b[34mSM_CURRENT_INSTANCE_GROUP_HOSTS=[\"algo-1\"]\u001b[0m\n",
      "\u001b[34mSM_INSTANCE_GROUPS=[\"homogeneousCluster\"]\u001b[0m\n",
      "\u001b[34mSM_INSTANCE_GROUPS_DICT={\"homogeneousCluster\":{\"hosts\":[\"algo-1\"],\"instance_group_name\":\"homogeneousCluster\",\"instance_type\":\"ml.p4d.24xlarge\"}}\u001b[0m\n",
      "\u001b[34mSM_DISTRIBUTION_INSTANCE_GROUPS=[]\u001b[0m\n",
      "\u001b[34mSM_IS_HETERO=false\u001b[0m\n",
      "\u001b[34mSM_MODULE_NAME=train.sh\u001b[0m\n",
      "\u001b[34mSM_LOG_LEVEL=20\u001b[0m\n",
      "\u001b[34mSM_FRAMEWORK_MODULE=sagemaker_pytorch_container.training:main\u001b[0m\n",
      "\u001b[34mSM_INPUT_DIR=/opt/ml/input\u001b[0m\n",
      "\u001b[34mSM_INPUT_CONFIG_DIR=/opt/ml/input/config\u001b[0m\n",
      "\u001b[34mSM_OUTPUT_DIR=/opt/ml/output\u001b[0m\n",
      "\u001b[34mSM_NUM_CPUS=96\u001b[0m\n",
      "\u001b[34mSM_NUM_GPUS=8\u001b[0m\n",
      "\u001b[34mSM_NUM_NEURONS=0\u001b[0m\n",
      "\u001b[34mSM_MODEL_DIR=/opt/ml/model\u001b[0m\n",
      "\u001b[34mSM_MODULE_DIR=s3://sagemaker-us-west-2-687912291502/baichuan-finetuning-2023-07-20-05-51-37-413/source/sourcedir.tar.gz\u001b[0m\n",
      "\u001b[34mSM_TRAINING_ENV={\"additional_framework_parameters\":{},\"channel_input_dirs\":{},\"current_host\":\"algo-1\",\"current_instance_group\":\"homogeneousCluster\",\"current_instance_group_hosts\":[\"algo-1\"],\"current_instance_type\":\"ml.p4d.24xlarge\",\"distribution_hosts\":[],\"distribution_instance_groups\":[],\"framework_module\":\"sagemaker_pytorch_container.training:main\",\"hosts\":[\"algo-1\"],\"hyperparameters\":{},\"input_config_dir\":\"/opt/ml/input/config\",\"input_data_config\":{},\"input_dir\":\"/opt/ml/input\",\"instance_groups\":[\"homogeneousCluster\"],\"instance_groups_dict\":{\"homogeneousCluster\":{\"hosts\":[\"algo-1\"],\"instance_group_name\":\"homogeneousCluster\",\"instance_type\":\"ml.p4d.24xlarge\"}},\"is_hetero\":false,\"is_master\":true,\"is_modelparallel_enabled\":null,\"is_smddpmprun_installed\":true,\"job_name\":\"baichuan-finetuning-2023-07-20-05-51-37-413\",\"log_level\":20,\"master_hostname\":\"algo-1\",\"model_dir\":\"/opt/ml/model\",\"module_dir\":\"s3://sagemaker-us-west-2-687912291502/baichuan-finetuning-2023-07-20-05-51-37-413/source/sourcedir.tar.gz\",\"module_name\":\"train.sh\",\"network_interface_name\":\"eth0\",\"num_cpus\":96,\"num_gpus\":8,\"num_neurons\":0,\"output_data_dir\":\"/opt/ml/output/data\",\"output_dir\":\"/opt/ml/output\",\"output_intermediate_dir\":\"/opt/ml/output/intermediate\",\"resource_config\":{\"current_group_name\":\"homogeneousCluster\",\"current_host\":\"algo-1\",\"current_instance_type\":\"ml.p4d.24xlarge\",\"hosts\":[\"algo-1\"],\"instance_groups\":[{\"hosts\":[\"algo-1\"],\"instance_group_name\":\"homogeneousCluster\",\"instance_type\":\"ml.p4d.24xlarge\"}],\"network_interface_name\":\"eth0\"},\"user_entry_point\":\"train.sh\"}\u001b[0m\n",
      "\u001b[34mSM_USER_ARGS=[]\u001b[0m\n",
      "\u001b[34mSM_OUTPUT_INTERMEDIATE_DIR=/opt/ml/output/intermediate\u001b[0m\n",
      "\u001b[34mPYTHONPATH=/opt/ml/code:/opt/conda/bin:/opt/conda/lib/python39.zip:/opt/conda/lib/python3.9:/opt/conda/lib/python3.9/lib-dynload:/opt/conda/lib/python3.9/site-packages\u001b[0m\n",
      "\u001b[34mInvoking script with the following command:\u001b[0m\n",
      "\u001b[34m/bin/sh -c \"./train.sh \"\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:01:03,884] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:01:06.664: W smdistributed/modelparallel/torch/nn/predefined_hooks.py:78] Found unsupported HuggingFace version 4.26.0 for automated tensor parallelism. HuggingFace modules will not be automatically distributed. You can use smp.tp_register_with_module API to register desired modules for tensor parallelism, or directly instantiate an smp.nn.DistributedModule. Supported HuggingFace transformers versions for automated tensor parallelism: ['4.17.0', '4.20.1', '4.21.0']\u001b[0m\n",
      "\u001b[34m2023-07-20 06:01:06,668 root         INFO     Using NamedTuple = typing._NamedTuple instead.\u001b[0m\n",
      "\u001b[34m2023-07-20 06:01:06,686 sagemaker-training-toolkit INFO     Exceptions not imported for SageMaker TF as Tensorflow is not installed.\u001b[0m\n",
      "\u001b[34mCollecting transformers>=4.29.2\u001b[0m\n",
      "\u001b[34mDownloading transformers-4.31.0-py3-none-any.whl (7.4 MB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 7.4/7.4 MB 70.0 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting datasets>=2.12.0\u001b[0m\n",
      "\u001b[34mDownloading datasets-2.13.1-py3-none-any.whl (486 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 486.2/486.2 kB 60.6 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting accelerate>=0.19.0\u001b[0m\n",
      "\u001b[34mDownloading accelerate-0.21.0-py3-none-any.whl (244 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 244.2/244.2 kB 49.9 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting peft>=0.3.0\u001b[0m\n",
      "\u001b[34mDownloading peft-0.4.0-py3-none-any.whl (72 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 72.9/72.9 kB 21.1 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting trl>=0.4.4\u001b[0m\n",
      "\u001b[34mDownloading trl-0.4.7-py3-none-any.whl (77 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 77.4/77.4 kB 22.1 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: sentencepiece in /opt/conda/lib/python3.9/site-packages (from -r requirements.txt (line 24)) (0.1.97)\u001b[0m\n",
      "\u001b[34mCollecting sentencepiece\u001b[0m\n",
      "\u001b[34mDownloading sentencepiece-0.1.99-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.3/1.3 MB 88.7 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting jieba\u001b[0m\n",
      "\u001b[34mDownloading jieba-0.42.1.tar.gz (19.2 MB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 19.2/19.2 MB 72.1 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mPreparing metadata (setup.py): started\u001b[0m\n",
      "\u001b[34mPreparing metadata (setup.py): finished with status 'done'\u001b[0m\n",
      "\u001b[34mCollecting rouge-chinese\u001b[0m\n",
      "\u001b[34mDownloading rouge_chinese-1.0.3-py3-none-any.whl (21 kB)\u001b[0m\n",
      "\u001b[34mCollecting nltk\u001b[0m\n",
      "\u001b[34mDownloading nltk-3.8.1-py3-none-any.whl (1.5 MB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.5/1.5 MB 98.9 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting gradio\u001b[0m\n",
      "\u001b[34mDownloading gradio-3.37.0-py3-none-any.whl (19.8 MB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 19.8/19.8 MB 63.2 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting mdtex2html\u001b[0m\n",
      "\u001b[34mDownloading mdtex2html-1.2.0-py3-none-any.whl (13 kB)\u001b[0m\n",
      "\u001b[34mCollecting uvicorn\u001b[0m\n",
      "\u001b[34mDownloading uvicorn-0.23.1-py3-none-any.whl (59 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 59.5/59.5 kB 17.6 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting fastapi\u001b[0m\n",
      "\u001b[34mDownloading fastapi-0.100.0-py3-none-any.whl (65 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 65.7/65.7 kB 14.8 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting sse-starlette\u001b[0m\n",
      "\u001b[34mDownloading sse_starlette-1.6.1-py3-none-any.whl (9.6 kB)\u001b[0m\n",
      "\u001b[34mCollecting transformers_stream_generator\u001b[0m\n",
      "\u001b[34mDownloading transformers-stream-generator-0.0.4.tar.gz (12 kB)\u001b[0m\n",
      "\u001b[34mPreparing metadata (setup.py): started\u001b[0m\n",
      "\u001b[34mPreparing metadata (setup.py): finished with status 'done'\u001b[0m\n",
      "\u001b[34mCollecting wandb\u001b[0m\n",
      "\u001b[34mDownloading wandb-0.15.5-py3-none-any.whl (2.1 MB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 2.1/2.1 MB 102.8 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /opt/conda/lib/python3.9/site-packages (from transformers>=4.29.2->-r requirements.txt (line 19)) (0.13.2)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.9/site-packages (from transformers>=4.29.2->-r requirements.txt (line 19)) (23.0)\u001b[0m\n",
      "\u001b[34mCollecting safetensors>=0.3.1\u001b[0m\n",
      "\u001b[34mDownloading safetensors-0.3.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.3/1.3 MB 104.3 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.9/site-packages (from transformers>=4.29.2->-r requirements.txt (line 19)) (5.4.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.9/site-packages (from transformers>=4.29.2->-r requirements.txt (line 19)) (1.23.5)\u001b[0m\n",
      "\u001b[34mCollecting huggingface-hub<1.0,>=0.14.1\u001b[0m\n",
      "\u001b[34mDownloading huggingface_hub-0.16.4-py3-none-any.whl (268 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 268.8/268.8 kB 54.9 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: filelock in /opt/conda/lib/python3.9/site-packages (from transformers>=4.29.2->-r requirements.txt (line 19)) (3.9.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: requests in /opt/conda/lib/python3.9/site-packages (from transformers>=4.29.2->-r requirements.txt (line 19)) (2.28.2)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: tqdm>=4.27 in /opt/conda/lib/python3.9/site-packages (from transformers>=4.29.2->-r requirements.txt (line 19)) (4.64.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: regex!=2019.12.17 in /opt/conda/lib/python3.9/site-packages (from transformers>=4.29.2->-r requirements.txt (line 19)) (2022.10.31)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: xxhash in /opt/conda/lib/python3.9/site-packages (from datasets>=2.12.0->-r requirements.txt (line 20)) (3.2.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: aiohttp in /opt/conda/lib/python3.9/site-packages (from datasets>=2.12.0->-r requirements.txt (line 20)) (3.8.4)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pandas in /opt/conda/lib/python3.9/site-packages (from datasets>=2.12.0->-r requirements.txt (line 20)) (1.5.3)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: fsspec[http]>=2021.11.1 in /opt/conda/lib/python3.9/site-packages (from datasets>=2.12.0->-r requirements.txt (line 20)) (2023.1.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: dill<0.3.7,>=0.3.0 in /opt/conda/lib/python3.9/site-packages (from datasets>=2.12.0->-r requirements.txt (line 20)) (0.3.6)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: multiprocess in /opt/conda/lib/python3.9/site-packages (from datasets>=2.12.0->-r requirements.txt (line 20)) (0.70.14)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pyarrow>=8.0.0 in /opt/conda/lib/python3.9/site-packages (from datasets>=2.12.0->-r requirements.txt (line 20)) (11.0.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: psutil in /opt/conda/lib/python3.9/site-packages (from accelerate>=0.19.0->-r requirements.txt (line 21)) (5.9.4)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: torch>=1.10.0 in /opt/conda/lib/python3.9/site-packages (from accelerate>=0.19.0->-r requirements.txt (line 21)) (1.13.1+cu117)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: six in /opt/conda/lib/python3.9/site-packages (from rouge-chinese->-r requirements.txt (line 26)) (1.16.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: click in /opt/conda/lib/python3.9/site-packages (from nltk->-r requirements.txt (line 27)) (8.1.2)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: joblib in /opt/conda/lib/python3.9/site-packages (from nltk->-r requirements.txt (line 27)) (1.2.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: typing-extensions~=4.0 in /opt/conda/lib/python3.9/site-packages (from gradio->-r requirements.txt (line 28)) (4.4.0)\u001b[0m\n",
      "\u001b[34mCollecting python-multipart\u001b[0m\n",
      "\u001b[34mDownloading python_multipart-0.0.6-py3-none-any.whl (45 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 45.7/45.7 kB 13.4 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting altair<6.0,>=4.2.0\u001b[0m\n",
      "\u001b[34mDownloading altair-5.0.1-py3-none-any.whl (471 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 471.5/471.5 kB 76.8 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: jinja2<4.0 in /opt/conda/lib/python3.9/site-packages (from gradio->-r requirements.txt (line 28)) (3.1.2)\u001b[0m\n",
      "\u001b[34mCollecting orjson~=3.0\u001b[0m\n",
      "\u001b[34mDownloading orjson-3.9.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (138 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 138.6/138.6 kB 35.1 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting websockets<12.0,>=10.0\u001b[0m\n",
      "\u001b[34mDownloading websockets-11.0.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (129 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 129.7/129.7 kB 27.8 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting httpx\u001b[0m\n",
      "\u001b[34mDownloading httpx-0.24.1-py3-none-any.whl (75 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 75.4/75.4 kB 23.8 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: markupsafe~=2.0 in /opt/conda/lib/python3.9/site-packages (from gradio->-r requirements.txt (line 28)) (2.1.2)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pillow<11.0,>=8.0 in /opt/conda/lib/python3.9/site-packages (from gradio->-r requirements.txt (line 28)) (9.4.0)\u001b[0m\n",
      "\u001b[34mCollecting gradio-client>=0.2.10\u001b[0m\n",
      "\u001b[34mDownloading gradio_client-0.2.10-py3-none-any.whl (288 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 289.0/289.0 kB 63.4 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting mdit-py-plugins<=0.3.3\u001b[0m\n",
      "\u001b[34mDownloading mdit_py_plugins-0.3.3-py3-none-any.whl (50 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 50.5/50.5 kB 14.9 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting ffmpy\u001b[0m\n",
      "\u001b[34mDownloading ffmpy-0.3.1.tar.gz (5.5 kB)\u001b[0m\n",
      "\u001b[34mPreparing metadata (setup.py): started\u001b[0m\n",
      "\u001b[34mPreparing metadata (setup.py): finished with status 'done'\u001b[0m\n",
      "\u001b[34mCollecting aiofiles<24.0,>=22.0\u001b[0m\n",
      "\u001b[34mDownloading aiofiles-23.1.0-py3-none-any.whl (14 kB)\u001b[0m\n",
      "\u001b[34mCollecting pydub\u001b[0m\n",
      "\u001b[34mDownloading pydub-0.25.1-py2.py3-none-any.whl (32 kB)\u001b[0m\n",
      "\u001b[34mCollecting markdown-it-py[linkify]>=2.0.0\u001b[0m\n",
      "\u001b[34mDownloading markdown_it_py-3.0.0-py3-none-any.whl (87 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 87.5/87.5 kB 26.9 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting semantic-version~=2.0\u001b[0m\n",
      "\u001b[34mDownloading semantic_version-2.10.0-py2.py3-none-any.whl (15 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pydantic!=1.8,!=1.8.1,!=2.0.0,!=2.0.1,<3.0.0,>=1.7.4 in /opt/conda/lib/python3.9/site-packages (from gradio->-r requirements.txt (line 28)) (1.10.4)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: matplotlib~=3.0 in /opt/conda/lib/python3.9/site-packages (from gradio->-r requirements.txt (line 28)) (3.6.3)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: markdown in /opt/conda/lib/python3.9/site-packages (from mdtex2html->-r requirements.txt (line 29)) (3.4.1)\u001b[0m\n",
      "\u001b[34mCollecting latex2mathml\u001b[0m\n",
      "\u001b[34mDownloading latex2mathml-3.76.0-py3-none-any.whl (73 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 73.4/73.4 kB 24.2 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting h11>=0.8\u001b[0m\n",
      "\u001b[34mDownloading h11-0.14.0-py3-none-any.whl (58 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 58.3/58.3 kB 18.1 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting starlette<0.28.0,>=0.27.0\u001b[0m\n",
      "\u001b[34mDownloading starlette-0.27.0-py3-none-any.whl (66 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 67.0/67.0 kB 17.2 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting typing-extensions~=4.0\u001b[0m\n",
      "\u001b[34mDownloading typing_extensions-4.7.1-py3-none-any.whl (33 kB)\u001b[0m\n",
      "\u001b[34mCollecting sentry-sdk>=1.0.0\u001b[0m\n",
      "\u001b[34mDownloading sentry_sdk-1.28.1-py2.py3-none-any.whl (214 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 214.7/214.7 kB 44.3 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: setuptools in /opt/conda/lib/python3.9/site-packages (from wandb->-r requirements.txt (line 34)) (65.6.3)\u001b[0m\n",
      "\u001b[34mCollecting GitPython!=3.1.29,>=1.0.0\u001b[0m\n",
      "\u001b[34mDownloading GitPython-3.1.32-py3-none-any.whl (188 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 188.5/188.5 kB 33.6 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting setproctitle\u001b[0m\n",
      "\u001b[34mDownloading setproctitle-1.3.2-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (30 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: appdirs>=1.4.3 in /opt/conda/lib/python3.9/site-packages (from wandb->-r requirements.txt (line 34)) (1.4.4)\u001b[0m\n",
      "\u001b[34mCollecting docker-pycreds>=0.4.0\u001b[0m\n",
      "\u001b[34mDownloading docker_pycreds-0.4.0-py2.py3-none-any.whl (9.0 kB)\u001b[0m\n",
      "\u001b[34mCollecting pathtools\u001b[0m\n",
      "\u001b[34mDownloading pathtools-0.1.2.tar.gz (11 kB)\u001b[0m\n",
      "\u001b[34mPreparing metadata (setup.py): started\u001b[0m\n",
      "\u001b[34mPreparing metadata (setup.py): finished with status 'done'\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: protobuf!=4.21.0,<5,>=3.15.0 in /opt/conda/lib/python3.9/site-packages (from wandb->-r requirements.txt (line 34)) (3.20.2)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: aiosignal>=1.1.2 in /opt/conda/lib/python3.9/site-packages (from aiohttp->datasets>=2.12.0->-r requirements.txt (line 20)) (1.3.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: charset-normalizer<4.0,>=2.0 in /opt/conda/lib/python3.9/site-packages (from aiohttp->datasets>=2.12.0->-r requirements.txt (line 20)) (2.1.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: yarl<2.0,>=1.0 in /opt/conda/lib/python3.9/site-packages (from aiohttp->datasets>=2.12.0->-r requirements.txt (line 20)) (1.8.2)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: multidict<7.0,>=4.5 in /opt/conda/lib/python3.9/site-packages (from aiohttp->datasets>=2.12.0->-r requirements.txt (line 20)) (6.0.4)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: attrs>=17.3.0 in /opt/conda/lib/python3.9/site-packages (from aiohttp->datasets>=2.12.0->-r requirements.txt (line 20)) (22.2.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: frozenlist>=1.1.1 in /opt/conda/lib/python3.9/site-packages (from aiohttp->datasets>=2.12.0->-r requirements.txt (line 20)) (1.3.3)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /opt/conda/lib/python3.9/site-packages (from aiohttp->datasets>=2.12.0->-r requirements.txt (line 20)) (4.0.2)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: jsonschema>=3.0 in /opt/conda/lib/python3.9/site-packages (from altair<6.0,>=4.2.0->gradio->-r requirements.txt (line 28)) (4.17.3)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: toolz in /opt/conda/lib/python3.9/site-packages (from altair<6.0,>=4.2.0->gradio->-r requirements.txt (line 28)) (0.12.0)\u001b[0m\n",
      "\u001b[34mCollecting gitdb<5,>=4.0.1\u001b[0m\n",
      "\u001b[34mDownloading gitdb-4.0.10-py3-none-any.whl (62 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 62.7/62.7 kB 14.1 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting mdurl~=0.1\u001b[0m\n",
      "\u001b[34mDownloading mdurl-0.1.2-py3-none-any.whl (10.0 kB)\u001b[0m\n",
      "\u001b[34mCollecting linkify-it-py<3,>=1\u001b[0m\n",
      "\u001b[34mDownloading linkify_it_py-2.0.2-py3-none-any.whl (19 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: kiwisolver>=1.0.1 in /opt/conda/lib/python3.9/site-packages (from matplotlib~=3.0->gradio->-r requirements.txt (line 28)) (1.4.4)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: cycler>=0.10 in /opt/conda/lib/python3.9/site-packages (from matplotlib~=3.0->gradio->-r requirements.txt (line 28)) (0.11.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: python-dateutil>=2.7 in /opt/conda/lib/python3.9/site-packages (from matplotlib~=3.0->gradio->-r requirements.txt (line 28)) (2.8.2)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: contourpy>=1.0.1 in /opt/conda/lib/python3.9/site-packages (from matplotlib~=3.0->gradio->-r requirements.txt (line 28)) (1.0.7)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pyparsing>=2.2.1 in /opt/conda/lib/python3.9/site-packages (from matplotlib~=3.0->gradio->-r requirements.txt (line 28)) (3.0.9)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: fonttools>=4.22.0 in /opt/conda/lib/python3.9/site-packages (from matplotlib~=3.0->gradio->-r requirements.txt (line 28)) (4.38.0)\u001b[0m\n",
      "\u001b[34mCollecting mdit-py-plugins<=0.3.3\u001b[0m\n",
      "\u001b[34mDownloading mdit_py_plugins-0.3.2-py3-none-any.whl (50 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 50.4/50.4 kB 16.1 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading mdit_py_plugins-0.3.1-py3-none-any.whl (46 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 46.5/46.5 kB 14.2 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading mdit_py_plugins-0.3.0-py3-none-any.whl (43 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 43.7/43.7 kB 12.9 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading mdit_py_plugins-0.2.8-py3-none-any.whl (41 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 41.0/41.0 kB 11.4 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading mdit_py_plugins-0.2.7-py3-none-any.whl (41 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 41.0/41.0 kB 11.8 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading mdit_py_plugins-0.2.6-py3-none-any.whl (39 kB)\u001b[0m\n",
      "\u001b[34mDownloading mdit_py_plugins-0.2.5-py3-none-any.whl (39 kB)\u001b[0m\n",
      "\u001b[34mDownloading mdit_py_plugins-0.2.4-py3-none-any.whl (39 kB)\u001b[0m\n",
      "\u001b[34mDownloading mdit_py_plugins-0.2.3-py3-none-any.whl (39 kB)\u001b[0m\n",
      "\u001b[34mDownloading mdit_py_plugins-0.2.2-py3-none-any.whl (39 kB)\u001b[0m\n",
      "\u001b[34mDownloading mdit_py_plugins-0.2.1-py3-none-any.whl (38 kB)\u001b[0m\n",
      "\u001b[34mDownloading mdit_py_plugins-0.2.0-py3-none-any.whl (38 kB)\u001b[0m\n",
      "\u001b[34mDownloading mdit_py_plugins-0.1.0-py3-none-any.whl (37 kB)\u001b[0m\n",
      "\u001b[34mINFO: pip is looking at multiple versions of matplotlib to determine which version is compatible with other requirements. This could take a while.\u001b[0m\n",
      "\u001b[34mCollecting matplotlib~=3.0\u001b[0m\n",
      "\u001b[34mDownloading matplotlib-3.7.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (11.6 MB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 11.6/11.6 MB 106.5 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mINFO: pip is looking at multiple versions of markupsafe to determine which version is compatible with other requirements. This could take a while.\u001b[0m\n",
      "\u001b[34mCollecting markupsafe~=2.0\u001b[0m\n",
      "\u001b[34mDownloading MarkupSafe-2.1.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (25 kB)\u001b[0m\n",
      "\u001b[34mINFO: pip is looking at multiple versions of markdown-it-py[linkify] to determine which version is compatible with other requirements. This could take a while.\u001b[0m\n",
      "\u001b[34mCollecting markdown-it-py[linkify]>=2.0.0\u001b[0m\n",
      "\u001b[34mDownloading markdown_it_py-2.2.0-py3-none-any.whl (84 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 84.5/84.5 kB 22.5 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pytz>=2020.1 in /opt/conda/lib/python3.9/site-packages (from pandas->datasets>=2.12.0->-r requirements.txt (line 20)) (2022.7.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.9/site-packages (from requests->transformers>=4.29.2->-r requirements.txt (line 19)) (2022.12.7)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.9/site-packages (from requests->transformers>=4.29.2->-r requirements.txt (line 19)) (1.26.14)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.9/site-packages (from requests->transformers>=4.29.2->-r requirements.txt (line 19)) (3.4)\u001b[0m\n",
      "\u001b[34mCollecting anyio<5,>=3.4.0\u001b[0m\n",
      "\u001b[34mDownloading anyio-3.7.1-py3-none-any.whl (80 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 80.9/80.9 kB 27.5 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting sniffio\u001b[0m\n",
      "\u001b[34mDownloading sniffio-1.3.0-py3-none-any.whl (10 kB)\u001b[0m\n",
      "\u001b[34mCollecting httpcore<0.18.0,>=0.15.0\u001b[0m\n",
      "\u001b[34mDownloading httpcore-0.17.3-py3-none-any.whl (74 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 74.5/74.5 kB 20.9 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: importlib-metadata>=4.4 in /opt/conda/lib/python3.9/site-packages (from markdown->mdtex2html->-r requirements.txt (line 29)) (4.13.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: exceptiongroup in /opt/conda/lib/python3.9/site-packages (from anyio<5,>=3.4.0->starlette<0.28.0,>=0.27.0->fastapi->-r requirements.txt (line 31)) (1.1.0)\u001b[0m\n",
      "\u001b[34mCollecting smmap<6,>=3.0.1\u001b[0m\n",
      "\u001b[34mDownloading smmap-5.0.0-py3-none-any.whl (24 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: zipp>=0.5 in /opt/conda/lib/python3.9/site-packages (from importlib-metadata>=4.4->markdown->mdtex2html->-r requirements.txt (line 29)) (3.13.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pyrsistent!=0.17.0,!=0.17.1,!=0.17.2,>=0.14.0 in /opt/conda/lib/python3.9/site-packages (from jsonschema>=3.0->altair<6.0,>=4.2.0->gradio->-r requirements.txt (line 28)) (0.19.3)\u001b[0m\n",
      "\u001b[34mCollecting uc-micro-py\u001b[0m\n",
      "\u001b[34mDownloading uc_micro_py-1.0.2-py3-none-any.whl (6.2 kB)\u001b[0m\n",
      "\u001b[34mBuilding wheels for collected packages: jieba, transformers_stream_generator, ffmpy, pathtools\u001b[0m\n",
      "\u001b[34mBuilding wheel for jieba (setup.py): started\u001b[0m\n",
      "\u001b[34mBuilding wheel for jieba (setup.py): finished with status 'done'\u001b[0m\n",
      "\u001b[34mCreated wheel for jieba: filename=jieba-0.42.1-py3-none-any.whl size=19314458 sha256=415033859a8e3c8cfc26df54b9c3e4af8e52fa645b265c50009c71f2f3c56894\u001b[0m\n",
      "\u001b[34mStored in directory: /root/.cache/pip/wheels/7d/74/cf/08c94db4b784e2c1ef675a600b7b5b281fd25240dcb954ee7e\u001b[0m\n",
      "\u001b[34mBuilding wheel for transformers_stream_generator (setup.py): started\u001b[0m\n",
      "\u001b[34mBuilding wheel for transformers_stream_generator (setup.py): finished with status 'done'\u001b[0m\n",
      "\u001b[34mCreated wheel for transformers_stream_generator: filename=transformers_stream_generator-0.0.4-py3-none-any.whl size=12319 sha256=f1f57183b99587afd70cb65efe017da879751432a84b4d985bb65eb6341b2c14\u001b[0m\n",
      "\u001b[34mStored in directory: /root/.cache/pip/wheels/ac/3d/06/a5d757e655bfa10557f97478e3d5108ef66415d8c032832b60\u001b[0m\n",
      "\u001b[34mBuilding wheel for ffmpy (setup.py): started\u001b[0m\n",
      "\u001b[34mBuilding wheel for ffmpy (setup.py): finished with status 'done'\u001b[0m\n",
      "\u001b[34mCreated wheel for ffmpy: filename=ffmpy-0.3.1-py3-none-any.whl size=5580 sha256=8f3311f34e6591f9a02b75f407faf3cf471c0ad84cb40e1414ef583b0828b3cd\u001b[0m\n",
      "\u001b[34mStored in directory: /root/.cache/pip/wheels/1f/f1/8d/367922b023b526b7c2ced5db30932def7b18cf39d7ac6e8572\u001b[0m\n",
      "\u001b[34mBuilding wheel for pathtools (setup.py): started\u001b[0m\n",
      "\u001b[34mBuilding wheel for pathtools (setup.py): finished with status 'done'\u001b[0m\n",
      "\u001b[34mCreated wheel for pathtools: filename=pathtools-0.1.2-py3-none-any.whl size=8791 sha256=4535707664c4a9b75cd8c0e68b5295457d1f6e8e509230acb2c8b224e020b774\u001b[0m\n",
      "\u001b[34mStored in directory: /root/.cache/pip/wheels/b7/0a/67/ada2a22079218c75a88361c0782855cc72aebc4d18d0289d05\u001b[0m\n",
      "\u001b[34mSuccessfully built jieba transformers_stream_generator ffmpy pathtools\u001b[0m\n",
      "\u001b[34mInstalling collected packages: sentencepiece, safetensors, pydub, pathtools, jieba, ffmpy, websockets, uc-micro-py, typing-extensions, sniffio, smmap, setproctitle, sentry-sdk, semantic-version, rouge-chinese, python-multipart, orjson, nltk, mdurl, latex2mathml, h11, docker-pycreds, aiofiles, uvicorn, markdown-it-py, linkify-it-py, huggingface-hub, gitdb, anyio, transformers, starlette, mdtex2html, mdit-py-plugins, httpcore, GitPython, altair, accelerate, wandb, transformers_stream_generator, sse-starlette, peft, httpx, fastapi, datasets, trl, gradio-client, gradio\u001b[0m\n",
      "\u001b[34mAttempting uninstall: sentencepiece\u001b[0m\n",
      "\u001b[34mFound existing installation: sentencepiece 0.1.97\u001b[0m\n",
      "\u001b[34mUninstalling sentencepiece-0.1.97:\u001b[0m\n",
      "\u001b[34mSuccessfully uninstalled sentencepiece-0.1.97\u001b[0m\n",
      "\u001b[34mAttempting uninstall: typing-extensions\u001b[0m\n",
      "\u001b[34mFound existing installation: typing_extensions 4.4.0\u001b[0m\n",
      "\u001b[34mUninstalling typing_extensions-4.4.0:\u001b[0m\n",
      "\u001b[34mSuccessfully uninstalled typing_extensions-4.4.0\u001b[0m\n",
      "\u001b[34mAttempting uninstall: huggingface-hub\u001b[0m\n",
      "\u001b[34mFound existing installation: huggingface-hub 0.12.0\u001b[0m\n",
      "\u001b[34mUninstalling huggingface-hub-0.12.0:\u001b[0m\n",
      "\u001b[34mSuccessfully uninstalled huggingface-hub-0.12.0\u001b[0m\n",
      "\u001b[34mAttempting uninstall: transformers\u001b[0m\n",
      "\u001b[34mFound existing installation: transformers 4.26.0\u001b[0m\n",
      "\u001b[34mUninstalling transformers-4.26.0:\u001b[0m\n",
      "\u001b[34mSuccessfully uninstalled transformers-4.26.0\u001b[0m\n",
      "\u001b[34mAttempting uninstall: accelerate\u001b[0m\n",
      "\u001b[34mFound existing installation: accelerate 0.16.0\u001b[0m\n",
      "\u001b[34mUninstalling accelerate-0.16.0:\u001b[0m\n",
      "\u001b[34mSuccessfully uninstalled accelerate-0.16.0\u001b[0m\n",
      "\u001b[34mAttempting uninstall: datasets\u001b[0m\n",
      "\u001b[34mFound existing installation: datasets 2.9.0\u001b[0m\n",
      "\u001b[34mUninstalling datasets-2.9.0:\u001b[0m\n",
      "\u001b[34mSuccessfully uninstalled datasets-2.9.0\u001b[0m\n",
      "\u001b[34mSuccessfully installed GitPython-3.1.32 accelerate-0.21.0 aiofiles-23.1.0 altair-5.0.1 anyio-3.7.1 datasets-2.13.1 docker-pycreds-0.4.0 fastapi-0.100.0 ffmpy-0.3.1 gitdb-4.0.10 gradio-3.37.0 gradio-client-0.2.10 h11-0.14.0 httpcore-0.17.3 httpx-0.24.1 huggingface-hub-0.16.4 jieba-0.42.1 latex2mathml-3.76.0 linkify-it-py-2.0.2 markdown-it-py-2.2.0 mdit-py-plugins-0.3.3 mdtex2html-1.2.0 mdurl-0.1.2 nltk-3.8.1 orjson-3.9.2 pathtools-0.1.2 peft-0.4.0 pydub-0.25.1 python-multipart-0.0.6 rouge-chinese-1.0.3 safetensors-0.3.1 semantic-version-2.10.0 sentencepiece-0.1.99 sentry-sdk-1.28.1 setproctitle-1.3.2 smmap-5.0.0 sniffio-1.3.0 sse-starlette-1.6.1 starlette-0.27.0 transformers-4.31.0 transformers_stream_generator-0.0.4 trl-0.4.7 typing-extensions-4.7.1 uc-micro-py-1.0.2 uvicorn-0.23.1 wandb-0.15.5 websockets-11.0.3\u001b[0m\n",
      "\u001b[34mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\n",
      "\u001b[34m[notice] A new release of pip is available: 23.0 -> 23.2\u001b[0m\n",
      "\u001b[34m[notice] To update, run: pip install --upgrade pip\u001b[0m\n",
      "\u001b[34mWARNING: apt does not have a stable CLI interface. Use with caution in scripts.\u001b[0m\n",
      "\u001b[34mReading package lists...\u001b[0m\n",
      "\u001b[34mBuilding dependency tree...\u001b[0m\n",
      "\u001b[34mReading state information...\u001b[0m\n",
      "\u001b[34mE: Unable to locate package inotify-tools\u001b[0m\n",
      "\u001b[34mcp s3://sagemaker-us-west-2-687912291502/llm/models/LLM_baichuan_model_13b/configuration_baichuan.py /tmp/baichun-13b/configuration_baichuan.py\u001b[0m\n",
      "\u001b[34mcp s3://sagemaker-us-west-2-687912291502/llm/models/LLM_baichuan_model_13b/generation_config.json /tmp/baichun-13b/generation_config.json\u001b[0m\n",
      "\u001b[34mcp s3://sagemaker-us-west-2-687912291502/llm/models/LLM_baichuan_model_13b/tokenizer_config.json /tmp/baichun-13b/tokenizer_config.json\u001b[0m\n",
      "\u001b[34mcp s3://sagemaker-us-west-2-687912291502/llm/models/LLM_baichuan_model_13b/config.json /tmp/baichun-13b/config.json\u001b[0m\n",
      "\u001b[34mcp s3://sagemaker-us-west-2-687912291502/llm/models/LLM_baichuan_model_13b/pytorch_model.bin.index.json /tmp/baichun-13b/pytorch_model.bin.index.json\u001b[0m\n",
      "\u001b[34mcp s3://sagemaker-us-west-2-687912291502/llm/models/LLM_baichuan_model_13b/modeling_baichuan.py /tmp/baichun-13b/modeling_baichuan.py\u001b[0m\n",
      "\u001b[34mcp s3://sagemaker-us-west-2-687912291502/llm/models/LLM_baichuan_model_13b/quantizer.py /tmp/baichun-13b/quantizer.py\u001b[0m\n",
      "\u001b[34mcp s3://sagemaker-us-west-2-687912291502/llm/models/LLM_baichuan_model_13b/tokenization_baichuan.py /tmp/baichun-13b/tokenization_baichuan.py\u001b[0m\n",
      "\u001b[34mcp s3://sagemaker-us-west-2-687912291502/llm/models/LLM_baichuan_model_13b/special_tokens_map.json /tmp/baichun-13b/special_tokens_map.json\u001b[0m\n",
      "\u001b[34mcp s3://sagemaker-us-west-2-687912291502/llm/models/LLM_baichuan_model_13b/.gitattributes /tmp/baichun-13b/.gitattributes\u001b[0m\n",
      "\u001b[34mcp s3://sagemaker-us-west-2-687912291502/llm/models/LLM_baichuan_model_13b/requirements.txt /tmp/baichun-13b/requirements.txt\u001b[0m\n",
      "\u001b[34mcp s3://sagemaker-us-west-2-687912291502/llm/models/LLM_baichuan_model_13b/README.md /tmp/baichun-13b/README.md\u001b[0m\n",
      "\u001b[34mcp s3://sagemaker-us-west-2-687912291502/llm/models/LLM_baichuan_model_13b/Community License for Baichuan-13B Model.pdf /tmp/baichun-13b/Community License for Baichuan-13B Model.pdf\u001b[0m\n",
      "\u001b[34mcp s3://sagemaker-us-west-2-687912291502/llm/models/LLM_baichuan_model_13b/Baichuan-13B 模型社区许可协议.pdf /tmp/baichun-13b/Baichuan-13B 模型社区许可协议.pdf\u001b[0m\n",
      "\u001b[34mcp s3://sagemaker-us-west-2-687912291502/llm/models/LLM_baichuan_model_13b/tokenizer.model /tmp/baichun-13b/tokenizer.model\u001b[0m\n",
      "\u001b[34mcp s3://sagemaker-us-west-2-687912291502/llm/models/LLM_baichuan_model_13b/pytorch_model-00003-of-00003.bin /tmp/baichun-13b/pytorch_model-00003-of-00003.bin\u001b[0m\n",
      "\u001b[34mcp s3://sagemaker-us-west-2-687912291502/llm/models/LLM_baichuan_model_13b/pytorch_model-00001-of-00003.bin /tmp/baichun-13b/pytorch_model-00001-of-00003.bin\u001b[0m\n",
      "\u001b[34mcp s3://sagemaker-us-west-2-687912291502/llm/models/LLM_baichuan_model_13b/pytorch_model-00002-of-00003.bin /tmp/baichun-13b/pytorch_model-00002-of-00003.bin\u001b[0m\n",
      "\u001b[34mcp: cannot create regular file '/tmp/baichuan-13b/': Not a directory\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:02:44,578] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)\u001b[0m\n",
      "\u001b[34mWARNING:torch.distributed.run:\u001b[0m\n",
      "\u001b[34m*****************************************\u001b[0m\n",
      "\u001b[34mSetting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. \u001b[0m\n",
      "\u001b[34m*****************************************\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:02:49,176] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:02:49,176] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:02:49,185] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:02:49,188] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:02:49,194] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:02:49,195] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:02:49,198] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:02:49,199] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:02:53,962] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:02:53,962] [INFO] [comm.py:616:init_distributed] cdb=None\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:02:53,966] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:02:53,966] [INFO] [comm.py:616:init_distributed] cdb=None\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:02:53,995] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:02:53,995] [INFO] [comm.py:616:init_distributed] cdb=None\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:02:54,005] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:02:54,005] [INFO] [comm.py:616:init_distributed] cdb=None\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:02:54,025] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:02:54,025] [INFO] [comm.py:616:init_distributed] cdb=None\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:02:54,047] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:02:54,047] [INFO] [comm.py:616:init_distributed] cdb=None\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:02:54,048] [INFO] [comm.py:643:init_distributed] Initializing TorchBackend in DeepSpeed with backend nccl\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:02:54,051] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:02:54,051] [INFO] [comm.py:616:init_distributed] cdb=None\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:02:54,154] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:02:54,154] [INFO] [comm.py:616:init_distributed] cdb=None\u001b[0m\n",
      "\u001b[34m07/20/2023 06:02:54 - WARNING - llmtuner.tuner.core.parser - `ddp_find_unused_parameters` needs to be set as False in DDP training.\u001b[0m\n",
      "\u001b[34m07/20/2023 06:02:54 - INFO - llmtuner.tuner.core.parser - Process rank: 7, device: cuda:7, n_gpu: 1\n",
      "  distributed training: True, 16-bits training: True\u001b[0m\n",
      "\u001b[34m07/20/2023 06:02:54 - WARNING - llmtuner.tuner.core.parser - `ddp_find_unused_parameters` needs to be set as False in DDP training.\u001b[0m\n",
      "\u001b[34m07/20/2023 06:02:54 - INFO - llmtuner.tuner.core.parser - Process rank: 2, device: cuda:2, n_gpu: 1\n",
      "  distributed training: True, 16-bits training: True\u001b[0m\n",
      "\u001b[34m07/20/2023 06:02:54 - INFO - llmtuner.tuner.core.parser - Training/evaluation parameters Seq2SeqTrainingArguments(\u001b[0m\n",
      "\u001b[34m_n_gpu=1,\u001b[0m\n",
      "\u001b[34madafactor=False,\u001b[0m\n",
      "\u001b[34madam_beta1=0.9,\u001b[0m\n",
      "\u001b[34madam_beta2=0.999,\u001b[0m\n",
      "\u001b[34madam_epsilon=1e-08,\u001b[0m\n",
      "\u001b[34mauto_find_batch_size=False,\u001b[0m\n",
      "\u001b[34mbf16=False,\u001b[0m\n",
      "\u001b[34mbf16_full_eval=False,\u001b[0m\n",
      "\u001b[34mdata_seed=None,\u001b[0m\n",
      "\u001b[34mdataloader_drop_last=False,\u001b[0m\n",
      "\u001b[34mdataloader_num_workers=0,\u001b[0m\n",
      "\u001b[34mdataloader_pin_memory=True,\u001b[0m\n",
      "\u001b[34mddp_backend=None,\u001b[0m\n",
      "\u001b[34mddp_broadcast_buffers=None,\u001b[0m\n",
      "\u001b[34mddp_bucket_cap_mb=None,\u001b[0m\n",
      "\u001b[34mddp_find_unused_parameters=False,\u001b[0m\n",
      "\u001b[34mddp_timeout=1800,\u001b[0m\n",
      "\u001b[34mdebug=[],\u001b[0m\n",
      "\u001b[34mdeepspeed=None,\u001b[0m\n",
      "\u001b[34mdisable_tqdm=False,\u001b[0m\n",
      "\u001b[34mdo_eval=False,\u001b[0m\n",
      "\u001b[34mdo_predict=False,\u001b[0m\n",
      "\u001b[34mdo_train=True,\u001b[0m\n",
      "\u001b[34meval_accumulation_steps=None,\u001b[0m\n",
      "\u001b[34meval_delay=0,\u001b[0m\n",
      "\u001b[34meval_steps=None,\u001b[0m\n",
      "\u001b[34mevaluation_strategy=no,\u001b[0m\n",
      "\u001b[34mfp16=True,\u001b[0m\n",
      "\u001b[34mfp16_backend=auto,\u001b[0m\n",
      "\u001b[34mfp16_full_eval=False,\u001b[0m\n",
      "\u001b[34mfp16_opt_level=O1,\u001b[0m\n",
      "\u001b[34mfsdp=[],\u001b[0m\n",
      "\u001b[34mfsdp_config={'fsdp_min_num_params': 0, 'xla': False, 'xla_fsdp_grad_ckpt': False},\u001b[0m\n",
      "\u001b[34mfsdp_min_num_params=0,\u001b[0m\n",
      "\u001b[34mfsdp_transformer_layer_cls_to_wrap=None,\u001b[0m\n",
      "\u001b[34mfull_determinism=False,\u001b[0m\n",
      "\u001b[34mgeneration_config=None,\u001b[0m\n",
      "\u001b[34mgeneration_max_length=None,\u001b[0m\n",
      "\u001b[34mgeneration_num_beams=None,\u001b[0m\n",
      "\u001b[34mgradient_accumulation_steps=2,\u001b[0m\n",
      "\u001b[34mgradient_checkpointing=False,\u001b[0m\n",
      "\u001b[34mgreater_is_better=None,\u001b[0m\n",
      "\u001b[34mgroup_by_length=False,\u001b[0m\n",
      "\u001b[34mhalf_precision_backend=auto,\u001b[0m\n",
      "\u001b[34mhub_model_id=None,\u001b[0m\n",
      "\u001b[34mhub_private_repo=False,\u001b[0m\n",
      "\u001b[34mhub_strategy=every_save,\u001b[0m\n",
      "\u001b[34mhub_token=<HUB_TOKEN>,\u001b[0m\n",
      "\u001b[34mignore_data_skip=False,\u001b[0m\n",
      "\u001b[34minclude_inputs_for_metrics=False,\u001b[0m\n",
      "\u001b[34mjit_mode_eval=False,\u001b[0m\n",
      "\u001b[34mlabel_names=None,\u001b[0m\n",
      "\u001b[34mlabel_smoothing_factor=0.0,\u001b[0m\n",
      "\u001b[34mlearning_rate=0.0003,\u001b[0m\n",
      "\u001b[34mlength_column_name=length,\u001b[0m\n",
      "\u001b[34mload_best_model_at_end=False,\u001b[0m\n",
      "\u001b[34mlocal_rank=7,\u001b[0m\n",
      "\u001b[34mlog_level=passive,\u001b[0m\n",
      "\u001b[34mlog_level_replica=warning,\u001b[0m\n",
      "\u001b[34mlog_on_each_node=True,\u001b[0m\n",
      "\u001b[34mlogging_dir=/tmp/ouput/runs/Jul20_06-02-54_algo-1,\u001b[0m\n",
      "\u001b[34mlogging_first_step=False,\u001b[0m\n",
      "\u001b[34mlogging_nan_inf_filter=True,\u001b[0m\n",
      "\u001b[34mlogging_steps=10,\u001b[0m\n",
      "\u001b[34mlogging_strategy=steps,\u001b[0m\n",
      "\u001b[34mlr_scheduler_type=cosine,\u001b[0m\n",
      "\u001b[34mmax_grad_norm=1.0,\u001b[0m\n",
      "\u001b[34mmax_steps=-1,\u001b[0m\n",
      "\u001b[34mmetric_for_best_model=None,\u001b[0m\n",
      "\u001b[34mmp_parameters=,\u001b[0m\n",
      "\u001b[34mno_cuda=False,\u001b[0m\n",
      "\u001b[34mnum_train_epochs=2.0,\u001b[0m\n",
      "\u001b[34moptim=adamw_torch,\u001b[0m\n",
      "\u001b[34moptim_args=None,\u001b[0m\n",
      "\u001b[34moutput_dir=/tmp/ouput/,\u001b[0m\n",
      "\u001b[34moverwrite_output_dir=False,\u001b[0m\n",
      "\u001b[34mpast_index=-1,\u001b[0m\n",
      "\u001b[34mper_device_eval_batch_size=8,\u001b[0m\n",
      "\u001b[34mper_device_train_batch_size=4,\u001b[0m\n",
      "\u001b[34mpredict_with_generate=False,\u001b[0m\n",
      "\u001b[34mprediction_loss_only=False,\u001b[0m\n",
      "\u001b[34mpush_to_hub=False,\u001b[0m\n",
      "\u001b[34mpush_to_hub_model_id=None,\u001b[0m\n",
      "\u001b[34mpush_to_hub_organization=None,\u001b[0m\n",
      "\u001b[34mpush_to_hub_token=<PUSH_TO_HUB_TOKEN>,\u001b[0m\n",
      "\u001b[34mray_scope=last,\u001b[0m\n",
      "\u001b[34mremove_unused_columns=True,\u001b[0m\n",
      "\u001b[34mreport_to=['wandb'],\u001b[0m\n",
      "\u001b[34mresume_from_checkpoint=None,\u001b[0m\n",
      "\u001b[34mrun_name=/tmp/ouput/,\u001b[0m\n",
      "\u001b[34msave_on_each_node=False,\u001b[0m\n",
      "\u001b[34msave_safetensors=False,\u001b[0m\n",
      "\u001b[34msave_steps=100,\u001b[0m\n",
      "\u001b[34msave_strategy=steps,\u001b[0m\n",
      "\u001b[34msave_total_limit=None,\u001b[0m\n",
      "\u001b[34mseed=42,\u001b[0m\n",
      "\u001b[34msharded_ddp=[],\u001b[0m\n",
      "\u001b[34mskip_memory_metrics=True,\u001b[0m\n",
      "\u001b[34msortish_sampler=False,\u001b[0m\n",
      "\u001b[34mtf32=None,\u001b[0m\n",
      "\u001b[34mtorch_compile=False,\u001b[0m\n",
      "\u001b[34mtorch_compile_backend=None,\u001b[0m\n",
      "\u001b[34mtorch_compile_mode=None,\u001b[0m\n",
      "\u001b[34mtorchdynamo=None,\u001b[0m\n",
      "\u001b[34mtpu_metrics_debug=False,\u001b[0m\n",
      "\u001b[34mtpu_num_cores=None,\u001b[0m\n",
      "\u001b[34muse_ipex=False,\u001b[0m\n",
      "\u001b[34muse_legacy_prediction_loop=False,\u001b[0m\n",
      "\u001b[34muse_mps_device=False,\u001b[0m\n",
      "\u001b[34mwarmup_ratio=0.0,\u001b[0m\n",
      "\u001b[34mwarmup_steps=0,\u001b[0m\n",
      "\u001b[34mweight_decay=0.0,\u001b[0m\n",
      "\u001b[34mxpu_backend=None,\u001b[0m\n",
      "\u001b[34m)\u001b[0m\n",
      "\u001b[34m07/20/2023 06:02:54 - INFO - llmtuner.dsets.loader - Loading dataset obsfs/corpus/zhetian_sft_instruction.json...\u001b[0m\n",
      "\u001b[34m07/20/2023 06:02:54 - WARNING - llmtuner.dsets.loader - Checksum failed: missing SHA-1 hash value in dataset_info.json or too many files.\u001b[0m\n",
      "\u001b[34m07/20/2023 06:02:54 - INFO - llmtuner.tuner.core.parser - Training/evaluation parameters Seq2SeqTrainingArguments(\u001b[0m\n",
      "\u001b[34m_n_gpu=1,\u001b[0m\n",
      "\u001b[34madafactor=False,\u001b[0m\n",
      "\u001b[34madam_beta1=0.9,\u001b[0m\n",
      "\u001b[34madam_beta2=0.999,\u001b[0m\n",
      "\u001b[34madam_epsilon=1e-08,\u001b[0m\n",
      "\u001b[34mauto_find_batch_size=False,\u001b[0m\n",
      "\u001b[34mbf16=False,\u001b[0m\n",
      "\u001b[34mbf16_full_eval=False,\u001b[0m\n",
      "\u001b[34mdata_seed=None,\u001b[0m\n",
      "\u001b[34mdataloader_drop_last=False,\u001b[0m\n",
      "\u001b[34mdataloader_num_workers=0,\u001b[0m\n",
      "\u001b[34mdataloader_pin_memory=True,\u001b[0m\n",
      "\u001b[34mddp_backend=None,\u001b[0m\n",
      "\u001b[34mddp_broadcast_buffers=None,\u001b[0m\n",
      "\u001b[34mddp_bucket_cap_mb=None,\u001b[0m\n",
      "\u001b[34mddp_find_unused_parameters=False,\u001b[0m\n",
      "\u001b[34mddp_timeout=1800,\u001b[0m\n",
      "\u001b[34mdebug=[],\u001b[0m\n",
      "\u001b[34mdeepspeed=None,\u001b[0m\n",
      "\u001b[34mdisable_tqdm=False,\u001b[0m\n",
      "\u001b[34mdo_eval=False,\u001b[0m\n",
      "\u001b[34mdo_predict=False,\u001b[0m\n",
      "\u001b[34mdo_train=True,\u001b[0m\n",
      "\u001b[34meval_accumulation_steps=None,\u001b[0m\n",
      "\u001b[34meval_delay=0,\u001b[0m\n",
      "\u001b[34meval_steps=None,\u001b[0m\n",
      "\u001b[34mevaluation_strategy=no,\u001b[0m\n",
      "\u001b[34mfp16=True,\u001b[0m\n",
      "\u001b[34mfp16_backend=auto,\u001b[0m\n",
      "\u001b[34mfp16_full_eval=False,\u001b[0m\n",
      "\u001b[34mfp16_opt_level=O1,\u001b[0m\n",
      "\u001b[34mfsdp=[],\u001b[0m\n",
      "\u001b[34mfsdp_config={'fsdp_min_num_params': 0, 'xla': False, 'xla_fsdp_grad_ckpt': False},\u001b[0m\n",
      "\u001b[34mfsdp_min_num_params=0,\u001b[0m\n",
      "\u001b[34mfsdp_transformer_layer_cls_to_wrap=None,\u001b[0m\n",
      "\u001b[34mfull_determinism=False,\u001b[0m\n",
      "\u001b[34mgeneration_config=None,\u001b[0m\n",
      "\u001b[34mgeneration_max_length=None,\u001b[0m\n",
      "\u001b[34mgeneration_num_beams=None,\u001b[0m\n",
      "\u001b[34mgradient_accumulation_steps=2,\u001b[0m\n",
      "\u001b[34mgradient_checkpointing=False,\u001b[0m\n",
      "\u001b[34mgreater_is_better=None,\u001b[0m\n",
      "\u001b[34mgroup_by_length=False,\u001b[0m\n",
      "\u001b[34mhalf_precision_backend=auto,\u001b[0m\n",
      "\u001b[34mhub_model_id=None,\u001b[0m\n",
      "\u001b[34mhub_private_repo=False,\u001b[0m\n",
      "\u001b[34mhub_strategy=every_save,\u001b[0m\n",
      "\u001b[34mhub_token=<HUB_TOKEN>,\u001b[0m\n",
      "\u001b[34mignore_data_skip=False,\u001b[0m\n",
      "\u001b[34minclude_inputs_for_metrics=False,\u001b[0m\n",
      "\u001b[34mjit_mode_eval=False,\u001b[0m\n",
      "\u001b[34mlabel_names=None,\u001b[0m\n",
      "\u001b[34mlabel_smoothing_factor=0.0,\u001b[0m\n",
      "\u001b[34mlearning_rate=0.0003,\u001b[0m\n",
      "\u001b[34mlength_column_name=length,\u001b[0m\n",
      "\u001b[34mload_best_model_at_end=False,\u001b[0m\n",
      "\u001b[34mlocal_rank=2,\u001b[0m\n",
      "\u001b[34mlog_level=passive,\u001b[0m\n",
      "\u001b[34mlog_level_replica=warning,\u001b[0m\n",
      "\u001b[34mlog_on_each_node=True,\u001b[0m\n",
      "\u001b[34mlogging_dir=/tmp/ouput/runs/Jul20_06-02-54_algo-1,\u001b[0m\n",
      "\u001b[34mlogging_first_step=False,\u001b[0m\n",
      "\u001b[34mlogging_nan_inf_filter=True,\u001b[0m\n",
      "\u001b[34mlogging_steps=10,\u001b[0m\n",
      "\u001b[34mlogging_strategy=steps,\u001b[0m\n",
      "\u001b[34mlr_scheduler_type=cosine,\u001b[0m\n",
      "\u001b[34mmax_grad_norm=1.0,\u001b[0m\n",
      "\u001b[34mmax_steps=-1,\u001b[0m\n",
      "\u001b[34mmetric_for_best_model=None,\u001b[0m\n",
      "\u001b[34mmp_parameters=,\u001b[0m\n",
      "\u001b[34mno_cuda=False,\u001b[0m\n",
      "\u001b[34mnum_train_epochs=2.0,\u001b[0m\n",
      "\u001b[34moptim=adamw_torch,\u001b[0m\n",
      "\u001b[34moptim_args=None,\u001b[0m\n",
      "\u001b[34moutput_dir=/tmp/ouput/,\u001b[0m\n",
      "\u001b[34moverwrite_output_dir=False,\u001b[0m\n",
      "\u001b[34mpast_index=-1,\u001b[0m\n",
      "\u001b[34mper_device_eval_batch_size=8,\u001b[0m\n",
      "\u001b[34mper_device_train_batch_size=4,\u001b[0m\n",
      "\u001b[34mpredict_with_generate=False,\u001b[0m\n",
      "\u001b[34mprediction_loss_only=False,\u001b[0m\n",
      "\u001b[34mpush_to_hub=False,\u001b[0m\n",
      "\u001b[34mpush_to_hub_model_id=None,\u001b[0m\n",
      "\u001b[34mpush_to_hub_organization=None,\u001b[0m\n",
      "\u001b[34mpush_to_hub_token=<PUSH_TO_HUB_TOKEN>,\u001b[0m\n",
      "\u001b[34mray_scope=last,\u001b[0m\n",
      "\u001b[34mremove_unused_columns=True,\u001b[0m\n",
      "\u001b[34mreport_to=['wandb'],\u001b[0m\n",
      "\u001b[34mresume_from_checkpoint=None,\u001b[0m\n",
      "\u001b[34mrun_name=/tmp/ouput/,\u001b[0m\n",
      "\u001b[34msave_on_each_node=False,\u001b[0m\n",
      "\u001b[34msave_safetensors=False,\u001b[0m\n",
      "\u001b[34msave_steps=100,\u001b[0m\n",
      "\u001b[34msave_strategy=steps,\u001b[0m\n",
      "\u001b[34msave_total_limit=None,\u001b[0m\n",
      "\u001b[34mseed=42,\u001b[0m\n",
      "\u001b[34msharded_ddp=[],\u001b[0m\n",
      "\u001b[34mskip_memory_metrics=True,\u001b[0m\n",
      "\u001b[34msortish_sampler=False,\u001b[0m\n",
      "\u001b[34mtf32=None,\u001b[0m\n",
      "\u001b[34mtorch_compile=False,\u001b[0m\n",
      "\u001b[34mtorch_compile_backend=None,\u001b[0m\n",
      "\u001b[34mtorch_compile_mode=None,\u001b[0m\n",
      "\u001b[34mtorchdynamo=None,\u001b[0m\n",
      "\u001b[34mtpu_metrics_debug=False,\u001b[0m\n",
      "\u001b[34mtpu_num_cores=None,\u001b[0m\n",
      "\u001b[34muse_ipex=False,\u001b[0m\n",
      "\u001b[34muse_legacy_prediction_loop=False,\u001b[0m\n",
      "\u001b[34muse_mps_device=False,\u001b[0m\n",
      "\u001b[34mwarmup_ratio=0.0,\u001b[0m\n",
      "\u001b[34mwarmup_steps=0,\u001b[0m\n",
      "\u001b[34mweight_decay=0.0,\u001b[0m\n",
      "\u001b[34mxpu_backend=None,\u001b[0m\n",
      "\u001b[34m)\u001b[0m\n",
      "\u001b[34m07/20/2023 06:02:54 - WARNING - llmtuner.tuner.core.parser - `ddp_find_unused_parameters` needs to be set as False in DDP training.\u001b[0m\n",
      "\u001b[34m07/20/2023 06:02:54 - INFO - llmtuner.tuner.core.parser - Process rank: 5, device: cuda:5, n_gpu: 1\n",
      "  distributed training: True, 16-bits training: True\u001b[0m\n",
      "\u001b[34m07/20/2023 06:02:54 - INFO - llmtuner.dsets.loader - Loading dataset obsfs/corpus/zhetian_sft_instruction.json...\u001b[0m\n",
      "\u001b[34m07/20/2023 06:02:54 - WARNING - llmtuner.dsets.loader - Checksum failed: missing SHA-1 hash value in dataset_info.json or too many files.\u001b[0m\n",
      "\u001b[34m07/20/2023 06:02:54 - WARNING - llmtuner.tuner.core.parser - `ddp_find_unused_parameters` needs to be set as False in DDP training.\u001b[0m\n",
      "\u001b[34m07/20/2023 06:02:54 - INFO - llmtuner.tuner.core.parser - Process rank: 1, device: cuda:1, n_gpu: 1\n",
      "  distributed training: True, 16-bits training: True\u001b[0m\n",
      "\u001b[34m07/20/2023 06:02:54 - INFO - llmtuner.tuner.core.parser - Training/evaluation parameters Seq2SeqTrainingArguments(\u001b[0m\n",
      "\u001b[34m_n_gpu=1,\u001b[0m\n",
      "\u001b[34madafactor=False,\u001b[0m\n",
      "\u001b[34madam_beta1=0.9,\u001b[0m\n",
      "\u001b[34madam_beta2=0.999,\u001b[0m\n",
      "\u001b[34madam_epsilon=1e-08,\u001b[0m\n",
      "\u001b[34mauto_find_batch_size=False,\u001b[0m\n",
      "\u001b[34mbf16=False,\u001b[0m\n",
      "\u001b[34mbf16_full_eval=False,\u001b[0m\n",
      "\u001b[34mdata_seed=None,\u001b[0m\n",
      "\u001b[34mdataloader_drop_last=False,\u001b[0m\n",
      "\u001b[34mdataloader_num_workers=0,\u001b[0m\n",
      "\u001b[34mdataloader_pin_memory=True,\u001b[0m\n",
      "\u001b[34mddp_backend=None,\u001b[0m\n",
      "\u001b[34mddp_broadcast_buffers=None,\u001b[0m\n",
      "\u001b[34mddp_bucket_cap_mb=None,\u001b[0m\n",
      "\u001b[34mddp_find_unused_parameters=False,\u001b[0m\n",
      "\u001b[34mddp_timeout=1800,\u001b[0m\n",
      "\u001b[34mdebug=[],\u001b[0m\n",
      "\u001b[34mdeepspeed=None,\u001b[0m\n",
      "\u001b[34mdisable_tqdm=False,\u001b[0m\n",
      "\u001b[34mdo_eval=False,\u001b[0m\n",
      "\u001b[34mdo_predict=False,\u001b[0m\n",
      "\u001b[34mdo_train=True,\u001b[0m\n",
      "\u001b[34meval_accumulation_steps=None,\u001b[0m\n",
      "\u001b[34meval_delay=0,\u001b[0m\n",
      "\u001b[34meval_steps=None,\u001b[0m\n",
      "\u001b[34mevaluation_strategy=no,\u001b[0m\n",
      "\u001b[34mfp16=True,\u001b[0m\n",
      "\u001b[34mfp16_backend=auto,\u001b[0m\n",
      "\u001b[34mfp16_full_eval=False,\u001b[0m\n",
      "\u001b[34mfp16_opt_level=O1,\u001b[0m\n",
      "\u001b[34mfsdp=[],\u001b[0m\n",
      "\u001b[34mfsdp_config={'fsdp_min_num_params': 0, 'xla': False, 'xla_fsdp_grad_ckpt': False},\u001b[0m\n",
      "\u001b[34mfsdp_min_num_params=0,\u001b[0m\n",
      "\u001b[34mfsdp_transformer_layer_cls_to_wrap=None,\u001b[0m\n",
      "\u001b[34mfull_determinism=False,\u001b[0m\n",
      "\u001b[34mgeneration_config=None,\u001b[0m\n",
      "\u001b[34mgeneration_max_length=None,\u001b[0m\n",
      "\u001b[34mgeneration_num_beams=None,\u001b[0m\n",
      "\u001b[34mgradient_accumulation_steps=2,\u001b[0m\n",
      "\u001b[34mgradient_checkpointing=False,\u001b[0m\n",
      "\u001b[34mgreater_is_better=None,\u001b[0m\n",
      "\u001b[34mgroup_by_length=False,\u001b[0m\n",
      "\u001b[34mhalf_precision_backend=auto,\u001b[0m\n",
      "\u001b[34mhub_model_id=None,\u001b[0m\n",
      "\u001b[34mhub_private_repo=False,\u001b[0m\n",
      "\u001b[34mhub_strategy=every_save,\u001b[0m\n",
      "\u001b[34mhub_token=<HUB_TOKEN>,\u001b[0m\n",
      "\u001b[34mignore_data_skip=False,\u001b[0m\n",
      "\u001b[34minclude_inputs_for_metrics=False,\u001b[0m\n",
      "\u001b[34mjit_mode_eval=False,\u001b[0m\n",
      "\u001b[34mlabel_names=None,\u001b[0m\n",
      "\u001b[34mlabel_smoothing_factor=0.0,\u001b[0m\n",
      "\u001b[34mlearning_rate=0.0003,\u001b[0m\n",
      "\u001b[34mlength_column_name=length,\u001b[0m\n",
      "\u001b[34mload_best_model_at_end=False,\u001b[0m\n",
      "\u001b[34mlocal_rank=5,\u001b[0m\n",
      "\u001b[34mlog_level=passive,\u001b[0m\n",
      "\u001b[34mlog_level_replica=warning,\u001b[0m\n",
      "\u001b[34mlog_on_each_node=True,\u001b[0m\n",
      "\u001b[34mlogging_dir=/tmp/ouput/runs/Jul20_06-02-53_algo-1,\u001b[0m\n",
      "\u001b[34mlogging_first_step=False,\u001b[0m\n",
      "\u001b[34mlogging_nan_inf_filter=True,\u001b[0m\n",
      "\u001b[34mlogging_steps=10,\u001b[0m\n",
      "\u001b[34mlogging_strategy=steps,\u001b[0m\n",
      "\u001b[34mlr_scheduler_type=cosine,\u001b[0m\n",
      "\u001b[34mmax_grad_norm=1.0,\u001b[0m\n",
      "\u001b[34mmax_steps=-1,\u001b[0m\n",
      "\u001b[34mmetric_for_best_model=None,\u001b[0m\n",
      "\u001b[34mmp_parameters=,\u001b[0m\n",
      "\u001b[34mno_cuda=False,\u001b[0m\n",
      "\u001b[34mnum_train_epochs=2.0,\u001b[0m\n",
      "\u001b[34moptim=adamw_torch,\u001b[0m\n",
      "\u001b[34moptim_args=None,\u001b[0m\n",
      "\u001b[34moutput_dir=/tmp/ouput/,\u001b[0m\n",
      "\u001b[34moverwrite_output_dir=False,\u001b[0m\n",
      "\u001b[34mpast_index=-1,\u001b[0m\n",
      "\u001b[34mper_device_eval_batch_size=8,\u001b[0m\n",
      "\u001b[34mper_device_train_batch_size=4,\u001b[0m\n",
      "\u001b[34mpredict_with_generate=False,\u001b[0m\n",
      "\u001b[34mprediction_loss_only=False,\u001b[0m\n",
      "\u001b[34mpush_to_hub=False,\u001b[0m\n",
      "\u001b[34mpush_to_hub_model_id=None,\u001b[0m\n",
      "\u001b[34mpush_to_hub_organization=None,\u001b[0m\n",
      "\u001b[34mpush_to_hub_token=<PUSH_TO_HUB_TOKEN>,\u001b[0m\n",
      "\u001b[34mray_scope=last,\u001b[0m\n",
      "\u001b[34mremove_unused_columns=True,\u001b[0m\n",
      "\u001b[34mreport_to=['wandb'],\u001b[0m\n",
      "\u001b[34mresume_from_checkpoint=None,\u001b[0m\n",
      "\u001b[34mrun_name=/tmp/ouput/,\u001b[0m\n",
      "\u001b[34msave_on_each_node=False,\u001b[0m\n",
      "\u001b[34msave_safetensors=False,\u001b[0m\n",
      "\u001b[34msave_steps=100,\u001b[0m\n",
      "\u001b[34msave_strategy=steps,\u001b[0m\n",
      "\u001b[34msave_total_limit=None,\u001b[0m\n",
      "\u001b[34mseed=42,\u001b[0m\n",
      "\u001b[34msharded_ddp=[],\u001b[0m\n",
      "\u001b[34mskip_memory_metrics=True,\u001b[0m\n",
      "\u001b[34msortish_sampler=False,\u001b[0m\n",
      "\u001b[34mtf32=None,\u001b[0m\n",
      "\u001b[34mtorch_compile=False,\u001b[0m\n",
      "\u001b[34mtorch_compile_backend=None,\u001b[0m\n",
      "\u001b[34mtorch_compile_mode=None,\u001b[0m\n",
      "\u001b[34mtorchdynamo=None,\u001b[0m\n",
      "\u001b[34mtpu_metrics_debug=False,\u001b[0m\n",
      "\u001b[34mtpu_num_cores=None,\u001b[0m\n",
      "\u001b[34muse_ipex=False,\u001b[0m\n",
      "\u001b[34muse_legacy_prediction_loop=False,\u001b[0m\n",
      "\u001b[34muse_mps_device=False,\u001b[0m\n",
      "\u001b[34mwarmup_ratio=0.0,\u001b[0m\n",
      "\u001b[34mwarmup_steps=0,\u001b[0m\n",
      "\u001b[34mweight_decay=0.0,\u001b[0m\n",
      "\u001b[34mxpu_backend=None,\u001b[0m\n",
      "\u001b[34m)\u001b[0m\n",
      "\u001b[34m07/20/2023 06:02:54 - INFO - llmtuner.dsets.loader - Loading dataset obsfs/corpus/zhetian_sft_instruction.json...\u001b[0m\n",
      "\u001b[34m07/20/2023 06:02:54 - WARNING - llmtuner.dsets.loader - Checksum failed: missing SHA-1 hash value in dataset_info.json or too many files.\u001b[0m\n",
      "\u001b[34m07/20/2023 06:02:54 - INFO - llmtuner.tuner.core.parser - Training/evaluation parameters Seq2SeqTrainingArguments(\u001b[0m\n",
      "\u001b[34m_n_gpu=1,\u001b[0m\n",
      "\u001b[34madafactor=False,\u001b[0m\n",
      "\u001b[34madam_beta1=0.9,\u001b[0m\n",
      "\u001b[34madam_beta2=0.999,\u001b[0m\n",
      "\u001b[34madam_epsilon=1e-08,\u001b[0m\n",
      "\u001b[34mauto_find_batch_size=False,\u001b[0m\n",
      "\u001b[34mbf16=False,\u001b[0m\n",
      "\u001b[34mbf16_full_eval=False,\u001b[0m\n",
      "\u001b[34mdata_seed=None,\u001b[0m\n",
      "\u001b[34mdataloader_drop_last=False,\u001b[0m\n",
      "\u001b[34mdataloader_num_workers=0,\u001b[0m\n",
      "\u001b[34mdataloader_pin_memory=True,\u001b[0m\n",
      "\u001b[34mddp_backend=None,\u001b[0m\n",
      "\u001b[34mddp_broadcast_buffers=None,\u001b[0m\n",
      "\u001b[34mddp_bucket_cap_mb=None,\u001b[0m\n",
      "\u001b[34mddp_find_unused_parameters=False,\u001b[0m\n",
      "\u001b[34mddp_timeout=1800,\u001b[0m\n",
      "\u001b[34mdebug=[],\u001b[0m\n",
      "\u001b[34mdeepspeed=None,\u001b[0m\n",
      "\u001b[34mdisable_tqdm=False,\u001b[0m\n",
      "\u001b[34mdo_eval=False,\u001b[0m\n",
      "\u001b[34mdo_predict=False,\u001b[0m\n",
      "\u001b[34mdo_train=True,\u001b[0m\n",
      "\u001b[34meval_accumulation_steps=None,\u001b[0m\n",
      "\u001b[34meval_delay=0,\u001b[0m\n",
      "\u001b[34meval_steps=None,\u001b[0m\n",
      "\u001b[34mevaluation_strategy=no,\u001b[0m\n",
      "\u001b[34mfp16=True,\u001b[0m\n",
      "\u001b[34mfp16_backend=auto,\u001b[0m\n",
      "\u001b[34mfp16_full_eval=False,\u001b[0m\n",
      "\u001b[34mfp16_opt_level=O1,\u001b[0m\n",
      "\u001b[34mfsdp=[],\u001b[0m\n",
      "\u001b[34mfsdp_config={'fsdp_min_num_params': 0, 'xla': False, 'xla_fsdp_grad_ckpt': False},\u001b[0m\n",
      "\u001b[34mfsdp_min_num_params=0,\u001b[0m\n",
      "\u001b[34mfsdp_transformer_layer_cls_to_wrap=None,\u001b[0m\n",
      "\u001b[34mfull_determinism=False,\u001b[0m\n",
      "\u001b[34mgeneration_config=None,\u001b[0m\n",
      "\u001b[34mgeneration_max_length=None,\u001b[0m\n",
      "\u001b[34mgeneration_num_beams=None,\u001b[0m\n",
      "\u001b[34mgradient_accumulation_steps=2,\u001b[0m\n",
      "\u001b[34mgradient_checkpointing=False,\u001b[0m\n",
      "\u001b[34mgreater_is_better=None,\u001b[0m\n",
      "\u001b[34mgroup_by_length=False,\u001b[0m\n",
      "\u001b[34mhalf_precision_backend=auto,\u001b[0m\n",
      "\u001b[34mhub_model_id=None,\u001b[0m\n",
      "\u001b[34mhub_private_repo=False,\u001b[0m\n",
      "\u001b[34mhub_strategy=every_save,\u001b[0m\n",
      "\u001b[34mhub_token=<HUB_TOKEN>,\u001b[0m\n",
      "\u001b[34mignore_data_skip=False,\u001b[0m\n",
      "\u001b[34minclude_inputs_for_metrics=False,\u001b[0m\n",
      "\u001b[34mjit_mode_eval=False,\u001b[0m\n",
      "\u001b[34mlabel_names=None,\u001b[0m\n",
      "\u001b[34mlabel_smoothing_factor=0.0,\u001b[0m\n",
      "\u001b[34mlearning_rate=0.0003,\u001b[0m\n",
      "\u001b[34mlength_column_name=length,\u001b[0m\n",
      "\u001b[34mload_best_model_at_end=False,\u001b[0m\n",
      "\u001b[34mlocal_rank=1,\u001b[0m\n",
      "\u001b[34mlog_level=passive,\u001b[0m\n",
      "\u001b[34mlog_level_replica=warning,\u001b[0m\n",
      "\u001b[34mlog_on_each_node=True,\u001b[0m\n",
      "\u001b[34mlogging_dir=/tmp/ouput/runs/Jul20_06-02-54_algo-1,\u001b[0m\n",
      "\u001b[34mlogging_first_step=False,\u001b[0m\n",
      "\u001b[34mlogging_nan_inf_filter=True,\u001b[0m\n",
      "\u001b[34mlogging_steps=10,\u001b[0m\n",
      "\u001b[34mlogging_strategy=steps,\u001b[0m\n",
      "\u001b[34mlr_scheduler_type=cosine,\u001b[0m\n",
      "\u001b[34mmax_grad_norm=1.0,\u001b[0m\n",
      "\u001b[34mmax_steps=-1,\u001b[0m\n",
      "\u001b[34mmetric_for_best_model=None,\u001b[0m\n",
      "\u001b[34mmp_parameters=,\u001b[0m\n",
      "\u001b[34mno_cuda=False,\u001b[0m\n",
      "\u001b[34mnum_train_epochs=2.0,\u001b[0m\n",
      "\u001b[34moptim=adamw_torch,\u001b[0m\n",
      "\u001b[34moptim_args=None,\u001b[0m\n",
      "\u001b[34moutput_dir=/tmp/ouput/,\u001b[0m\n",
      "\u001b[34moverwrite_output_dir=False,\u001b[0m\n",
      "\u001b[34mpast_index=-1,\u001b[0m\n",
      "\u001b[34mper_device_eval_batch_size=8,\u001b[0m\n",
      "\u001b[34mper_device_train_batch_size=4,\u001b[0m\n",
      "\u001b[34mpredict_with_generate=False,\u001b[0m\n",
      "\u001b[34mprediction_loss_only=False,\u001b[0m\n",
      "\u001b[34mpush_to_hub=False,\u001b[0m\n",
      "\u001b[34mpush_to_hub_model_id=None,\u001b[0m\n",
      "\u001b[34mpush_to_hub_organization=None,\u001b[0m\n",
      "\u001b[34mpush_to_hub_token=<PUSH_TO_HUB_TOKEN>,\u001b[0m\n",
      "\u001b[34mray_scope=last,\u001b[0m\n",
      "\u001b[34mremove_unused_columns=True,\u001b[0m\n",
      "\u001b[34mreport_to=['wandb'],\u001b[0m\n",
      "\u001b[34mresume_from_checkpoint=None,\u001b[0m\n",
      "\u001b[34mrun_name=/tmp/ouput/,\u001b[0m\n",
      "\u001b[34msave_on_each_node=False,\u001b[0m\n",
      "\u001b[34msave_safetensors=False,\u001b[0m\n",
      "\u001b[34msave_steps=100,\u001b[0m\n",
      "\u001b[34msave_strategy=steps,\u001b[0m\n",
      "\u001b[34msave_total_limit=None,\u001b[0m\n",
      "\u001b[34mseed=42,\u001b[0m\n",
      "\u001b[34msharded_ddp=[],\u001b[0m\n",
      "\u001b[34mskip_memory_metrics=True,\u001b[0m\n",
      "\u001b[34msortish_sampler=False,\u001b[0m\n",
      "\u001b[34mtf32=None,\u001b[0m\n",
      "\u001b[34mtorch_compile=False,\u001b[0m\n",
      "\u001b[34mtorch_compile_backend=None,\u001b[0m\n",
      "\u001b[34mtorch_compile_mode=None,\u001b[0m\n",
      "\u001b[34mtorchdynamo=None,\u001b[0m\n",
      "\u001b[34mtpu_metrics_debug=False,\u001b[0m\n",
      "\u001b[34mtpu_num_cores=None,\u001b[0m\n",
      "\u001b[34muse_ipex=False,\u001b[0m\n",
      "\u001b[34muse_legacy_prediction_loop=False,\u001b[0m\n",
      "\u001b[34muse_mps_device=False,\u001b[0m\n",
      "\u001b[34mwarmup_ratio=0.0,\u001b[0m\n",
      "\u001b[34mwarmup_steps=0,\u001b[0m\n",
      "\u001b[34mweight_decay=0.0,\u001b[0m\n",
      "\u001b[34mxpu_backend=None,\u001b[0m\n",
      "\u001b[34m)\u001b[0m\n",
      "\u001b[34m07/20/2023 06:02:54 - INFO - llmtuner.dsets.loader - Loading dataset obsfs/corpus/zhetian_sft_instruction.json...\u001b[0m\n",
      "\u001b[34m07/20/2023 06:02:54 - WARNING - llmtuner.dsets.loader - Checksum failed: missing SHA-1 hash value in dataset_info.json or too many files.\u001b[0m\n",
      "\u001b[34m07/20/2023 06:02:54 - WARNING - llmtuner.tuner.core.parser - `ddp_find_unused_parameters` needs to be set as False in DDP training.\u001b[0m\n",
      "\u001b[34m07/20/2023 06:02:54 - INFO - llmtuner.tuner.core.parser - Process rank: 3, device: cuda:3, n_gpu: 1\n",
      "  distributed training: True, 16-bits training: True\u001b[0m\n",
      "\u001b[34m07/20/2023 06:02:54 - INFO - llmtuner.tuner.core.parser - Training/evaluation parameters Seq2SeqTrainingArguments(\u001b[0m\n",
      "\u001b[34m_n_gpu=1,\u001b[0m\n",
      "\u001b[34madafactor=False,\u001b[0m\n",
      "\u001b[34madam_beta1=0.9,\u001b[0m\n",
      "\u001b[34madam_beta2=0.999,\u001b[0m\n",
      "\u001b[34madam_epsilon=1e-08,\u001b[0m\n",
      "\u001b[34mauto_find_batch_size=False,\u001b[0m\n",
      "\u001b[34mbf16=False,\u001b[0m\n",
      "\u001b[34mbf16_full_eval=False,\u001b[0m\n",
      "\u001b[34mdata_seed=None,\u001b[0m\n",
      "\u001b[34mdataloader_drop_last=False,\u001b[0m\n",
      "\u001b[34mdataloader_num_workers=0,\u001b[0m\n",
      "\u001b[34mdataloader_pin_memory=True,\u001b[0m\n",
      "\u001b[34mddp_backend=None,\u001b[0m\n",
      "\u001b[34mddp_broadcast_buffers=None,\u001b[0m\n",
      "\u001b[34mddp_bucket_cap_mb=None,\u001b[0m\n",
      "\u001b[34mddp_find_unused_parameters=False,\u001b[0m\n",
      "\u001b[34mddp_timeout=1800,\u001b[0m\n",
      "\u001b[34mdebug=[],\u001b[0m\n",
      "\u001b[34mdeepspeed=None,\u001b[0m\n",
      "\u001b[34mdisable_tqdm=False,\u001b[0m\n",
      "\u001b[34mdo_eval=False,\u001b[0m\n",
      "\u001b[34mdo_predict=False,\u001b[0m\n",
      "\u001b[34mdo_train=True,\u001b[0m\n",
      "\u001b[34meval_accumulation_steps=None,\u001b[0m\n",
      "\u001b[34meval_delay=0,\u001b[0m\n",
      "\u001b[34meval_steps=None,\u001b[0m\n",
      "\u001b[34mevaluation_strategy=no,\u001b[0m\n",
      "\u001b[34mfp16=True,\u001b[0m\n",
      "\u001b[34mfp16_backend=auto,\u001b[0m\n",
      "\u001b[34mfp16_full_eval=False,\u001b[0m\n",
      "\u001b[34mfp16_opt_level=O1,\u001b[0m\n",
      "\u001b[34mfsdp=[],\u001b[0m\n",
      "\u001b[34mfsdp_config={'fsdp_min_num_params': 0, 'xla': False, 'xla_fsdp_grad_ckpt': False},\u001b[0m\n",
      "\u001b[34mfsdp_min_num_params=0,\u001b[0m\n",
      "\u001b[34mfsdp_transformer_layer_cls_to_wrap=None,\u001b[0m\n",
      "\u001b[34mfull_determinism=False,\u001b[0m\n",
      "\u001b[34mgeneration_config=None,\u001b[0m\n",
      "\u001b[34mgeneration_max_length=None,\u001b[0m\n",
      "\u001b[34mgeneration_num_beams=None,\u001b[0m\n",
      "\u001b[34mgradient_accumulation_steps=2,\u001b[0m\n",
      "\u001b[34mgradient_checkpointing=False,\u001b[0m\n",
      "\u001b[34mgreater_is_better=None,\u001b[0m\n",
      "\u001b[34mgroup_by_length=False,\u001b[0m\n",
      "\u001b[34mhalf_precision_backend=auto,\u001b[0m\n",
      "\u001b[34mhub_model_id=None,\u001b[0m\n",
      "\u001b[34mhub_private_repo=False,\u001b[0m\n",
      "\u001b[34mhub_strategy=every_save,\u001b[0m\n",
      "\u001b[34mhub_token=<HUB_TOKEN>,\u001b[0m\n",
      "\u001b[34mignore_data_skip=False,\u001b[0m\n",
      "\u001b[34minclude_inputs_for_metrics=False,\u001b[0m\n",
      "\u001b[34mjit_mode_eval=False,\u001b[0m\n",
      "\u001b[34mlabel_names=None,\u001b[0m\n",
      "\u001b[34mlabel_smoothing_factor=0.0,\u001b[0m\n",
      "\u001b[34mlearning_rate=0.0003,\u001b[0m\n",
      "\u001b[34mlength_column_name=length,\u001b[0m\n",
      "\u001b[34mload_best_model_at_end=False,\u001b[0m\n",
      "\u001b[34mlocal_rank=3,\u001b[0m\n",
      "\u001b[34mlog_level=passive,\u001b[0m\n",
      "\u001b[34mlog_level_replica=warning,\u001b[0m\n",
      "\u001b[34mlog_on_each_node=True,\u001b[0m\n",
      "\u001b[34mlogging_dir=/tmp/ouput/runs/Jul20_06-02-53_algo-1,\u001b[0m\n",
      "\u001b[34mlogging_first_step=False,\u001b[0m\n",
      "\u001b[34mlogging_nan_inf_filter=True,\u001b[0m\n",
      "\u001b[34mlogging_steps=10,\u001b[0m\n",
      "\u001b[34mlogging_strategy=steps,\u001b[0m\n",
      "\u001b[34mlr_scheduler_type=cosine,\u001b[0m\n",
      "\u001b[34mmax_grad_norm=1.0,\u001b[0m\n",
      "\u001b[34mmax_steps=-1,\u001b[0m\n",
      "\u001b[34mmetric_for_best_model=None,\u001b[0m\n",
      "\u001b[34mmp_parameters=,\u001b[0m\n",
      "\u001b[34mno_cuda=False,\u001b[0m\n",
      "\u001b[34mnum_train_epochs=2.0,\u001b[0m\n",
      "\u001b[34moptim=adamw_torch,\u001b[0m\n",
      "\u001b[34moptim_args=None,\u001b[0m\n",
      "\u001b[34moutput_dir=/tmp/ouput/,\u001b[0m\n",
      "\u001b[34moverwrite_output_dir=False,\u001b[0m\n",
      "\u001b[34mpast_index=-1,\u001b[0m\n",
      "\u001b[34mper_device_eval_batch_size=8,\u001b[0m\n",
      "\u001b[34mper_device_train_batch_size=4,\u001b[0m\n",
      "\u001b[34mpredict_with_generate=False,\u001b[0m\n",
      "\u001b[34mprediction_loss_only=False,\u001b[0m\n",
      "\u001b[34mpush_to_hub=False,\u001b[0m\n",
      "\u001b[34mpush_to_hub_model_id=None,\u001b[0m\n",
      "\u001b[34mpush_to_hub_organization=None,\u001b[0m\n",
      "\u001b[34mpush_to_hub_token=<PUSH_TO_HUB_TOKEN>,\u001b[0m\n",
      "\u001b[34mray_scope=last,\u001b[0m\n",
      "\u001b[34mremove_unused_columns=True,\u001b[0m\n",
      "\u001b[34mreport_to=['wandb'],\u001b[0m\n",
      "\u001b[34mresume_from_checkpoint=None,\u001b[0m\n",
      "\u001b[34mrun_name=/tmp/ouput/,\u001b[0m\n",
      "\u001b[34msave_on_each_node=False,\u001b[0m\n",
      "\u001b[34msave_safetensors=False,\u001b[0m\n",
      "\u001b[34msave_steps=100,\u001b[0m\n",
      "\u001b[34msave_strategy=steps,\u001b[0m\n",
      "\u001b[34msave_total_limit=None,\u001b[0m\n",
      "\u001b[34mseed=42,\u001b[0m\n",
      "\u001b[34msharded_ddp=[],\u001b[0m\n",
      "\u001b[34mskip_memory_metrics=True,\u001b[0m\n",
      "\u001b[34msortish_sampler=False,\u001b[0m\n",
      "\u001b[34mtf32=None,\u001b[0m\n",
      "\u001b[34mtorch_compile=False,\u001b[0m\n",
      "\u001b[34mtorch_compile_backend=None,\u001b[0m\n",
      "\u001b[34mtorch_compile_mode=None,\u001b[0m\n",
      "\u001b[34mtorchdynamo=None,\u001b[0m\n",
      "\u001b[34mtpu_metrics_debug=False,\u001b[0m\n",
      "\u001b[34mtpu_num_cores=None,\u001b[0m\n",
      "\u001b[34muse_ipex=False,\u001b[0m\n",
      "\u001b[34muse_legacy_prediction_loop=False,\u001b[0m\n",
      "\u001b[34muse_mps_device=False,\u001b[0m\n",
      "\u001b[34mwarmup_ratio=0.0,\u001b[0m\n",
      "\u001b[34mwarmup_steps=0,\u001b[0m\n",
      "\u001b[34mweight_decay=0.0,\u001b[0m\n",
      "\u001b[34mxpu_backend=None,\u001b[0m\n",
      "\u001b[34m)\u001b[0m\n",
      "\u001b[34m07/20/2023 06:02:54 - INFO - llmtuner.dsets.loader - Loading dataset obsfs/corpus/zhetian_sft_instruction.json...\u001b[0m\n",
      "\u001b[34m07/20/2023 06:02:54 - WARNING - llmtuner.dsets.loader - Checksum failed: missing SHA-1 hash value in dataset_info.json or too many files.\u001b[0m\n",
      "\u001b[34m07/20/2023 06:02:54 - WARNING - llmtuner.tuner.core.parser - `ddp_find_unused_parameters` needs to be set as False in DDP training.\u001b[0m\n",
      "\u001b[34m07/20/2023 06:02:54 - INFO - llmtuner.tuner.core.parser - Process rank: 0, device: cuda:0, n_gpu: 1\n",
      "  distributed training: True, 16-bits training: True\u001b[0m\n",
      "\u001b[34m07/20/2023 06:02:54 - INFO - llmtuner.tuner.core.parser - Training/evaluation parameters Seq2SeqTrainingArguments(\u001b[0m\n",
      "\u001b[34m_n_gpu=1,\u001b[0m\n",
      "\u001b[34madafactor=False,\u001b[0m\n",
      "\u001b[34madam_beta1=0.9,\u001b[0m\n",
      "\u001b[34madam_beta2=0.999,\u001b[0m\n",
      "\u001b[34madam_epsilon=1e-08,\u001b[0m\n",
      "\u001b[34mauto_find_batch_size=False,\u001b[0m\n",
      "\u001b[34mbf16=False,\u001b[0m\n",
      "\u001b[34mbf16_full_eval=False,\u001b[0m\n",
      "\u001b[34mdata_seed=None,\u001b[0m\n",
      "\u001b[34mdataloader_drop_last=False,\u001b[0m\n",
      "\u001b[34mdataloader_num_workers=0,\u001b[0m\n",
      "\u001b[34mdataloader_pin_memory=True,\u001b[0m\n",
      "\u001b[34mddp_backend=None,\u001b[0m\n",
      "\u001b[34mddp_broadcast_buffers=None,\u001b[0m\n",
      "\u001b[34mddp_bucket_cap_mb=None,\u001b[0m\n",
      "\u001b[34mddp_find_unused_parameters=False,\u001b[0m\n",
      "\u001b[34mddp_timeout=1800,\u001b[0m\n",
      "\u001b[34mdebug=[],\u001b[0m\n",
      "\u001b[34mdeepspeed=None,\u001b[0m\n",
      "\u001b[34mdisable_tqdm=False,\u001b[0m\n",
      "\u001b[34mdo_eval=False,\u001b[0m\n",
      "\u001b[34mdo_predict=False,\u001b[0m\n",
      "\u001b[34mdo_train=True,\u001b[0m\n",
      "\u001b[34meval_accumulation_steps=None,\u001b[0m\n",
      "\u001b[34meval_delay=0,\u001b[0m\n",
      "\u001b[34meval_steps=None,\u001b[0m\n",
      "\u001b[34mevaluation_strategy=no,\u001b[0m\n",
      "\u001b[34mfp16=True,\u001b[0m\n",
      "\u001b[34mfp16_backend=auto,\u001b[0m\n",
      "\u001b[34mfp16_full_eval=False,\u001b[0m\n",
      "\u001b[34mfp16_opt_level=O1,\u001b[0m\n",
      "\u001b[34mfsdp=[],\u001b[0m\n",
      "\u001b[34mfsdp_config={'fsdp_min_num_params': 0, 'xla': False, 'xla_fsdp_grad_ckpt': False},\u001b[0m\n",
      "\u001b[34mfsdp_min_num_params=0,\u001b[0m\n",
      "\u001b[34mfsdp_transformer_layer_cls_to_wrap=None,\u001b[0m\n",
      "\u001b[34mfull_determinism=False,\u001b[0m\n",
      "\u001b[34mgeneration_config=None,\u001b[0m\n",
      "\u001b[34mgeneration_max_length=None,\u001b[0m\n",
      "\u001b[34mgeneration_num_beams=None,\u001b[0m\n",
      "\u001b[34mgradient_accumulation_steps=2,\u001b[0m\n",
      "\u001b[34mgradient_checkpointing=False,\u001b[0m\n",
      "\u001b[34mgreater_is_better=None,\u001b[0m\n",
      "\u001b[34mgroup_by_length=False,\u001b[0m\n",
      "\u001b[34mhalf_precision_backend=auto,\u001b[0m\n",
      "\u001b[34mhub_model_id=None,\u001b[0m\n",
      "\u001b[34mhub_private_repo=False,\u001b[0m\n",
      "\u001b[34mhub_strategy=every_save,\u001b[0m\n",
      "\u001b[34mhub_token=<HUB_TOKEN>,\u001b[0m\n",
      "\u001b[34mignore_data_skip=False,\u001b[0m\n",
      "\u001b[34minclude_inputs_for_metrics=False,\u001b[0m\n",
      "\u001b[34mjit_mode_eval=False,\u001b[0m\n",
      "\u001b[34mlabel_names=None,\u001b[0m\n",
      "\u001b[34mlabel_smoothing_factor=0.0,\u001b[0m\n",
      "\u001b[34mlearning_rate=0.0003,\u001b[0m\n",
      "\u001b[34mlength_column_name=length,\u001b[0m\n",
      "\u001b[34mload_best_model_at_end=False,\u001b[0m\n",
      "\u001b[34mlocal_rank=0,\u001b[0m\n",
      "\u001b[34mlog_level=passive,\u001b[0m\n",
      "\u001b[34mlog_level_replica=warning,\u001b[0m\n",
      "\u001b[34mlog_on_each_node=True,\u001b[0m\n",
      "\u001b[34mlogging_dir=/tmp/ouput/runs/Jul20_06-02-54_algo-1,\u001b[0m\n",
      "\u001b[34mlogging_first_step=False,\u001b[0m\n",
      "\u001b[34mlogging_nan_inf_filter=True,\u001b[0m\n",
      "\u001b[34mlogging_steps=10,\u001b[0m\n",
      "\u001b[34mlogging_strategy=steps,\u001b[0m\n",
      "\u001b[34mlr_scheduler_type=cosine,\u001b[0m\n",
      "\u001b[34mmax_grad_norm=1.0,\u001b[0m\n",
      "\u001b[34mmax_steps=-1,\u001b[0m\n",
      "\u001b[34mmetric_for_best_model=None,\u001b[0m\n",
      "\u001b[34mmp_parameters=,\u001b[0m\n",
      "\u001b[34mno_cuda=False,\u001b[0m\n",
      "\u001b[34mnum_train_epochs=2.0,\u001b[0m\n",
      "\u001b[34moptim=adamw_torch,\u001b[0m\n",
      "\u001b[34moptim_args=None,\u001b[0m\n",
      "\u001b[34moutput_dir=/tmp/ouput/,\u001b[0m\n",
      "\u001b[34moverwrite_output_dir=False,\u001b[0m\n",
      "\u001b[34mpast_index=-1,\u001b[0m\n",
      "\u001b[34mper_device_eval_batch_size=8,\u001b[0m\n",
      "\u001b[34mper_device_train_batch_size=4,\u001b[0m\n",
      "\u001b[34mpredict_with_generate=False,\u001b[0m\n",
      "\u001b[34mprediction_loss_only=False,\u001b[0m\n",
      "\u001b[34mpush_to_hub=False,\u001b[0m\n",
      "\u001b[34mpush_to_hub_model_id=None,\u001b[0m\n",
      "\u001b[34mpush_to_hub_organization=None,\u001b[0m\n",
      "\u001b[34mpush_to_hub_token=<PUSH_TO_HUB_TOKEN>,\u001b[0m\n",
      "\u001b[34mray_scope=last,\u001b[0m\n",
      "\u001b[34mremove_unused_columns=True,\u001b[0m\n",
      "\u001b[34mreport_to=['wandb'],\u001b[0m\n",
      "\u001b[34mresume_from_checkpoint=None,\u001b[0m\n",
      "\u001b[34mrun_name=/tmp/ouput/,\u001b[0m\n",
      "\u001b[34msave_on_each_node=False,\u001b[0m\n",
      "\u001b[34msave_safetensors=False,\u001b[0m\n",
      "\u001b[34msave_steps=100,\u001b[0m\n",
      "\u001b[34msave_strategy=steps,\u001b[0m\n",
      "\u001b[34msave_total_limit=None,\u001b[0m\n",
      "\u001b[34mseed=42,\u001b[0m\n",
      "\u001b[34msharded_ddp=[],\u001b[0m\n",
      "\u001b[34mskip_memory_metrics=True,\u001b[0m\n",
      "\u001b[34msortish_sampler=False,\u001b[0m\n",
      "\u001b[34mtf32=None,\u001b[0m\n",
      "\u001b[34mtorch_compile=False,\u001b[0m\n",
      "\u001b[34mtorch_compile_backend=None,\u001b[0m\n",
      "\u001b[34mtorch_compile_mode=None,\u001b[0m\n",
      "\u001b[34mtorchdynamo=None,\u001b[0m\n",
      "\u001b[34mtpu_metrics_debug=False,\u001b[0m\n",
      "\u001b[34mtpu_num_cores=None,\u001b[0m\n",
      "\u001b[34muse_ipex=False,\u001b[0m\n",
      "\u001b[34muse_legacy_prediction_loop=False,\u001b[0m\n",
      "\u001b[34muse_mps_device=False,\u001b[0m\n",
      "\u001b[34mwarmup_ratio=0.0,\u001b[0m\n",
      "\u001b[34mwarmup_steps=0,\u001b[0m\n",
      "\u001b[34mweight_decay=0.0,\u001b[0m\n",
      "\u001b[34mxpu_backend=None,\u001b[0m\n",
      "\u001b[34m)\u001b[0m\n",
      "\u001b[34m07/20/2023 06:02:54 - INFO - llmtuner.dsets.loader - Loading dataset obsfs/corpus/zhetian_sft_instruction.json...\u001b[0m\n",
      "\u001b[34m07/20/2023 06:02:54 - WARNING - llmtuner.dsets.loader - Checksum failed: missing SHA-1 hash value in dataset_info.json or too many files.\u001b[0m\n",
      "\u001b[34m07/20/2023 06:02:54 - WARNING - llmtuner.tuner.core.parser - `ddp_find_unused_parameters` needs to be set as False in DDP training.\u001b[0m\n",
      "\u001b[34m07/20/2023 06:02:54 - INFO - llmtuner.tuner.core.parser - Process rank: 6, device: cuda:6, n_gpu: 1\n",
      "  distributed training: True, 16-bits training: True\u001b[0m\n",
      "\u001b[34m07/20/2023 06:02:54 - WARNING - llmtuner.tuner.core.parser - `ddp_find_unused_parameters` needs to be set as False in DDP training.\u001b[0m\n",
      "\u001b[34m07/20/2023 06:02:54 - INFO - llmtuner.tuner.core.parser - Process rank: 4, device: cuda:4, n_gpu: 1\n",
      "  distributed training: True, 16-bits training: True\u001b[0m\n",
      "\u001b[34m07/20/2023 06:02:54 - INFO - llmtuner.tuner.core.parser - Training/evaluation parameters Seq2SeqTrainingArguments(\u001b[0m\n",
      "\u001b[34m_n_gpu=1,\u001b[0m\n",
      "\u001b[34madafactor=False,\u001b[0m\n",
      "\u001b[34madam_beta1=0.9,\u001b[0m\n",
      "\u001b[34madam_beta2=0.999,\u001b[0m\n",
      "\u001b[34madam_epsilon=1e-08,\u001b[0m\n",
      "\u001b[34mauto_find_batch_size=False,\u001b[0m\n",
      "\u001b[34mbf16=False,\u001b[0m\n",
      "\u001b[34mbf16_full_eval=False,\u001b[0m\n",
      "\u001b[34mdata_seed=None,\u001b[0m\n",
      "\u001b[34mdataloader_drop_last=False,\u001b[0m\n",
      "\u001b[34mdataloader_num_workers=0,\u001b[0m\n",
      "\u001b[34mdataloader_pin_memory=True,\u001b[0m\n",
      "\u001b[34mddp_backend=None,\u001b[0m\n",
      "\u001b[34mddp_broadcast_buffers=None,\u001b[0m\n",
      "\u001b[34mddp_bucket_cap_mb=None,\u001b[0m\n",
      "\u001b[34mddp_find_unused_parameters=False,\u001b[0m\n",
      "\u001b[34mddp_timeout=1800,\u001b[0m\n",
      "\u001b[34mdebug=[],\u001b[0m\n",
      "\u001b[34mdeepspeed=None,\u001b[0m\n",
      "\u001b[34mdisable_tqdm=False,\u001b[0m\n",
      "\u001b[34mdo_eval=False,\u001b[0m\n",
      "\u001b[34mdo_predict=False,\u001b[0m\n",
      "\u001b[34mdo_train=True,\u001b[0m\n",
      "\u001b[34meval_accumulation_steps=None,\u001b[0m\n",
      "\u001b[34meval_delay=0,\u001b[0m\n",
      "\u001b[34meval_steps=None,\u001b[0m\n",
      "\u001b[34mevaluation_strategy=no,\u001b[0m\n",
      "\u001b[34mfp16=True,\u001b[0m\n",
      "\u001b[34mfp16_backend=auto,\u001b[0m\n",
      "\u001b[34mfp16_full_eval=False,\u001b[0m\n",
      "\u001b[34mfp16_opt_level=O1,\u001b[0m\n",
      "\u001b[34mfsdp=[],\u001b[0m\n",
      "\u001b[34mfsdp_config={'fsdp_min_num_params': 0, 'xla': False, 'xla_fsdp_grad_ckpt': False},\u001b[0m\n",
      "\u001b[34mfsdp_min_num_params=0,\u001b[0m\n",
      "\u001b[34mfsdp_transformer_layer_cls_to_wrap=None,\u001b[0m\n",
      "\u001b[34mfull_determinism=False,\u001b[0m\n",
      "\u001b[34mgeneration_config=None,\u001b[0m\n",
      "\u001b[34mgeneration_max_length=None,\u001b[0m\n",
      "\u001b[34mgeneration_num_beams=None,\u001b[0m\n",
      "\u001b[34mgradient_accumulation_steps=2,\u001b[0m\n",
      "\u001b[34mgradient_checkpointing=False,\u001b[0m\n",
      "\u001b[34mgreater_is_better=None,\u001b[0m\n",
      "\u001b[34mgroup_by_length=False,\u001b[0m\n",
      "\u001b[34mhalf_precision_backend=auto,\u001b[0m\n",
      "\u001b[34mhub_model_id=None,\u001b[0m\n",
      "\u001b[34mhub_private_repo=False,\u001b[0m\n",
      "\u001b[34mhub_strategy=every_save,\u001b[0m\n",
      "\u001b[34mhub_token=<HUB_TOKEN>,\u001b[0m\n",
      "\u001b[34mignore_data_skip=False,\u001b[0m\n",
      "\u001b[34minclude_inputs_for_metrics=False,\u001b[0m\n",
      "\u001b[34mjit_mode_eval=False,\u001b[0m\n",
      "\u001b[34mlabel_names=None,\u001b[0m\n",
      "\u001b[34mlabel_smoothing_factor=0.0,\u001b[0m\n",
      "\u001b[34mlearning_rate=0.0003,\u001b[0m\n",
      "\u001b[34mlength_column_name=length,\u001b[0m\n",
      "\u001b[34mload_best_model_at_end=False,\u001b[0m\n",
      "\u001b[34mlocal_rank=6,\u001b[0m\n",
      "\u001b[34mlog_level=passive,\u001b[0m\n",
      "\u001b[34mlog_level_replica=warning,\u001b[0m\n",
      "\u001b[34mlog_on_each_node=True,\u001b[0m\n",
      "\u001b[34mlogging_dir=/tmp/ouput/runs/Jul20_06-02-54_algo-1,\u001b[0m\n",
      "\u001b[34mlogging_first_step=False,\u001b[0m\n",
      "\u001b[34mlogging_nan_inf_filter=True,\u001b[0m\n",
      "\u001b[34mlogging_steps=10,\u001b[0m\n",
      "\u001b[34mlogging_strategy=steps,\u001b[0m\n",
      "\u001b[34mlr_scheduler_type=cosine,\u001b[0m\n",
      "\u001b[34mmax_grad_norm=1.0,\u001b[0m\n",
      "\u001b[34mmax_steps=-1,\u001b[0m\n",
      "\u001b[34mmetric_for_best_model=None,\u001b[0m\n",
      "\u001b[34mmp_parameters=,\u001b[0m\n",
      "\u001b[34mno_cuda=False,\u001b[0m\n",
      "\u001b[34mnum_train_epochs=2.0,\u001b[0m\n",
      "\u001b[34moptim=adamw_torch,\u001b[0m\n",
      "\u001b[34moptim_args=None,\u001b[0m\n",
      "\u001b[34moutput_dir=/tmp/ouput/,\u001b[0m\n",
      "\u001b[34moverwrite_output_dir=False,\u001b[0m\n",
      "\u001b[34mpast_index=-1,\u001b[0m\n",
      "\u001b[34mper_device_eval_batch_size=8,\u001b[0m\n",
      "\u001b[34mper_device_train_batch_size=4,\u001b[0m\n",
      "\u001b[34mpredict_with_generate=False,\u001b[0m\n",
      "\u001b[34mprediction_loss_only=False,\u001b[0m\n",
      "\u001b[34mpush_to_hub=False,\u001b[0m\n",
      "\u001b[34mpush_to_hub_model_id=None,\u001b[0m\n",
      "\u001b[34mpush_to_hub_organization=None,\u001b[0m\n",
      "\u001b[34mpush_to_hub_token=<PUSH_TO_HUB_TOKEN>,\u001b[0m\n",
      "\u001b[34mray_scope=last,\u001b[0m\n",
      "\u001b[34mremove_unused_columns=True,\u001b[0m\n",
      "\u001b[34mreport_to=['wandb'],\u001b[0m\n",
      "\u001b[34mresume_from_checkpoint=None,\u001b[0m\n",
      "\u001b[34mrun_name=/tmp/ouput/,\u001b[0m\n",
      "\u001b[34msave_on_each_node=False,\u001b[0m\n",
      "\u001b[34msave_safetensors=False,\u001b[0m\n",
      "\u001b[34msave_steps=100,\u001b[0m\n",
      "\u001b[34msave_strategy=steps,\u001b[0m\n",
      "\u001b[34msave_total_limit=None,\u001b[0m\n",
      "\u001b[34mseed=42,\u001b[0m\n",
      "\u001b[34msharded_ddp=[],\u001b[0m\n",
      "\u001b[34mskip_memory_metrics=True,\u001b[0m\n",
      "\u001b[34msortish_sampler=False,\u001b[0m\n",
      "\u001b[34mtf32=None,\u001b[0m\n",
      "\u001b[34mtorch_compile=False,\u001b[0m\n",
      "\u001b[34mtorch_compile_backend=None,\u001b[0m\n",
      "\u001b[34mtorch_compile_mode=None,\u001b[0m\n",
      "\u001b[34mtorchdynamo=None,\u001b[0m\n",
      "\u001b[34mtpu_metrics_debug=False,\u001b[0m\n",
      "\u001b[34mtpu_num_cores=None,\u001b[0m\n",
      "\u001b[34muse_ipex=False,\u001b[0m\n",
      "\u001b[34muse_legacy_prediction_loop=False,\u001b[0m\n",
      "\u001b[34muse_mps_device=False,\u001b[0m\n",
      "\u001b[34mwarmup_ratio=0.0,\u001b[0m\n",
      "\u001b[34mwarmup_steps=0,\u001b[0m\n",
      "\u001b[34mweight_decay=0.0,\u001b[0m\n",
      "\u001b[34mxpu_backend=None,\u001b[0m\n",
      "\u001b[34m)\u001b[0m\n",
      "\u001b[34m07/20/2023 06:02:54 - INFO - llmtuner.dsets.loader - Loading dataset obsfs/corpus/zhetian_sft_instruction.json...\u001b[0m\n",
      "\u001b[34m07/20/2023 06:02:54 - WARNING - llmtuner.dsets.loader - Checksum failed: missing SHA-1 hash value in dataset_info.json or too many files.\u001b[0m\n",
      "\u001b[34m07/20/2023 06:02:54 - INFO - llmtuner.tuner.core.parser - Training/evaluation parameters Seq2SeqTrainingArguments(\u001b[0m\n",
      "\u001b[34m_n_gpu=1,\u001b[0m\n",
      "\u001b[34madafactor=False,\u001b[0m\n",
      "\u001b[34madam_beta1=0.9,\u001b[0m\n",
      "\u001b[34madam_beta2=0.999,\u001b[0m\n",
      "\u001b[34madam_epsilon=1e-08,\u001b[0m\n",
      "\u001b[34mauto_find_batch_size=False,\u001b[0m\n",
      "\u001b[34mbf16=False,\u001b[0m\n",
      "\u001b[34mbf16_full_eval=False,\u001b[0m\n",
      "\u001b[34mdata_seed=None,\u001b[0m\n",
      "\u001b[34mdataloader_drop_last=False,\u001b[0m\n",
      "\u001b[34mdataloader_num_workers=0,\u001b[0m\n",
      "\u001b[34mdataloader_pin_memory=True,\u001b[0m\n",
      "\u001b[34mddp_backend=None,\u001b[0m\n",
      "\u001b[34mddp_broadcast_buffers=None,\u001b[0m\n",
      "\u001b[34mddp_bucket_cap_mb=None,\u001b[0m\n",
      "\u001b[34mddp_find_unused_parameters=False,\u001b[0m\n",
      "\u001b[34mddp_timeout=1800,\u001b[0m\n",
      "\u001b[34mdebug=[],\u001b[0m\n",
      "\u001b[34mdeepspeed=None,\u001b[0m\n",
      "\u001b[34mdisable_tqdm=False,\u001b[0m\n",
      "\u001b[34mdo_eval=False,\u001b[0m\n",
      "\u001b[34mdo_predict=False,\u001b[0m\n",
      "\u001b[34mdo_train=True,\u001b[0m\n",
      "\u001b[34meval_accumulation_steps=None,\u001b[0m\n",
      "\u001b[34meval_delay=0,\u001b[0m\n",
      "\u001b[34meval_steps=None,\u001b[0m\n",
      "\u001b[34mevaluation_strategy=no,\u001b[0m\n",
      "\u001b[34mfp16=True,\u001b[0m\n",
      "\u001b[34mfp16_backend=auto,\u001b[0m\n",
      "\u001b[34mfp16_full_eval=False,\u001b[0m\n",
      "\u001b[34mfp16_opt_level=O1,\u001b[0m\n",
      "\u001b[34mfsdp=[],\u001b[0m\n",
      "\u001b[34mfsdp_config={'fsdp_min_num_params': 0, 'xla': False, 'xla_fsdp_grad_ckpt': False},\u001b[0m\n",
      "\u001b[34mfsdp_min_num_params=0,\u001b[0m\n",
      "\u001b[34mfsdp_transformer_layer_cls_to_wrap=None,\u001b[0m\n",
      "\u001b[34mfull_determinism=False,\u001b[0m\n",
      "\u001b[34mgeneration_config=None,\u001b[0m\n",
      "\u001b[34mgeneration_max_length=None,\u001b[0m\n",
      "\u001b[34mgeneration_num_beams=None,\u001b[0m\n",
      "\u001b[34mgradient_accumulation_steps=2,\u001b[0m\n",
      "\u001b[34mgradient_checkpointing=False,\u001b[0m\n",
      "\u001b[34mgreater_is_better=None,\u001b[0m\n",
      "\u001b[34mgroup_by_length=False,\u001b[0m\n",
      "\u001b[34mhalf_precision_backend=auto,\u001b[0m\n",
      "\u001b[34mhub_model_id=None,\u001b[0m\n",
      "\u001b[34mhub_private_repo=False,\u001b[0m\n",
      "\u001b[34mhub_strategy=every_save,\u001b[0m\n",
      "\u001b[34mhub_token=<HUB_TOKEN>,\u001b[0m\n",
      "\u001b[34mignore_data_skip=False,\u001b[0m\n",
      "\u001b[34minclude_inputs_for_metrics=False,\u001b[0m\n",
      "\u001b[34mjit_mode_eval=False,\u001b[0m\n",
      "\u001b[34mlabel_names=None,\u001b[0m\n",
      "\u001b[34mlabel_smoothing_factor=0.0,\u001b[0m\n",
      "\u001b[34mlearning_rate=0.0003,\u001b[0m\n",
      "\u001b[34mlength_column_name=length,\u001b[0m\n",
      "\u001b[34mload_best_model_at_end=False,\u001b[0m\n",
      "\u001b[34mlocal_rank=4,\u001b[0m\n",
      "\u001b[34mlog_level=passive,\u001b[0m\n",
      "\u001b[34mlog_level_replica=warning,\u001b[0m\n",
      "\u001b[34mlog_on_each_node=True,\u001b[0m\n",
      "\u001b[34mlogging_dir=/tmp/ouput/runs/Jul20_06-02-53_algo-1,\u001b[0m\n",
      "\u001b[34mlogging_first_step=False,\u001b[0m\n",
      "\u001b[34mlogging_nan_inf_filter=True,\u001b[0m\n",
      "\u001b[34mlogging_steps=10,\u001b[0m\n",
      "\u001b[34mlogging_strategy=steps,\u001b[0m\n",
      "\u001b[34mlr_scheduler_type=cosine,\u001b[0m\n",
      "\u001b[34mmax_grad_norm=1.0,\u001b[0m\n",
      "\u001b[34mmax_steps=-1,\u001b[0m\n",
      "\u001b[34mmetric_for_best_model=None,\u001b[0m\n",
      "\u001b[34mmp_parameters=,\u001b[0m\n",
      "\u001b[34mno_cuda=False,\u001b[0m\n",
      "\u001b[34mnum_train_epochs=2.0,\u001b[0m\n",
      "\u001b[34moptim=adamw_torch,\u001b[0m\n",
      "\u001b[34moptim_args=None,\u001b[0m\n",
      "\u001b[34moutput_dir=/tmp/ouput/,\u001b[0m\n",
      "\u001b[34moverwrite_output_dir=False,\u001b[0m\n",
      "\u001b[34mpast_index=-1,\u001b[0m\n",
      "\u001b[34mper_device_eval_batch_size=8,\u001b[0m\n",
      "\u001b[34mper_device_train_batch_size=4,\u001b[0m\n",
      "\u001b[34mpredict_with_generate=False,\u001b[0m\n",
      "\u001b[34mprediction_loss_only=False,\u001b[0m\n",
      "\u001b[34mpush_to_hub=False,\u001b[0m\n",
      "\u001b[34mpush_to_hub_model_id=None,\u001b[0m\n",
      "\u001b[34mpush_to_hub_organization=None,\u001b[0m\n",
      "\u001b[34mpush_to_hub_token=<PUSH_TO_HUB_TOKEN>,\u001b[0m\n",
      "\u001b[34mray_scope=last,\u001b[0m\n",
      "\u001b[34mremove_unused_columns=True,\u001b[0m\n",
      "\u001b[34mreport_to=['wandb'],\u001b[0m\n",
      "\u001b[34mresume_from_checkpoint=None,\u001b[0m\n",
      "\u001b[34mrun_name=/tmp/ouput/,\u001b[0m\n",
      "\u001b[34msave_on_each_node=False,\u001b[0m\n",
      "\u001b[34msave_safetensors=False,\u001b[0m\n",
      "\u001b[34msave_steps=100,\u001b[0m\n",
      "\u001b[34msave_strategy=steps,\u001b[0m\n",
      "\u001b[34msave_total_limit=None,\u001b[0m\n",
      "\u001b[34mseed=42,\u001b[0m\n",
      "\u001b[34msharded_ddp=[],\u001b[0m\n",
      "\u001b[34mskip_memory_metrics=True,\u001b[0m\n",
      "\u001b[34msortish_sampler=False,\u001b[0m\n",
      "\u001b[34mtf32=None,\u001b[0m\n",
      "\u001b[34mtorch_compile=False,\u001b[0m\n",
      "\u001b[34mtorch_compile_backend=None,\u001b[0m\n",
      "\u001b[34mtorch_compile_mode=None,\u001b[0m\n",
      "\u001b[34mtorchdynamo=None,\u001b[0m\n",
      "\u001b[34mtpu_metrics_debug=False,\u001b[0m\n",
      "\u001b[34mtpu_num_cores=None,\u001b[0m\n",
      "\u001b[34muse_ipex=False,\u001b[0m\n",
      "\u001b[34muse_legacy_prediction_loop=False,\u001b[0m\n",
      "\u001b[34muse_mps_device=False,\u001b[0m\n",
      "\u001b[34mwarmup_ratio=0.0,\u001b[0m\n",
      "\u001b[34mwarmup_steps=0,\u001b[0m\n",
      "\u001b[34mweight_decay=0.0,\u001b[0m\n",
      "\u001b[34mxpu_backend=None,\u001b[0m\n",
      "\u001b[34m)\u001b[0m\n",
      "\u001b[34m07/20/2023 06:02:54 - INFO - llmtuner.dsets.loader - Loading dataset obsfs/corpus/zhetian_sft_instruction.json...\u001b[0m\n",
      "\u001b[34m07/20/2023 06:02:54 - WARNING - llmtuner.dsets.loader - Checksum failed: missing SHA-1 hash value in dataset_info.json or too many files.\u001b[0m\n",
      "\u001b[34mDownloading and preparing dataset json/default to /root/.cache/huggingface/datasets/json/default-cc8d4835c7fbe295/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96...\u001b[0m\n",
      "\u001b[34mDownloading data files:   0%|          | 0/1 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[34mDownloading data files: 100%|██████████| 1/1 [00:00<00:00, 11184.81it/s]\u001b[0m\n",
      "\u001b[34mExtracting data files:   0%|          | 0/1 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[34mExtracting data files: 100%|██████████| 1/1 [00:00<00:00, 1465.52it/s]\u001b[0m\n",
      "\u001b[34mGenerating train split: 0 examples [00:00, ? examples/s]\u001b[0m\n",
      "\u001b[34mGenerating train split: 11264 examples [00:00, 36322.42 examples/s]\u001b[0m\n",
      "\u001b[34mDataset json downloaded and prepared to /root/.cache/huggingface/datasets/json/default-cc8d4835c7fbe295/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96. Subsequent calls will reuse this data.\u001b[0m\n",
      "\u001b[34m0%|          | 0/1 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[34mFound cached dataset json (/root/.cache/huggingface/datasets/json/default-cc8d4835c7fbe295/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)\u001b[0m\n",
      "\u001b[34m100%|██████████| 1/1 [00:00<00:00, 745.26it/s]\u001b[0m\n",
      "\u001b[34m0%|          | 0/1 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[34mFound cached dataset json (/root/.cache/huggingface/datasets/json/default-cc8d4835c7fbe295/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)\u001b[0m\n",
      "\u001b[34m100%|██████████| 1/1 [00:00<00:00, 563.52it/s]\u001b[0m\n",
      "\u001b[34m0%|          | 0/1 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[34mFound cached dataset json (/root/.cache/huggingface/datasets/json/default-cc8d4835c7fbe295/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)\u001b[0m\n",
      "\u001b[34m100%|██████████| 1/1 [00:00<00:00, 577.41it/s]\u001b[0m\n",
      "\u001b[34m0%|          | 0/1 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[34mFound cached dataset json (/root/.cache/huggingface/datasets/json/default-cc8d4835c7fbe295/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)\u001b[0m\n",
      "\u001b[34m0%|          | 0/1 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[34m100%|██████████| 1/1 [00:00<00:00, 549.57it/s]\u001b[0m\n",
      "\u001b[34m100%|██████████| 1/1 [00:00<00:00, 553.27it/s]\u001b[0m\n",
      "\u001b[34mFound cached dataset json (/root/.cache/huggingface/datasets/json/default-cc8d4835c7fbe295/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)\u001b[0m\n",
      "\u001b[34m0%|          | 0/1 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[34mFound cached dataset json (/root/.cache/huggingface/datasets/json/default-cc8d4835c7fbe295/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)\u001b[0m\n",
      "\u001b[34mFound cached dataset json (/root/.cache/huggingface/datasets/json/default-cc8d4835c7fbe295/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)\u001b[0m\n",
      "\u001b[34m100%|██████████| 1/1 [00:00<00:00, 570.11it/s]\u001b[0m\n",
      "\u001b[34m0%|          | 0/1 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[34m0%|          | 0/1 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[34m100%|██████████| 1/1 [00:00<00:00, 556.86it/s]\u001b[0m\n",
      "\u001b[34m100%|██████████| 1/1 [00:00<00:00, 576.46it/s]\u001b[0m\n",
      "\u001b[34m[INFO|tokenization_utils_base.py:1837] 2023-07-20 06:02:54,948 >> loading file tokenizer.model\u001b[0m\n",
      "\u001b[34m[INFO|tokenization_utils_base.py:1837] 2023-07-20 06:02:54,948 >> loading file added_tokens.json\u001b[0m\n",
      "\u001b[34m[INFO|tokenization_utils_base.py:1837] 2023-07-20 06:02:54,948 >> loading file special_tokens_map.json\u001b[0m\n",
      "\u001b[34m[INFO|tokenization_utils_base.py:1837] 2023-07-20 06:02:54,948 >> loading file tokenizer_config.json\u001b[0m\n",
      "\u001b[34m[INFO|tokenization_utils_base.py:1837] 2023-07-20 06:02:54,948 >> loading file tokenizer.model\u001b[0m\n",
      "\u001b[34m[INFO|tokenization_utils_base.py:1837] 2023-07-20 06:02:54,948 >> loading file added_tokens.json\u001b[0m\n",
      "\u001b[34m[INFO|tokenization_utils_base.py:1837] 2023-07-20 06:02:54,948 >> loading file special_tokens_map.json\u001b[0m\n",
      "\u001b[34m[INFO|tokenization_utils_base.py:1837] 2023-07-20 06:02:54,948 >> loading file tokenizer_config.json\u001b[0m\n",
      "\u001b[34m[INFO|configuration_utils.py:710] 2023-07-20 06:02:54,977 >> loading configuration file /tmp/baichun-13b/config.json\u001b[0m\n",
      "\u001b[34m[INFO|configuration_utils.py:710] 2023-07-20 06:02:54,977 >> loading configuration file /tmp/baichun-13b/config.json\u001b[0m\n",
      "\u001b[34m[INFO|configuration_utils.py:710] 2023-07-20 06:02:54,978 >> loading configuration file /tmp/baichun-13b/config.json\u001b[0m\n",
      "\u001b[34m[INFO|configuration_utils.py:710] 2023-07-20 06:02:54,978 >> loading configuration file /tmp/baichun-13b/config.json\u001b[0m\n",
      "\u001b[34m[INFO|configuration_utils.py:768] 2023-07-20 06:02:54,978 >> Model config BaichuanConfig {\n",
      "  \"_from_model_config\": true,\n",
      "  \"_name_or_path\": \"/tmp/baichun-13b/\",\n",
      "  \"architectures\": [\n",
      "    \"BaichuanForCausalLM\"\n",
      "  ],\n",
      "  \"auto_map\": {\n",
      "    \"AutoConfig\": \"configuration_baichuan.BaichuanConfig\",\n",
      "    \"AutoModelForCausalLM\": \"modeling_baichuan.BaichuanForCausalLM\"\n",
      "  },\n",
      "  \"bos_token_id\": 1,\n",
      "  \"eos_token_id\": 2,\n",
      "  \"gradient_checkpointing\": [\n",
      "    false\n",
      "  ],\n",
      "  \"hidden_act\": \"silu\",\n",
      "  \"hidden_size\": 5120,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 13696,\n",
      "  \"model_max_length\": 4096,\n",
      "  \"model_type\": \"baichuan\",\n",
      "  \"num_attention_heads\": 40,\n",
      "  \"num_hidden_layers\": 40,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"rms_norm_eps\": 1e-06,\n",
      "  \"tie_word_embeddings\": false,\n",
      "  \"torch_dtype\": \"bfloat16\",\n",
      "  \"transformers_version\": \"4.31.0\",\n",
      "  \"use_cache\": true,\n",
      "  \"vocab_size\": 64000\u001b[0m\n",
      "\u001b[34m}\u001b[0m\n",
      "\u001b[34m[INFO|configuration_utils.py:768] 2023-07-20 06:02:54,978 >> Model config BaichuanConfig {\n",
      "  \"_from_model_config\": true,\n",
      "  \"_name_or_path\": \"/tmp/baichun-13b/\",\n",
      "  \"architectures\": [\n",
      "    \"BaichuanForCausalLM\"\n",
      "  ],\n",
      "  \"auto_map\": {\n",
      "    \"AutoConfig\": \"configuration_baichuan.BaichuanConfig\",\n",
      "    \"AutoModelForCausalLM\": \"modeling_baichuan.BaichuanForCausalLM\"\n",
      "  },\n",
      "  \"bos_token_id\": 1,\n",
      "  \"eos_token_id\": 2,\n",
      "  \"gradient_checkpointing\": [\n",
      "    false\n",
      "  ],\n",
      "  \"hidden_act\": \"silu\",\n",
      "  \"hidden_size\": 5120,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 13696,\n",
      "  \"model_max_length\": 4096,\n",
      "  \"model_type\": \"baichuan\",\n",
      "  \"num_attention_heads\": 40,\n",
      "  \"num_hidden_layers\": 40,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"rms_norm_eps\": 1e-06,\n",
      "  \"tie_word_embeddings\": false,\n",
      "  \"torch_dtype\": \"bfloat16\",\n",
      "  \"transformers_version\": \"4.31.0\",\n",
      "  \"use_cache\": true,\n",
      "  \"vocab_size\": 64000\u001b[0m\n",
      "\u001b[34m}\u001b[0m\n",
      "\u001b[34m[INFO|modeling_utils.py:2600] 2023-07-20 06:02:55,004 >> loading weights file /tmp/baichun-13b/pytorch_model.bin.index.json\u001b[0m\n",
      "\u001b[34m[INFO|modeling_utils.py:2600] 2023-07-20 06:02:55,004 >> loading weights file /tmp/baichun-13b/pytorch_model.bin.index.json\u001b[0m\n",
      "\u001b[34m[INFO|modeling_utils.py:1172] 2023-07-20 06:02:55,004 >> Instantiating BaichuanForCausalLM model under default dtype torch.float16.\u001b[0m\n",
      "\u001b[34m[INFO|modeling_utils.py:1172] 2023-07-20 06:02:55,004 >> Instantiating BaichuanForCausalLM model under default dtype torch.float16.\u001b[0m\n",
      "\u001b[34m[INFO|configuration_utils.py:599] 2023-07-20 06:02:55,005 >> Generate config GenerationConfig {\n",
      "  \"_from_model_config\": true,\n",
      "  \"bos_token_id\": 1,\n",
      "  \"eos_token_id\": 2,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.31.0\"\u001b[0m\n",
      "\u001b[34m}\u001b[0m\n",
      "\u001b[34m[INFO|configuration_utils.py:599] 2023-07-20 06:02:55,005 >> Generate config GenerationConfig {\n",
      "  \"_from_model_config\": true,\n",
      "  \"bos_token_id\": 1,\n",
      "  \"eos_token_id\": 2,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.31.0\"\u001b[0m\n",
      "\u001b[34m}\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s]#015Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  33%|███▎      | 1/3 [00:12<00:25, 12.54s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  33%|███▎      | 1/3 [00:13<00:26, 13.20s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  33%|███▎      | 1/3 [00:13<00:26, 13.27s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  33%|███▎      | 1/3 [00:13<00:27, 13.57s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  33%|███▎      | 1/3 [00:13<00:27, 13.61s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  33%|███▎      | 1/3 [00:13<00:27, 13.65s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  33%|███▎      | 1/3 [00:13<00:27, 13.63s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  33%|███▎      | 1/3 [00:13<00:27, 13.68s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  67%|██████▋   | 2/3 [00:25<00:12, 12.81s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  67%|██████▋   | 2/3 [00:26<00:13, 13.27s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  67%|██████▋   | 2/3 [00:26<00:13, 13.32s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  67%|██████▋   | 2/3 [00:26<00:13, 13.45s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  67%|██████▋   | 2/3 [00:26<00:13, 13.46s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  67%|██████▋   | 2/3 [00:27<00:13, 13.48s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  67%|██████▋   | 2/3 [00:26<00:13, 13.46s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  67%|██████▋   | 2/3 [00:27<00:13, 13.48s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 3/3 [00:34<00:00, 10.97s/it]#015Loading checkpoint shards: 100%|██████████| 3/3 [00:34<00:00, 11.44s/it]\u001b[0m\n",
      "\u001b[34m07/20/2023 06:03:29 - INFO - llmtuner.tuner.core.adapter - Fine-tuning method: LoRA\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 3/3 [00:35<00:00, 11.28s/it]#015Loading checkpoint shards: 100%|██████████| 3/3 [00:35<00:00, 11.82s/it]\u001b[0m\n",
      "\u001b[34m07/20/2023 06:03:30 - INFO - llmtuner.tuner.core.adapter - Fine-tuning method: LoRA\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 3/3 [00:35<00:00, 11.30s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 3/3 [00:35<00:00, 11.83s/it]\u001b[0m\n",
      "\u001b[34m07/20/2023 06:03:30 - INFO - llmtuner.tuner.core.adapter - Fine-tuning method: LoRA\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 3/3 [00:35<00:00, 11.38s/it]#015Loading checkpoint shards: 100%|██████████| 3/3 [00:35<00:00, 11.95s/it]\u001b[0m\n",
      "\u001b[34m[INFO|modeling_utils.py:3329] 2023-07-20 06:03:30,909 >> All model checkpoint weights were used when initializing BaichuanForCausalLM.\u001b[0m\n",
      "\u001b[34m[INFO|modeling_utils.py:3337] 2023-07-20 06:03:30,909 >> All the weights of BaichuanForCausalLM were initialized from the model checkpoint at /tmp/baichun-13b/.\u001b[0m\n",
      "\u001b[34mIf your task is similar to the task the model of the checkpoint was trained on, you can already use BaichuanForCausalLM for predictions without further training.\u001b[0m\n",
      "\u001b[34m[INFO|modeling_utils.py:3329] 2023-07-20 06:03:30,909 >> All model checkpoint weights were used when initializing BaichuanForCausalLM.\u001b[0m\n",
      "\u001b[34m[INFO|modeling_utils.py:3337] 2023-07-20 06:03:30,909 >> All the weights of BaichuanForCausalLM were initialized from the model checkpoint at /tmp/baichun-13b/.\u001b[0m\n",
      "\u001b[34mIf your task is similar to the task the model of the checkpoint was trained on, you can already use BaichuanForCausalLM for predictions without further training.\u001b[0m\n",
      "\u001b[34m[INFO|configuration_utils.py:559] 2023-07-20 06:03:30,912 >> loading configuration file /tmp/baichun-13b/generation_config.json\u001b[0m\n",
      "\u001b[34m[INFO|configuration_utils.py:559] 2023-07-20 06:03:30,912 >> loading configuration file /tmp/baichun-13b/generation_config.json\u001b[0m\n",
      "\u001b[34m[INFO|configuration_utils.py:599] 2023-07-20 06:03:30,912 >> Generate config GenerationConfig {\n",
      "  \"assistant_token_id\": 196,\n",
      "  \"bos_token_id\": 1,\n",
      "  \"do_sample\": true,\n",
      "  \"eos_token_id\": 2,\n",
      "  \"max_new_tokens\": 2048,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"repetition_penalty\": 1.1,\n",
      "  \"temperature\": 0.3,\n",
      "  \"top_k\": 5,\n",
      "  \"top_p\": 0.85,\n",
      "  \"transformers_version\": \"4.31.0\",\n",
      "  \"user_token_id\": 195\u001b[0m\n",
      "\u001b[34m}\u001b[0m\n",
      "\u001b[34m[INFO|configuration_utils.py:599] 2023-07-20 06:03:30,912 >> Generate config GenerationConfig {\n",
      "  \"assistant_token_id\": 196,\n",
      "  \"bos_token_id\": 1,\n",
      "  \"do_sample\": true,\n",
      "  \"eos_token_id\": 2,\n",
      "  \"max_new_tokens\": 2048,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"repetition_penalty\": 1.1,\n",
      "  \"temperature\": 0.3,\n",
      "  \"top_k\": 5,\n",
      "  \"top_p\": 0.85,\n",
      "  \"transformers_version\": \"4.31.0\",\n",
      "  \"user_token_id\": 195\u001b[0m\n",
      "\u001b[34m}\u001b[0m\n",
      "\u001b[34m07/20/2023 06:03:30 - INFO - llmtuner.tuner.core.adapter - Fine-tuning method: LoRA\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 3/3 [00:35<00:00, 11.39s/it]#015Loading checkpoint shards: 100%|██████████| 3/3 [00:35<00:00, 11.96s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 3/3 [00:35<00:00, 11.37s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 3/3 [00:35<00:00, 11.95s/it]\u001b[0m\n",
      "\u001b[34m07/20/2023 06:03:30 - INFO - llmtuner.tuner.core.adapter - Fine-tuning method: LoRA\u001b[0m\n",
      "\u001b[34m07/20/2023 06:03:30 - INFO - llmtuner.tuner.core.adapter - Fine-tuning method: LoRA\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 3/3 [00:35<00:00, 11.39s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 3/3 [00:35<00:00, 11.97s/it]\u001b[0m\n",
      "\u001b[34m07/20/2023 06:03:30 - INFO - llmtuner.tuner.core.adapter - Fine-tuning method: LoRA\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 3/3 [00:35<00:00, 11.38s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 3/3 [00:35<00:00, 11.97s/it]\u001b[0m\n",
      "\u001b[34m07/20/2023 06:03:30 - INFO - llmtuner.tuner.core.adapter - Fine-tuning method: LoRA\u001b[0m\n",
      "\u001b[34mtrainable params: 223150080 || all params: 13488051200 || trainable%: 1.6544\u001b[0m\n",
      "\u001b[34mtrainable params: 223150080 || all params: 13488051200 || trainable%: 1.6544\u001b[0m\n",
      "\u001b[34mtrainable params: 223150080 || all params: 13488051200 || trainable%: 1.6544\u001b[0m\n",
      "\u001b[34mtrainable params: 223150080 || all params: 13488051200 || trainable%: 1.6544\u001b[0m\n",
      "\u001b[34mtrainable params: 223150080 || all params: 13488051200 || trainable%: 1.6544\u001b[0m\n",
      "\u001b[34mtrainable params: 223150080 || all params: 13488051200 || trainable%: 1.6544\u001b[0m\n",
      "\u001b[34mtrainable params: 223150080 || all params: 13488051200 || trainable%: 1.6544\u001b[0m\n",
      "\u001b[34mtrainable params: 223150080 || all params: 13488051200 || trainable%: 1.6544\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:   0%|          | 0/11264 [00:00<?, ? examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:   9%|▉         | 1000/11264 [00:00<00:02, 4712.25 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  18%|█▊        | 2000/11264 [00:00<00:01, 4827.32 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  27%|██▋       | 3000/11264 [00:00<00:01, 4885.67 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  36%|███▌      | 4000/11264 [00:00<00:01, 4909.66 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  44%|████▍     | 5000/11264 [00:01<00:01, 5069.19 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  53%|█████▎    | 6000/11264 [00:01<00:01, 3383.29 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  62%|██████▏   | 7000/11264 [00:02<00:01, 2609.55 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  71%|███████   | 8000/11264 [00:02<00:01, 2304.66 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  80%|███████▉  | 9000/11264 [00:03<00:01, 2139.36 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  89%|████████▉ | 10000/11264 [00:03<00:00, 2041.18 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  98%|█████████▊| 11000/11264 [00:04<00:00, 1976.36 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset: 100%|██████████| 11264/11264 [00:04<00:00, 1958.67 examples/s]\u001b[0m\n",
      "\u001b[34minput_ids:\u001b[0m\n",
      "\u001b[34m[99, 195, 9875, 3137, 31379, 32020, 17324, 31135, 32094, 32991, 7713, 7308, 7654, 31763, 75, 31106, 196, 31106, 31106, 32094, 32991, 31395, 4321, 3068, 31449, 18534, 31475, 31729, 72, 31354, 18218, 3876, 12434, 19508, 3942, 73, 2]\u001b[0m\n",
      "\u001b[34minputs:\n",
      "  <reserved_102> 你觉得那块神秘的绿铜有什么特殊的地方吗？ <reserved_103>  绿铜被认为是一件强大的至宝，但对我来说似乎没有任何作用。</s>\u001b[0m\n",
      "\u001b[34mlabel_ids:\u001b[0m\n",
      "\u001b[34m[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 31106, 32094, 32991, 31395, 4321, 3068, 31449, 18534, 31475, 31729, 72, 31354, 18218, 3876, 12434, 19508, 3942, 73, 2]\u001b[0m\n",
      "\u001b[34mlabels:\u001b[0m\n",
      "\u001b[34m<unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk> 绿铜被认为是一件强大的至宝，但对我来说似乎没有任何作用。</s>\u001b[0m\n",
      "\u001b[34mNCCL version 2.14.3+cuda11.7\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:   0%|          | 0/11264 [00:00<?, ? examples/s]#015Running tokenizer on dataset:   0%|          | 0/11264 [00:00<?, ? examples/s]#015Running tokenizer on dataset:   0%|          | 0/11264 [00:00<?, ? examples/s]#015Running tokenizer on dataset:   0%|          | 0/11264 [00:00<?, ? examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:   0%|          | 0/11264 [00:00<?, ? examples/s]#015Running tokenizer on dataset:   0%|          | 0/11264 [00:00<?, ? examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:   0%|          | 0/11264 [00:00<?, ? examples/s]\u001b[0m\n",
      "\u001b[34m[INFO|deepspeed.py:291] 2023-07-20 06:06:12,406 >> Detected ZeRO Offload and non-DeepSpeed optimizers: This combination should work as long as the custom optimizer has both CPU and GPU implementation (except LAMB)\u001b[0m\n",
      "\u001b[34m#015Running tokenizer on dataset:   9%|▉         | 1000/11264 [00:00<00:02, 4716.12 examples/s]\u001b[0m\n",
      "\u001b[34m[INFO|deepspeed.py:291] 2023-07-20 06:06:12,406 >> Detected ZeRO Offload and non-DeepSpeed optimizers: This combination should work as long as the custom optimizer has both CPU and GPU implementation (except LAMB)\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:   9%|▉         | 1000/11264 [00:00<00:02, 4700.39 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:   9%|▉         | 1000/11264 [00:00<00:02, 4694.17 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:   9%|▉         | 1000/11264 [00:00<00:02, 4685.72 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:   9%|▉         | 1000/11264 [00:00<00:02, 4678.27 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:   9%|▉         | 1000/11264 [00:00<00:02, 4674.06 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:   9%|▉         | 1000/11264 [00:00<00:02, 4592.42 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  18%|█▊        | 2000/11264 [00:00<00:01, 4896.97 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  18%|█▊        | 2000/11264 [00:00<00:01, 4886.67 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  18%|█▊        | 2000/11264 [00:00<00:01, 4870.42 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  18%|█▊        | 2000/11264 [00:00<00:01, 4856.14 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  18%|█▊        | 2000/11264 [00:00<00:01, 4847.36 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  18%|█▊        | 2000/11264 [00:00<00:01, 4757.41 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  18%|█▊        | 2000/11264 [00:00<00:02, 4413.55 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  27%|██▋       | 3000/11264 [00:00<00:01, 4992.85 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  27%|██▋       | 3000/11264 [00:00<00:01, 4976.32 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  27%|██▋       | 3000/11264 [00:00<00:01, 4965.03 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  27%|██▋       | 3000/11264 [00:00<00:01, 4943.22 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  27%|██▋       | 3000/11264 [00:00<00:01, 4937.55 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  27%|██▋       | 3000/11264 [00:00<00:01, 4849.04 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  27%|██▋       | 3000/11264 [00:00<00:01, 4690.12 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  36%|███▌      | 4000/11264 [00:00<00:01, 5048.00 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  36%|███▌      | 4000/11264 [00:00<00:01, 5031.72 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  36%|███▌      | 4000/11264 [00:00<00:01, 5017.57 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  36%|███▌      | 4000/11264 [00:00<00:01, 5002.37 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  36%|███▌      | 4000/11264 [00:00<00:01, 4994.50 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  36%|███▌      | 4000/11264 [00:00<00:01, 4904.36 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  36%|███▌      | 4000/11264 [00:00<00:01, 4841.57 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  44%|████▍     | 5000/11264 [00:00<00:01, 5234.07 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  44%|████▍     | 5000/11264 [00:00<00:01, 5209.31 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  44%|████▍     | 5000/11264 [00:00<00:01, 5192.02 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  44%|████▍     | 5000/11264 [00:00<00:01, 5178.81 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  44%|████▍     | 5000/11264 [00:00<00:01, 5170.46 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  44%|████▍     | 5000/11264 [00:01<00:01, 5075.16 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  44%|████▍     | 5000/11264 [00:01<00:01, 5070.28 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  53%|█████▎    | 6000/11264 [00:01<00:01, 3576.96 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  53%|█████▎    | 6000/11264 [00:01<00:01, 3554.22 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  53%|█████▎    | 6000/11264 [00:01<00:01, 3557.05 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  53%|█████▎    | 6000/11264 [00:01<00:01, 3536.75 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  53%|█████▎    | 6000/11264 [00:01<00:01, 3531.74 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  53%|█████▎    | 6000/11264 [00:01<00:01, 3475.69 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  53%|█████▎    | 6000/11264 [00:01<00:01, 3504.12 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  62%|██████▏   | 7000/11264 [00:01<00:01, 2698.67 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  62%|██████▏   | 7000/11264 [00:02<00:01, 2670.08 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  62%|██████▏   | 7000/11264 [00:02<00:01, 2657.29 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  62%|██████▏   | 7000/11264 [00:02<00:01, 2645.23 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  62%|██████▏   | 7000/11264 [00:02<00:01, 2636.59 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  62%|██████▏   | 7000/11264 [00:02<00:01, 2614.44 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  62%|██████▏   | 7000/11264 [00:02<00:01, 2638.02 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  71%|███████   | 8000/11264 [00:02<00:01, 2278.70 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  71%|███████   | 8000/11264 [00:02<00:01, 2274.73 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  71%|███████   | 8000/11264 [00:02<00:01, 2270.59 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  71%|███████   | 8000/11264 [00:02<00:01, 2260.94 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  71%|███████   | 8000/11264 [00:02<00:01, 2253.86 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  71%|███████   | 8000/11264 [00:02<00:01, 2259.82 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  71%|███████   | 8000/11264 [00:02<00:01, 2240.49 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  80%|███████▉  | 9000/11264 [00:03<00:01, 2090.26 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  80%|███████▉  | 9000/11264 [00:03<00:01, 2079.05 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  80%|███████▉  | 9000/11264 [00:03<00:01, 2072.38 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  80%|███████▉  | 9000/11264 [00:03<00:01, 2061.92 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  80%|███████▉  | 9000/11264 [00:03<00:01, 2057.31 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  80%|███████▉  | 9000/11264 [00:03<00:01, 2047.31 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  80%|███████▉  | 9000/11264 [00:03<00:01, 2042.48 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  89%|████████▉ | 10000/11264 [00:03<00:00, 1961.44 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  89%|████████▉ | 10000/11264 [00:03<00:00, 1956.95 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  89%|████████▉ | 10000/11264 [00:03<00:00, 1958.86 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  89%|████████▉ | 10000/11264 [00:03<00:00, 1948.13 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  89%|████████▉ | 10000/11264 [00:03<00:00, 1944.19 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  89%|████████▉ | 10000/11264 [00:03<00:00, 1935.15 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  89%|████████▉ | 10000/11264 [00:03<00:00, 1924.50 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  98%|█████████▊| 11000/11264 [00:04<00:00, 1888.66 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  98%|█████████▊| 11000/11264 [00:04<00:00, 1885.71 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  98%|█████████▊| 11000/11264 [00:04<00:00, 1873.77 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  98%|█████████▊| 11000/11264 [00:04<00:00, 1876.03 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  98%|█████████▊| 11000/11264 [00:04<00:00, 1873.63 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  98%|█████████▊| 11000/11264 [00:04<00:00, 1863.93 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  98%|█████████▊| 11000/11264 [00:04<00:00, 1835.97 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset: 100%|██████████| 11264/11264 [00:04<00:00, 1863.94 examples/s]\u001b[0m\n",
      "\u001b[34minput_ids:\u001b[0m\n",
      "\u001b[34m[99, 195, 9875, 3137, 31379, 32020, 17324, 31135, 32094, 32991, 7713, 7308, 7654, 31763, 75, 31106, 196, 31106, 31106, 32094, 32991, 31395, 4321, 3068, 31449, 18534, 31475, 31729, 72, 31354, 18218, 3876, 12434, 19508, 3942, 73, 2]\u001b[0m\n",
      "\u001b[34minputs:\n",
      "  <reserved_102> 你觉得那块神秘的绿铜有什么特殊的地方吗？ <reserved_103>  绿铜被认为是一件强大的至宝，但对我来说似乎没有任何作用。</s>\u001b[0m\n",
      "\u001b[34mlabel_ids:\u001b[0m\n",
      "\u001b[34m[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 31106, 32094, 32991, 31395, 4321, 3068, 31449, 18534, 31475, 31729, 72, 31354, 18218, 3876, 12434, 19508, 3942, 73, 2]\u001b[0m\n",
      "\u001b[34mlabels:\u001b[0m\n",
      "\u001b[34m<unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk> 绿铜被认为是一件强大的至宝，但对我来说似乎没有任何作用。</s>\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset: 100%|██████████| 11264/11264 [00:04<00:00, 1861.85 examples/s]\u001b[0m\n",
      "\u001b[34minput_ids:\u001b[0m\n",
      "\u001b[34m[99, 195, 9875, 3137, 31379, 32020, 17324, 31135, 32094, 32991, 7713, 7308, 7654, 31763, 75, 31106, 196, 31106, 31106, 32094, 32991, 31395, 4321, 3068, 31449, 18534, 31475, 31729, 72, 31354, 18218, 3876, 12434, 19508, 3942, 73, 2]\u001b[0m\n",
      "\u001b[34minputs:\n",
      "  <reserved_102> 你觉得那块神秘的绿铜有什么特殊的地方吗？ <reserved_103>  绿铜被认为是一件强大的至宝，但对我来说似乎没有任何作用。</s>\u001b[0m\n",
      "\u001b[34mlabel_ids:\u001b[0m\n",
      "\u001b[34m[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 31106, 32094, 32991, 31395, 4321, 3068, 31449, 18534, 31475, 31729, 72, 31354, 18218, 3876, 12434, 19508, 3942, 73, 2]\u001b[0m\n",
      "\u001b[34mlabels:\u001b[0m\n",
      "\u001b[34m<unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk> 绿铜被认为是一件强大的至宝，但对我来说似乎没有任何作用。</s>\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset: 100%|██████████| 11264/11264 [00:04<00:00, 1848.56 examples/s]\u001b[0m\n",
      "\u001b[34minput_ids:\u001b[0m\n",
      "\u001b[34m[99, 195, 9875, 3137, 31379, 32020, 17324, 31135, 32094, 32991, 7713, 7308, 7654, 31763, 75, 31106, 196, 31106, 31106, 32094, 32991, 31395, 4321, 3068, 31449, 18534, 31475, 31729, 72, 31354, 18218, 3876, 12434, 19508, 3942, 73, 2]\u001b[0m\n",
      "\u001b[34minputs:\n",
      "  <reserved_102> 你觉得那块神秘的绿铜有什么特殊的地方吗？ <reserved_103>  绿铜被认为是一件强大的至宝，但对我来说似乎没有任何作用。</s>\u001b[0m\n",
      "\u001b[34mlabel_ids:\u001b[0m\n",
      "\u001b[34m[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 31106, 32094, 32991, 31395, 4321, 3068, 31449, 18534, 31475, 31729, 72, 31354, 18218, 3876, 12434, 19508, 3942, 73, 2]\u001b[0m\n",
      "\u001b[34mlabels:\u001b[0m\n",
      "\u001b[34m<unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk> 绿铜被认为是一件强大的至宝，但对我来说似乎没有任何作用。</s>\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset: 100%|██████████| 11264/11264 [00:04<00:00, 1852.61 examples/s]\u001b[0m\n",
      "\u001b[34minput_ids:\u001b[0m\n",
      "\u001b[34m[99, 195, 9875, 3137, 31379, 32020, 17324, 31135, 32094, 32991, 7713, 7308, 7654, 31763, 75, 31106, 196, 31106, 31106, 32094, 32991, 31395, 4321, 3068, 31449, 18534, 31475, 31729, 72, 31354, 18218, 3876, 12434, 19508, 3942, 73, 2]\u001b[0m\n",
      "\u001b[34minputs:\n",
      "  <reserved_102> 你觉得那块神秘的绿铜有什么特殊的地方吗？ <reserved_103>  绿铜被认为是一件强大的至宝，但对我来说似乎没有任何作用。</s>\u001b[0m\n",
      "\u001b[34mlabel_ids:\u001b[0m\n",
      "\u001b[34m[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 31106, 32094, 32991, 31395, 4321, 3068, 31449, 18534, 31475, 31729, 72, 31354, 18218, 3876, 12434, 19508, 3942, 73, 2]\u001b[0m\n",
      "\u001b[34mlabels:\u001b[0m\n",
      "\u001b[34m<unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk> 绿铜被认为是一件强大的至宝，但对我来说似乎没有任何作用。</s>\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset: 100%|██████████| 11264/11264 [00:04<00:00, 1850.14 examples/s]\u001b[0m\n",
      "\u001b[34minput_ids:\u001b[0m\n",
      "\u001b[34m[99, 195, 9875, 3137, 31379, 32020, 17324, 31135, 32094, 32991, 7713, 7308, 7654, 31763, 75, 31106, 196, 31106, 31106, 32094, 32991, 31395, 4321, 3068, 31449, 18534, 31475, 31729, 72, 31354, 18218, 3876, 12434, 19508, 3942, 73, 2]\u001b[0m\n",
      "\u001b[34minputs:\n",
      "  <reserved_102> 你觉得那块神秘的绿铜有什么特殊的地方吗？ <reserved_103>  绿铜被认为是一件强大的至宝，但对我来说似乎没有任何作用。</s>\u001b[0m\n",
      "\u001b[34mlabel_ids:\u001b[0m\n",
      "\u001b[34m[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 31106, 32094, 32991, 31395, 4321, 3068, 31449, 18534, 31475, 31729, 72, 31354, 18218, 3876, 12434, 19508, 3942, 73, 2]\u001b[0m\n",
      "\u001b[34mlabels:\u001b[0m\n",
      "\u001b[34m<unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk> 绿铜被认为是一件强大的至宝，但对我来说似乎没有任何作用。</s>\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset: 100%|██████████| 11264/11264 [00:04<00:00, 1841.05 examples/s]\u001b[0m\n",
      "\u001b[34minput_ids:\u001b[0m\n",
      "\u001b[34m[99, 195, 9875, 3137, 31379, 32020, 17324, 31135, 32094, 32991, 7713, 7308, 7654, 31763, 75, 31106, 196, 31106, 31106, 32094, 32991, 31395, 4321, 3068, 31449, 18534, 31475, 31729, 72, 31354, 18218, 3876, 12434, 19508, 3942, 73, 2]\u001b[0m\n",
      "\u001b[34minputs:\n",
      "  <reserved_102> 你觉得那块神秘的绿铜有什么特殊的地方吗？ <reserved_103>  绿铜被认为是一件强大的至宝，但对我来说似乎没有任何作用。</s>\u001b[0m\n",
      "\u001b[34mlabel_ids:\u001b[0m\n",
      "\u001b[34m[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 31106, 32094, 32991, 31395, 4321, 3068, 31449, 18534, 31475, 31729, 72, 31354, 18218, 3876, 12434, 19508, 3942, 73, 2]\u001b[0m\n",
      "\u001b[34mlabels:\u001b[0m\n",
      "\u001b[34m<unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk> 绿铜被认为是一件强大的至宝，但对我来说似乎没有任何作用。</s>\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset: 100%|██████████| 11264/11264 [00:04<00:00, 1817.13 examples/s]\u001b[0m\n",
      "\u001b[34minput_ids:\u001b[0m\n",
      "\u001b[34m[99, 195, 9875, 3137, 31379, 32020, 17324, 31135, 32094, 32991, 7713, 7308, 7654, 31763, 75, 31106, 196, 31106, 31106, 32094, 32991, 31395, 4321, 3068, 31449, 18534, 31475, 31729, 72, 31354, 18218, 3876, 12434, 19508, 3942, 73, 2]\u001b[0m\n",
      "\u001b[34minputs:\n",
      "  <reserved_102> 你觉得那块神秘的绿铜有什么特殊的地方吗？ <reserved_103>  绿铜被认为是一件强大的至宝，但对我来说似乎没有任何作用。</s>\u001b[0m\n",
      "\u001b[34mlabel_ids:\u001b[0m\n",
      "\u001b[34m[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 31106, 32094, 32991, 31395, 4321, 3068, 31449, 18534, 31475, 31729, 72, 31354, 18218, 3876, 12434, 19508, 3942, 73, 2]\u001b[0m\n",
      "\u001b[34mlabels:\u001b[0m\n",
      "\u001b[34m<unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk> 绿铜被认为是一件强大的至宝，但对我来说似乎没有任何作用。</s>\u001b[0m\n",
      "\u001b[34mUsing /root/.cache/torch_extensions/py39_cu117 as PyTorch extensions root...\u001b[0m\n",
      "\u001b[34mCreating extension directory /root/.cache/torch_extensions/py39_cu117/cpu_adam...\u001b[0m\n",
      "\u001b[34mDetected CUDA files, patching ldflags\u001b[0m\n",
      "\u001b[34mEmitting ninja build file /root/.cache/torch_extensions/py39_cu117/cpu_adam/build.ninja...\u001b[0m\n",
      "\u001b[34mBuilding extension module cpu_adam...\u001b[0m\n",
      "\u001b[34mAllowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N)\u001b[0m\n",
      "\u001b[34mUsing /root/.cache/torch_extensions/py39_cu117 as PyTorch extensions root...\u001b[0m\n",
      "\u001b[34mUsing /root/.cache/torch_extensions/py39_cu117 as PyTorch extensions root...\u001b[0m\n",
      "\u001b[34mUsing /root/.cache/torch_extensions/py39_cu117 as PyTorch extensions root...\u001b[0m\n",
      "\u001b[34mUsing /root/.cache/torch_extensions/py39_cu117 as PyTorch extensions root...\u001b[0m\n",
      "\u001b[34mUsing /root/.cache/torch_extensions/py39_cu117 as PyTorch extensions root...\u001b[0m\n",
      "\u001b[34mUsing /root/.cache/torch_extensions/py39_cu117 as PyTorch extensions root...\u001b[0m\n",
      "\u001b[34mUsing /root/.cache/torch_extensions/py39_cu117 as PyTorch extensions root...\u001b[0m\n",
      "\u001b[34m[1/3] /usr/local/cuda/bin/nvcc  -DTORCH_EXTENSION_NAME=cpu_adam -DTORCH_API_INCLUDE_EXTENSION_H -DPYBIND11_COMPILER_TYPE=\\\"_gcc\\\" -DPYBIND11_STDLIB=\\\"_libstdcpp\\\" -DPYBIND11_BUILD_ABI=\\\"_cxxabi1011\\\" -I/opt/conda/lib/python3.9/site-packages/deepspeed/ops/csrc/includes -I/usr/local/cuda/include -isystem /opt/conda/lib/python3.9/site-packages/torch/include -isystem /opt/conda/lib/python3.9/site-packages/torch/include/torch/csrc/api/include -isystem /opt/conda/lib/python3.9/site-packages/torch/include/TH -isystem /opt/conda/lib/python3.9/site-packages/torch/include/THC -isystem /usr/local/cuda/include -isystem /opt/conda/include/python3.9 -D_GLIBCXX_USE_CXX11_ABI=0 -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_BFLOAT16_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --expt-relaxed-constexpr -gencode=arch=compute_80,code=compute_80 -gencode=arch=compute_80,code=sm_80 --compiler-options '-fPIC' -O3 --use_fast_math -std=c++17 -U__CUDA_NO_HALF_OPERATORS__ -U__CUDA_NO_HALF_CONVERSIONS__ -U__CUDA_NO_HALF2_OPERATORS__ -gencode=arch=compute_80,code=sm_80 -gencode=arch=compute_80,code=compute_80 -DBF16_AVAILABLE -c /opt/conda/lib/python3.9/site-packages/deepspeed/ops/csrc/common/custom_cuda_kernel.cu -o custom_cuda_kernel.cuda.o\u001b[0m\n",
      "\u001b[34m[2/3] c++ -MMD -MF cpu_adam.o.d -DTORCH_EXTENSION_NAME=cpu_adam -DTORCH_API_INCLUDE_EXTENSION_H -DPYBIND11_COMPILER_TYPE=\\\"_gcc\\\" -DPYBIND11_STDLIB=\\\"_libstdcpp\\\" -DPYBIND11_BUILD_ABI=\\\"_cxxabi1011\\\" -I/opt/conda/lib/python3.9/site-packages/deepspeed/ops/csrc/includes -I/usr/local/cuda/include -isystem /opt/conda/lib/python3.9/site-packages/torch/include -isystem /opt/conda/lib/python3.9/site-packages/torch/include/torch/csrc/api/include -isystem /opt/conda/lib/python3.9/site-packages/torch/include/TH -isystem /opt/conda/lib/python3.9/site-packages/torch/include/THC -isystem /usr/local/cuda/include -isystem /opt/conda/include/python3.9 -D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -std=c++14 -O3 -std=c++17 -g -Wno-reorder -L/usr/local/cuda/lib64 -lcudart -lcublas -g -march=native -fopenmp -D__AVX512__ -D__ENABLE_CUDA__ -DBF16_AVAILABLE -c /opt/conda/lib/python3.9/site-packages/deepspeed/ops/csrc/adam/cpu_adam.cpp -o cpu_adam.o\u001b[0m\n",
      "\u001b[34m[3/3] c++ cpu_adam.o custom_cuda_kernel.cuda.o -shared -lcurand -L/opt/conda/lib/python3.9/site-packages/torch/lib -lc10 -lc10_cuda -ltorch_cpu -ltorch_cuda_cu -ltorch_cuda_cpp -ltorch -ltorch_python -L/usr/local/cuda/lib64 -lcudart -o cpu_adam.so\u001b[0m\n",
      "\u001b[34mLoading extension module cpu_adam...\u001b[0m\n",
      "\u001b[34mTime to load cpu_adam op: 34.65175247192383 seconds\u001b[0m\n",
      "\u001b[34mLoading extension module cpu_adam...\u001b[0m\n",
      "\u001b[34mTime to load cpu_adam op: 30.35076594352722 seconds\u001b[0m\n",
      "\u001b[34mLoading extension module cpu_adam...\u001b[0m\n",
      "\u001b[34mTime to load cpu_adam op: 30.060192108154297 seconds\u001b[0m\n",
      "\u001b[34mLoading extension module cpu_adam...\u001b[0m\n",
      "\u001b[34mTime to load cpu_adam op: 30.048989295959473 seconds\u001b[0m\n",
      "\u001b[34mLoading extension module cpu_adam...\u001b[0m\n",
      "\u001b[34mLoading extension module cpu_adam...\u001b[0m\n",
      "\u001b[34mTime to load cpu_adam op: 30.35587215423584 seconds\u001b[0m\n",
      "\u001b[34mTime to load cpu_adam op: 30.294903993606567 seconds\u001b[0m\n",
      "\u001b[34mLoading extension module cpu_adam...\u001b[0m\n",
      "\u001b[34mTime to load cpu_adam op: 30.541180849075317 seconds\u001b[0m\n",
      "\u001b[34mLoading extension module cpu_adam...\u001b[0m\n",
      "\u001b[34mTime to load cpu_adam op: 30.23192858695984 seconds\u001b[0m\n",
      "\u001b[34mAdam Optimizer #0 is created with AVX512 arithmetic capability.\u001b[0m\n",
      "\u001b[34mConfig: alpha=0.000300, betas=(0.900000, 0.999000), weight_decay=0.010000, adam_w=1\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:06:53,010] [INFO] [logging.py:96:log_dist] [Rank 0] DeepSpeed info: version=0.10.0, git-hash=unknown, git-branch=unknown\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:03,581] [INFO] [logging.py:96:log_dist] [Rank 0] DeepSpeed Flops Profiler Enabled: False\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:03,586] [INFO] [logging.py:96:log_dist] [Rank 0] Removing param_group that has no 'params' in the client Optimizer\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:03,586] [INFO] [logging.py:96:log_dist] [Rank 0] Using client Optimizer as basic optimizer\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:03,621] [INFO] [logging.py:96:log_dist] [Rank 0] DeepSpeed Basic Optimizer = DeepSpeedCPUAdam\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:03,621] [INFO] [utils.py:54:is_zero_supported_optimizer] Checking ZeRO support for optimizer=DeepSpeedCPUAdam type=<class 'deepspeed.ops.adam.cpu_adam.DeepSpeedCPUAdam'>\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:03,621] [INFO] [logging.py:96:log_dist] [Rank 0] Creating torch.float16 ZeRO stage 2 optimizer\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:03,621] [INFO] [stage_1_and_2.py:133:__init__] Reduce bucket size 500,000,000\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:03,621] [INFO] [stage_1_and_2.py:134:__init__] Allgather bucket size 500,000,000\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:03,621] [INFO] [stage_1_and_2.py:135:__init__] CPU Offload: True\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:03,621] [INFO] [stage_1_and_2.py:136:__init__] Round robin gradient partitioning: False\u001b[0m\n",
      "\u001b[34mRank: 3 partition count [8] and sizes[(27893760, False)]\u001b[0m\n",
      "\u001b[34mRank: 4 partition count [8] and sizes[(27893760, False)]\u001b[0m\n",
      "\u001b[34mRank: 7 partition count [8] and sizes[(27893760, False)]\u001b[0m\n",
      "\u001b[34mRank: 6 partition count [8] and sizes[(27893760, False)]\u001b[0m\n",
      "\u001b[34mRank: 1 partition count [8] and sizes[(27893760, False)]\u001b[0m\n",
      "\u001b[34mRank: 0 partition count [8] and sizes[(27893760, False)]\u001b[0m\n",
      "\u001b[34mRank: 5 partition count [8] and sizes[(27893760, False)]\u001b[0m\n",
      "\u001b[34mRank: 2 partition count [8] and sizes[(27893760, False)]\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:04,793] [INFO] [utils.py:785:see_memory_usage] Before initializing optimizer states\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:04,794] [INFO] [utils.py:786:see_memory_usage] MA 25.16 GB         Max_MA 25.16 GB         CA 25.19 GB         Max_CA 25 GB\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:04,794] [INFO] [utils.py:793:see_memory_usage] CPU Virtual Memory:  used = 68.22 GB, percent = 6.1%\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,253] [INFO] [utils.py:785:see_memory_usage] After initializing optimizer states\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,253] [INFO] [utils.py:786:see_memory_usage] MA 25.16 GB         Max_MA 25.16 GB         CA 25.19 GB         Max_CA 25 GB\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,253] [INFO] [utils.py:793:see_memory_usage] CPU Virtual Memory:  used = 67.83 GB, percent = 6.0%\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,253] [INFO] [stage_1_and_2.py:493:__init__] optimizer state initialized\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,423] [INFO] [utils.py:785:see_memory_usage] After initializing ZeRO optimizer\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,423] [INFO] [utils.py:786:see_memory_usage] MA 25.16 GB         Max_MA 25.16 GB         CA 25.19 GB         Max_CA 25 GB\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,423] [INFO] [utils.py:793:see_memory_usage] CPU Virtual Memory:  used = 67.82 GB, percent = 6.0%\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,426] [INFO] [logging.py:96:log_dist] [Rank 0] DeepSpeed Final Optimizer = DeepSpeedCPUAdam\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,426] [INFO] [logging.py:96:log_dist] [Rank 0] DeepSpeed using client LR scheduler\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,426] [INFO] [logging.py:96:log_dist] [Rank 0] DeepSpeed LR Scheduler = None\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,426] [INFO] [logging.py:96:log_dist] [Rank 0] step=0, skipped=0, lr=[0.0003], mom=[(0.9, 0.999)]\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,429] [INFO] [config.py:960:print] DeepSpeedEngine configuration:\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,430] [INFO] [config.py:964:print]   activation_checkpointing_config  {\n",
      "    \"partition_activations\": false, \n",
      "    \"contiguous_memory_optimization\": false, \n",
      "    \"cpu_checkpointing\": false, \n",
      "    \"number_checkpoints\": null, \n",
      "    \"synchronize_checkpoint_boundary\": false, \n",
      "    \"profile\": false\u001b[0m\n",
      "\u001b[34m}\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,430] [INFO] [config.py:964:print]   aio_config ................... {'block_size': 1048576, 'queue_depth': 8, 'thread_count': 1, 'single_submit': False, 'overlap_events': True}\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,430] [INFO] [config.py:964:print]   amp_enabled .................. False\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,430] [INFO] [config.py:964:print]   amp_params ................... False\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,430] [INFO] [config.py:964:print]   autotuning_config ............ {\n",
      "    \"enabled\": false, \n",
      "    \"start_step\": null, \n",
      "    \"end_step\": null, \n",
      "    \"metric_path\": null, \n",
      "    \"arg_mappings\": null, \n",
      "    \"metric\": \"throughput\", \n",
      "    \"model_info\": null, \n",
      "    \"results_dir\": \"autotuning_results\", \n",
      "    \"exps_dir\": \"autotuning_exps\", \n",
      "    \"overwrite\": true, \n",
      "    \"fast\": true, \n",
      "    \"start_profile_step\": 3, \n",
      "    \"end_profile_step\": 5, \n",
      "    \"tuner_type\": \"gridsearch\", \n",
      "    \"tuner_early_stopping\": 5, \n",
      "    \"tuner_num_trials\": 50, \n",
      "    \"model_info_path\": null, \n",
      "    \"mp_size\": 1, \n",
      "    \"max_train_batch_size\": null, \n",
      "    \"min_train_batch_size\": 1, \n",
      "    \"max_train_micro_batch_size_per_gpu\": 1.024000e+03, \n",
      "    \"min_train_micro_batch_size_per_gpu\": 1, \n",
      "    \"num_tuning_micro_batch_sizes\": 3\u001b[0m\n",
      "\u001b[34m}\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,430] [INFO] [config.py:964:print]   bfloat16_enabled ............. False\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,430] [INFO] [config.py:964:print]   checkpoint_parallel_write_pipeline  False\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,430] [INFO] [config.py:964:print]   checkpoint_tag_validation_enabled  True\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,430] [INFO] [config.py:964:print]   checkpoint_tag_validation_fail  False\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,430] [INFO] [config.py:964:print]   comms_config ................. <deepspeed.comm.config.DeepSpeedCommsConfig object at 0x7f16580b4100>\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,430] [INFO] [config.py:964:print]   communication_data_type ...... None\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,430] [INFO] [config.py:964:print]   compression_config ........... {'weight_quantization': {'shared_parameters': {'enabled': False, 'quantizer_kernel': False, 'schedule_offset': 0, 'quantize_groups': 1, 'quantize_verbose': False, 'quantization_type': 'symmetric', 'quantize_weight_in_forward': False, 'rounding': 'nearest', 'fp16_mixed_quantize': False, 'quantize_change_ratio': 0.001}, 'different_groups': {}}, 'activation_quantization': {'shared_parameters': {'enabled': False, 'quantization_type': 'symmetric', 'range_calibration': 'dynamic', 'schedule_offset': 1000}, 'different_groups': {}}, 'sparse_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'row_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'head_pruning': {'shared_parameters': {'enabled': False, 'method': 'topk', 'schedule_offset': 1000}, 'different_groups': {}}, 'channel_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'layer_reduction': {'enabled': False}}\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,430] [INFO] [config.py:964:print]   curriculum_enabled_legacy .... False\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,430] [INFO] [config.py:964:print]   curriculum_params_legacy ..... False\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,430] [INFO] [config.py:964:print]   data_efficiency_config ....... {'enabled': False, 'seed': 1234, 'data_sampling': {'enabled': False, 'num_epochs': 1000, 'num_workers': 0, 'curriculum_learning': {'enabled': False}}, 'data_routing': {'enabled': False, 'random_ltd': {'enabled': False, 'layer_token_lr_schedule': {'enabled': False}}}}\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,430] [INFO] [config.py:964:print]   data_efficiency_enabled ...... False\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,430] [INFO] [config.py:964:print]   dataloader_drop_last ......... False\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,430] [INFO] [config.py:964:print]   disable_allgather ............ False\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,430] [INFO] [config.py:964:print]   dump_state ................... False\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,430] [INFO] [config.py:964:print]   dynamic_loss_scale_args ...... None\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,431] [INFO] [config.py:964:print]   eigenvalue_enabled ........... False\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,431] [INFO] [config.py:964:print]   eigenvalue_gas_boundary_resolution  1\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,431] [INFO] [config.py:964:print]   eigenvalue_layer_name ........ bert.encoder.layer\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,431] [INFO] [config.py:964:print]   eigenvalue_layer_num ......... 0\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,431] [INFO] [config.py:964:print]   eigenvalue_max_iter .......... 100\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,431] [INFO] [config.py:964:print]   eigenvalue_stability ......... 1e-06\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,431] [INFO] [config.py:964:print]   eigenvalue_tol ............... 0.01\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,431] [INFO] [config.py:964:print]   eigenvalue_verbose ........... False\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,431] [INFO] [config.py:964:print]   elasticity_enabled ........... False\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,431] [INFO] [config.py:964:print]   flops_profiler_config ........ {\n",
      "    \"enabled\": false, \n",
      "    \"recompute_fwd_factor\": 0.0, \n",
      "    \"profile_step\": 1, \n",
      "    \"module_depth\": -1, \n",
      "    \"top_modules\": 1, \n",
      "    \"detailed\": true, \n",
      "    \"output_file\": null\u001b[0m\n",
      "\u001b[34m}\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,431] [INFO] [config.py:964:print]   fp16_auto_cast ............... True\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,431] [INFO] [config.py:964:print]   fp16_enabled ................. True\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,431] [INFO] [config.py:964:print]   fp16_master_weights_and_gradients  False\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,431] [INFO] [config.py:964:print]   global_rank .................. 0\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,431] [INFO] [config.py:964:print]   grad_accum_dtype ............. None\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,431] [INFO] [config.py:964:print]   gradient_accumulation_steps .. 2\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,431] [INFO] [config.py:964:print]   gradient_clipping ............ 1.0\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,431] [INFO] [config.py:964:print]   gradient_predivide_factor .... 1.0\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,431] [INFO] [config.py:964:print]   hybrid_engine ................ enabled=False max_out_tokens=512 inference_tp_size=1 release_inference_cache=False pin_parameters=True tp_gather_partition_size=8\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,431] [INFO] [config.py:964:print]   initial_dynamic_scale ........ 65536\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,431] [INFO] [config.py:964:print]   load_universal_checkpoint .... False\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,431] [INFO] [config.py:964:print]   loss_scale ................... 0\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,431] [INFO] [config.py:964:print]   memory_breakdown ............. False\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,431] [INFO] [config.py:964:print]   mics_hierarchial_params_gather  False\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,431] [INFO] [config.py:964:print]   mics_shard_size .............. -1\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,431] [INFO] [config.py:964:print]   monitor_config ............... tensorboard=TensorBoardConfig(enabled=False, output_path='', job_name='DeepSpeedJobName') wandb=WandbConfig(enabled=False, group=None, team=None, project='deepspeed') csv_monitor=CSVConfig(enabled=False, output_path='', job_name='DeepSpeedJobName') enabled=False\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,431] [INFO] [config.py:964:print]   nebula_config ................ {\n",
      "    \"enabled\": false, \n",
      "    \"persistent_storage_path\": null, \n",
      "    \"persistent_time_interval\": 100, \n",
      "    \"num_of_version_in_retention\": 2, \n",
      "    \"enable_nebula_load\": true, \n",
      "    \"load_path\": null\u001b[0m\n",
      "\u001b[34m}\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,431] [INFO] [config.py:964:print]   optimizer_legacy_fusion ...... False\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,431] [INFO] [config.py:964:print]   optimizer_name ............... None\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,431] [INFO] [config.py:964:print]   optimizer_params ............. None\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,431] [INFO] [config.py:964:print]   pipeline ..................... {'stages': 'auto', 'partition': 'best', 'seed_layers': False, 'activation_checkpoint_interval': 0}\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,431] [INFO] [config.py:964:print]   pld_enabled .................. False\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,431] [INFO] [config.py:964:print]   pld_params ................... False\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,431] [INFO] [config.py:964:print]   prescale_gradients ........... False\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,431] [INFO] [config.py:964:print]   scheduler_name ............... None\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,431] [INFO] [config.py:964:print]   scheduler_params ............. None\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,431] [INFO] [config.py:964:print]   sparse_attention ............. None\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,431] [INFO] [config.py:964:print]   sparse_gradients_enabled ..... False\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,431] [INFO] [config.py:964:print]   steps_per_print .............. inf\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,431] [INFO] [config.py:964:print]   train_batch_size ............. 64\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,431] [INFO] [config.py:964:print]   train_micro_batch_size_per_gpu  4\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,431] [INFO] [config.py:964:print]   use_node_local_storage ....... False\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,431] [INFO] [config.py:964:print]   wall_clock_breakdown ......... False\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,431] [INFO] [config.py:964:print]   world_size ................... 8\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,431] [INFO] [config.py:964:print]   zero_allow_untested_optimizer  True\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,431] [INFO] [config.py:964:print]   zero_config .................. stage=2 contiguous_gradients=True reduce_scatter=True reduce_bucket_size=500,000,000 allgather_partitions=True allgather_bucket_size=500,000,000 overlap_comm=False load_from_fp32_weights=True elastic_checkpoint=False offload_param=DeepSpeedZeroOffloadParamConfig(device='cpu', nvme_path=None, buffer_count=5, buffer_size=100,000,000, max_in_cpu=1,000,000,000, pin_memory=False) offload_optimizer=DeepSpeedZeroOffloadOptimizerConfig(device='cpu', nvme_path=None, buffer_count=4, pin_memory=False, pipeline=False, pipeline_read=False, pipeline_write=False, fast_init=False) sub_group_size=1,000,000,000 cpu_offload_param=None cpu_offload_use_pin_memory=None cpu_offload=None prefetch_bucket_size=50,000,000 param_persistence_threshold=100,000 model_persistence_threshold=sys.maxsize max_live_parameters=1,000,000,000 max_reuse_distance=1,000,000,000 gather_16bit_weights_on_model_save=False stage3_gather_fp16_weights_on_model_save=False ignore_unused_parameters=True legacy_stage1=False round_robin_gradients=False zero_hpz_partition_size=1 zero_quantized_weights=False zero_quantized_gradients=False mics_shard_size=-1 mics_hierarchical_params_gather=False memory_efficient_linear=True\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,431] [INFO] [config.py:964:print]   zero_enabled ................. True\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,432] [INFO] [config.py:964:print]   zero_force_ds_cpu_optimizer .. True\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,432] [INFO] [config.py:964:print]   zero_optimization_stage ...... 2\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:05,432] [INFO] [config.py:950:print_user_config]   json = {\n",
      "    \"train_batch_size\": 64, \n",
      "    \"train_micro_batch_size_per_gpu\": 4, \n",
      "    \"gradient_accumulation_steps\": 2, \n",
      "    \"zero_optimization\": {\n",
      "        \"stage\": 2, \n",
      "        \"offload_optimizer\": {\n",
      "            \"device\": \"cpu\", \n",
      "            \"nvme_path\": null\n",
      "        }, \n",
      "        \"offload_param\": {\n",
      "            \"device\": \"cpu\", \n",
      "            \"nvme_path\": null\n",
      "        }, \n",
      "        \"stage3_gather_16bit_weights_on_model_save\": false\n",
      "    }, \n",
      "    \"gradient_clipping\": 1.0, \n",
      "    \"steps_per_print\": inf, \n",
      "    \"fp16\": {\n",
      "        \"enabled\": true, \n",
      "        \"auto_cast\": true\n",
      "    }, \n",
      "    \"bf16\": {\n",
      "        \"enabled\": false\n",
      "    }, \n",
      "    \"zero_allow_untested_optimizer\": true\u001b[0m\n",
      "\u001b[34m}\u001b[0m\n",
      "\u001b[34m[INFO|trainer.py:1686] 2023-07-20 06:07:05,432 >> ***** Running training *****\u001b[0m\n",
      "\u001b[34m[INFO|trainer.py:1687] 2023-07-20 06:07:05,432 >>   Num examples = 11,264\u001b[0m\n",
      "\u001b[34m[INFO|trainer.py:1688] 2023-07-20 06:07:05,432 >>   Num Epochs = 2\u001b[0m\n",
      "\u001b[34m[INFO|trainer.py:1689] 2023-07-20 06:07:05,432 >>   Instantaneous batch size per device = 4\u001b[0m\n",
      "\u001b[34m[INFO|trainer.py:1686] 2023-07-20 06:07:05,432 >> ***** Running training *****\u001b[0m\n",
      "\u001b[34m[INFO|trainer.py:1687] 2023-07-20 06:07:05,432 >>   Num examples = 11,264\u001b[0m\n",
      "\u001b[34m[INFO|trainer.py:1688] 2023-07-20 06:07:05,432 >>   Num Epochs = 2\u001b[0m\n",
      "\u001b[34m[INFO|trainer.py:1689] 2023-07-20 06:07:05,432 >>   Instantaneous batch size per device = 4\u001b[0m\n",
      "\u001b[34m[INFO|trainer.py:1692] 2023-07-20 06:07:05,432 >>   Total train batch size (w. parallel, distributed & accumulation) = 64\u001b[0m\n",
      "\u001b[34m[INFO|trainer.py:1692] 2023-07-20 06:07:05,432 >>   Total train batch size (w. parallel, distributed & accumulation) = 64\u001b[0m\n",
      "\u001b[34m[INFO|trainer.py:1693] 2023-07-20 06:07:05,432 >>   Gradient Accumulation steps = 2\u001b[0m\n",
      "\u001b[34m[INFO|trainer.py:1694] 2023-07-20 06:07:05,432 >>   Total optimization steps = 352\u001b[0m\n",
      "\u001b[34m[INFO|trainer.py:1693] 2023-07-20 06:07:05,432 >>   Gradient Accumulation steps = 2\u001b[0m\n",
      "\u001b[34m[INFO|trainer.py:1694] 2023-07-20 06:07:05,432 >>   Total optimization steps = 352\u001b[0m\n",
      "\u001b[34m[INFO|trainer.py:1695] 2023-07-20 06:07:05,436 >>   Number of trainable parameters = 223,150,080\u001b[0m\n",
      "\u001b[34m[INFO|trainer.py:1695] 2023-07-20 06:07:05,436 >>   Number of trainable parameters = 223,150,080\u001b[0m\n",
      "\u001b[34m[INFO|integrations.py:716] 2023-07-20 06:07:05,440 >> Automatic Weights & Biases logging enabled, to disable set os.environ[\"WANDB_DISABLED\"] = \"true\"\u001b[0m\n",
      "\u001b[34m[INFO|integrations.py:716] 2023-07-20 06:07:05,440 >> Automatic Weights & Biases logging enabled, to disable set os.environ[\"WANDB_DISABLED\"] = \"true\"\u001b[0m\n",
      "\u001b[34mgit root error: Cmd('git') failed due to: exit code(128)\n",
      "  cmdline: git rev-parse --show-toplevel\n",
      "  stderr: 'fatal: detected dubious ownership in repository at '/opt/ml/code'\u001b[0m\n",
      "\u001b[34mTo add an exception for this directory, call:\u001b[0m\n",
      "\u001b[34m#011git config --global --add safe.directory /opt/ml/code'\u001b[0m\n",
      "\u001b[34mwandb: Currently logged in as: imtty. Use `wandb login --relogin` to force relogin\u001b[0m\n",
      "\u001b[34mwandb: Tracking run with wandb version 0.15.5\u001b[0m\n",
      "\u001b[34mwandb: Run data is saved locally in /opt/ml/code/wandb/run-20230720_060706-baichuan-finetuning-2023-07-20-05-51-37-413-sqcyuk-algo-1\u001b[0m\n",
      "\u001b[34mwandb: Run `wandb offline` to turn off syncing.\u001b[0m\n",
      "\u001b[34mwandb: Syncing run baichuan-finetuning-2023-07-20-05-51-37-413-sqcyuk-algo-1\u001b[0m\n",
      "\u001b[34mwandb: ⭐️ View project at https://wandb.ai/imtty/sm-baichuan7b-sft\u001b[0m\n",
      "\u001b[34mwandb: 🚀 View run at https://wandb.ai/imtty/sm-baichuan7b-sft/runs/baichuan-finetuning-2023-07-20-05-51-37-413-sqcyuk-algo-1\u001b[0m\n",
      "\u001b[34m0%|          | 0/352 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:16.462: W smdistributed/modelparallel/torch/nn/predefined_hooks.py:78] Found unsupported HuggingFace version 4.31.0 for automated tensor parallelism. HuggingFace modules will not be automatically distributed. You can use smp.tp_register_with_module API to register desired modules for tensor parallelism, or directly instantiate an smp.nn.DistributedModule. Supported HuggingFace transformers versions for automated tensor parallelism: ['4.17.0', '4.20.1', '4.21.0']\u001b[0m\n",
      "\u001b[34mINFO:root:Using NamedTuple = typing._NamedTuple instead.\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:16.490 algo-1:285 INFO utils.py:28] RULE_JOB_STOP_SIGNAL_FILENAME: None\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:16.509 algo-1:285 INFO profiler_config_parser.py:111] Unable to find config at /opt/ml/input/config/profilerconfig.json. Profiler is disabled.\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:16.715: W smdistributed/modelparallel/torch/nn/predefined_hooks.py:78] Found unsupported HuggingFace version 4.31.0 for automated tensor parallelism. HuggingFace modules will not be automatically distributed. You can use smp.tp_register_with_module API to register desired modules for tensor parallelism, or directly instantiate an smp.nn.DistributedModule. Supported HuggingFace transformers versions for automated tensor parallelism: ['4.17.0', '4.20.1', '4.21.0']\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:16.715: W smdistributed/modelparallel/torch/nn/predefined_hooks.py:78] Found unsupported HuggingFace version 4.31.0 for automated tensor parallelism. HuggingFace modules will not be automatically distributed. You can use smp.tp_register_with_module API to register desired modules for tensor parallelism, or directly instantiate an smp.nn.DistributedModule. Supported HuggingFace transformers versions for automated tensor parallelism: ['4.17.0', '4.20.1', '4.21.0']\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:16.715: W smdistributed/modelparallel/torch/nn/predefined_hooks.py:78] Found unsupported HuggingFace version 4.31.0 for automated tensor parallelism. HuggingFace modules will not be automatically distributed. You can use smp.tp_register_with_module API to register desired modules for tensor parallelism, or directly instantiate an smp.nn.DistributedModule. Supported HuggingFace transformers versions for automated tensor parallelism: ['4.17.0', '4.20.1', '4.21.0']\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:16.715: W smdistributed/modelparallel/torch/nn/predefined_hooks.py:78] Found unsupported HuggingFace version 4.31.0 for automated tensor parallelism. HuggingFace modules will not be automatically distributed. You can use smp.tp_register_with_module API to register desired modules for tensor parallelism, or directly instantiate an smp.nn.DistributedModule. Supported HuggingFace transformers versions for automated tensor parallelism: ['4.17.0', '4.20.1', '4.21.0']\u001b[0m\n",
      "\u001b[34mINFO:root:Using NamedTuple = typing._NamedTuple instead.\u001b[0m\n",
      "\u001b[34mINFO:root:Using NamedTuple = typing._NamedTuple instead.\u001b[0m\n",
      "\u001b[34mINFO:root:Using NamedTuple = typing._NamedTuple instead.\u001b[0m\n",
      "\u001b[34mINFO:root:Using NamedTuple = typing._NamedTuple instead.\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:16.733: W smdistributed/modelparallel/torch/nn/predefined_hooks.py:78] Found unsupported HuggingFace version 4.31.0 for automated tensor parallelism. HuggingFace modules will not be automatically distributed. You can use smp.tp_register_with_module API to register desired modules for tensor parallelism, or directly instantiate an smp.nn.DistributedModule. Supported HuggingFace transformers versions for automated tensor parallelism: ['4.17.0', '4.20.1', '4.21.0']\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:16.733: W smdistributed/modelparallel/torch/nn/predefined_hooks.py:78] Found unsupported HuggingFace version 4.31.0 for automated tensor parallelism. HuggingFace modules will not be automatically distributed. You can use smp.tp_register_with_module API to register desired modules for tensor parallelism, or directly instantiate an smp.nn.DistributedModule. Supported HuggingFace transformers versions for automated tensor parallelism: ['4.17.0', '4.20.1', '4.21.0']\u001b[0m\n",
      "\u001b[34mINFO:root:Using NamedTuple = typing._NamedTuple instead.\u001b[0m\n",
      "\u001b[34mINFO:root:Using NamedTuple = typing._NamedTuple instead.\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:16.743 algo-1:288 INFO utils.py:28] RULE_JOB_STOP_SIGNAL_FILENAME: None\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:16.743 algo-1:289 INFO utils.py:28] RULE_JOB_STOP_SIGNAL_FILENAME: None\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:16.743 algo-1:292 INFO utils.py:28] RULE_JOB_STOP_SIGNAL_FILENAME: None\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:16.743 algo-1:286 INFO utils.py:28] RULE_JOB_STOP_SIGNAL_FILENAME: None\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:16.759 algo-1:290 INFO utils.py:28] RULE_JOB_STOP_SIGNAL_FILENAME: None\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:16.760 algo-1:287 INFO utils.py:28] RULE_JOB_STOP_SIGNAL_FILENAME: None\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:16.761 algo-1:292 INFO profiler_config_parser.py:111] Unable to find config at /opt/ml/input/config/profilerconfig.json. Profiler is disabled.\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:16.761 algo-1:288 INFO profiler_config_parser.py:111] Unable to find config at /opt/ml/input/config/profilerconfig.json. Profiler is disabled.\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:16.761 algo-1:286 INFO profiler_config_parser.py:111] Unable to find config at /opt/ml/input/config/profilerconfig.json. Profiler is disabled.\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:16.761 algo-1:289 INFO profiler_config_parser.py:111] Unable to find config at /opt/ml/input/config/profilerconfig.json. Profiler is disabled.\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:16.777 algo-1:290 INFO profiler_config_parser.py:111] Unable to find config at /opt/ml/input/config/profilerconfig.json. Profiler is disabled.\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:16.777 algo-1:287 INFO profiler_config_parser.py:111] Unable to find config at /opt/ml/input/config/profilerconfig.json. Profiler is disabled.\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:16.870: W smdistributed/modelparallel/torch/nn/predefined_hooks.py:78] Found unsupported HuggingFace version 4.31.0 for automated tensor parallelism. HuggingFace modules will not be automatically distributed. You can use smp.tp_register_with_module API to register desired modules for tensor parallelism, or directly instantiate an smp.nn.DistributedModule. Supported HuggingFace transformers versions for automated tensor parallelism: ['4.17.0', '4.20.1', '4.21.0']\u001b[0m\n",
      "\u001b[34mINFO:root:Using NamedTuple = typing._NamedTuple instead.\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:16.898 algo-1:291 INFO utils.py:28] RULE_JOB_STOP_SIGNAL_FILENAME: None\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:16.916 algo-1:291 INFO profiler_config_parser.py:111] Unable to find config at /opt/ml/input/config/profilerconfig.json. Profiler is disabled.\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:18,870] [INFO] [loss_scaler.py:183:update_scale] [deepspeed] OVERFLOW! Rank 0 Skipping step. Attempted loss scale: 4294967296, reducing to 2147483648\u001b[0m\n",
      "\u001b[34m0%|          | 1/352 [00:02<14:41,  2.51s/it]\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:20,591] [INFO] [loss_scaler.py:183:update_scale] [deepspeed] OVERFLOW! Rank 0 Skipping step. Attempted loss scale: 2147483648, reducing to 1073741824\u001b[0m\n",
      "\u001b[34m1%|          | 2/352 [00:04<11:56,  2.05s/it]\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:22,346] [INFO] [loss_scaler.py:183:update_scale] [deepspeed] OVERFLOW! Rank 0 Skipping step. Attempted loss scale: 1073741824, reducing to 536870912\u001b[0m\n",
      "\u001b[34m1%|          | 3/352 [00:05<11:07,  1.91s/it]\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:23,984] [INFO] [loss_scaler.py:183:update_scale] [deepspeed] OVERFLOW! Rank 0 Skipping step. Attempted loss scale: 536870912, reducing to 268435456\u001b[0m\n",
      "\u001b[34m1%|          | 4/352 [00:07<10:28,  1.80s/it]\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:26,344] [INFO] [loss_scaler.py:183:update_scale] [deepspeed] OVERFLOW! Rank 0 Skipping step. Attempted loss scale: 268435456, reducing to 134217728\u001b[0m\n",
      "\u001b[34m1%|▏         | 5/352 [00:09<11:35,  2.01s/it]\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:28,584] [INFO] [loss_scaler.py:183:update_scale] [deepspeed] OVERFLOW! Rank 0 Skipping step. Attempted loss scale: 134217728, reducing to 67108864\u001b[0m\n",
      "\u001b[34m2%|▏         | 6/352 [00:12<12:01,  2.08s/it]\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:30,217] [INFO] [loss_scaler.py:183:update_scale] [deepspeed] OVERFLOW! Rank 0 Skipping step. Attempted loss scale: 67108864, reducing to 33554432\u001b[0m\n",
      "\u001b[34m2%|▏         | 7/352 [00:13<11:08,  1.94s/it]\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:31,834] [INFO] [loss_scaler.py:183:update_scale] [deepspeed] OVERFLOW! Rank 0 Skipping step. Attempted loss scale: 33554432, reducing to 16777216\u001b[0m\n",
      "\u001b[34m2%|▏         | 8/352 [00:15<10:31,  1.84s/it]\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:33,602] [INFO] [loss_scaler.py:183:update_scale] [deepspeed] OVERFLOW! Rank 0 Skipping step. Attempted loss scale: 16777216, reducing to 8388608\u001b[0m\n",
      "\u001b[34m3%|▎         | 9/352 [00:17<10:22,  1.81s/it]\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:35,290] [INFO] [loss_scaler.py:183:update_scale] [deepspeed] OVERFLOW! Rank 0 Skipping step. Attempted loss scale: 8388608, reducing to 4194304\u001b[0m\n",
      "\u001b[34m3%|▎         | 10/352 [00:18<10:07,  1.78s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 2.1642, 'learning_rate': 0.0003, 'epoch': 0.06}\u001b[0m\n",
      "\u001b[34m3%|▎         | 10/352 [00:18<10:07,  1.78s/it]\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:36,889] [INFO] [loss_scaler.py:183:update_scale] [deepspeed] OVERFLOW! Rank 0 Skipping step. Attempted loss scale: 4194304, reducing to 2097152\u001b[0m\n",
      "\u001b[34m3%|▎         | 11/352 [00:20<09:46,  1.72s/it]\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:38,804] [INFO] [loss_scaler.py:183:update_scale] [deepspeed] OVERFLOW! Rank 0 Skipping step. Attempted loss scale: 2097152, reducing to 1048576\u001b[0m\n",
      "\u001b[34m3%|▎         | 12/352 [00:22<10:05,  1.78s/it]\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:07:40,208] [INFO] [loss_scaler.py:183:update_scale] [deepspeed] OVERFLOW! Rank 0 Skipping step. Attempted loss scale: 1048576, reducing to 524288\u001b[0m\n",
      "\u001b[34m4%|▎         | 13/352 [00:23<09:24,  1.67s/it]\u001b[0m\n",
      "\u001b[34m4%|▍         | 14/352 [00:25<10:01,  1.78s/it]\u001b[0m\n",
      "\u001b[34m4%|▍         | 15/352 [00:27<10:13,  1.82s/it]\u001b[0m\n",
      "\u001b[34m5%|▍         | 16/352 [00:29<10:17,  1.84s/it]\u001b[0m\n",
      "\u001b[34m5%|▍         | 17/352 [00:31<10:22,  1.86s/it]\u001b[0m\n",
      "\u001b[34m5%|▌         | 18/352 [00:33<10:22,  1.86s/it]\u001b[0m\n",
      "\u001b[34m5%|▌         | 19/352 [00:35<10:10,  1.83s/it]\u001b[0m\n",
      "\u001b[34m6%|▌         | 20/352 [00:37<10:07,  1.83s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.9422, 'learning_rate': 0.00029970736214885883, 'epoch': 0.11}\u001b[0m\n",
      "\u001b[34m6%|▌         | 20/352 [00:37<10:07,  1.83s/it]\u001b[0m\n",
      "\u001b[34m6%|▌         | 21/352 [00:38<10:09,  1.84s/it]\u001b[0m\n",
      "\u001b[34m6%|▋         | 22/352 [00:40<09:38,  1.75s/it]\u001b[0m\n",
      "\u001b[34m7%|▋         | 23/352 [00:42<10:20,  1.89s/it]\u001b[0m\n",
      "\u001b[34m7%|▋         | 24/352 [00:45<11:08,  2.04s/it]\u001b[0m\n",
      "\u001b[34m7%|▋         | 25/352 [00:47<11:18,  2.07s/it]\u001b[0m\n",
      "\u001b[34m7%|▋         | 26/352 [00:49<10:53,  2.00s/it]\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:08:07,131] [INFO] [loss_scaler.py:183:update_scale] [deepspeed] OVERFLOW! Rank 0 Skipping step. Attempted loss scale: 524288, reducing to 262144\u001b[0m\n",
      "\u001b[34m8%|▊         | 27/352 [00:50<10:23,  1.92s/it]\u001b[0m\n",
      "\u001b[34m8%|▊         | 28/352 [00:52<09:54,  1.84s/it]\u001b[0m\n",
      "\u001b[34m8%|▊         | 29/352 [00:54<09:45,  1.81s/it]\u001b[0m\n",
      "\u001b[34m9%|▊         | 30/352 [00:55<09:38,  1.80s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.5378, 'learning_rate': 0.0002984732162821399, 'epoch': 0.17}\u001b[0m\n",
      "\u001b[34m9%|▊         | 30/352 [00:55<09:38,  1.80s/it]\u001b[0m\n",
      "\u001b[34m9%|▉         | 31/352 [00:57<09:29,  1.77s/it]\u001b[0m\n",
      "\u001b[34m9%|▉         | 32/352 [00:59<09:12,  1.73s/it]\u001b[0m\n",
      "\u001b[34m9%|▉         | 33/352 [01:01<09:56,  1.87s/it]\u001b[0m\n",
      "\u001b[34m10%|▉         | 34/352 [01:03<09:33,  1.80s/it]\u001b[0m\n",
      "\u001b[34m10%|▉         | 35/352 [01:05<10:00,  1.90s/it]\u001b[0m\n",
      "\u001b[34m10%|█         | 36/352 [01:06<09:27,  1.80s/it]\u001b[0m\n",
      "\u001b[34m11%|█         | 37/352 [01:08<09:36,  1.83s/it]\u001b[0m\n",
      "\u001b[34m11%|█         | 38/352 [01:11<10:24,  1.99s/it]\u001b[0m\n",
      "\u001b[34m11%|█         | 39/352 [01:13<10:59,  2.11s/it]\u001b[0m\n",
      "\u001b[34m11%|█▏        | 40/352 [01:15<10:37,  2.04s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.5338, 'learning_rate': 0.00029597956804266047, 'epoch': 0.23}\u001b[0m\n",
      "\u001b[34m11%|█▏        | 40/352 [01:15<10:37,  2.04s/it]\u001b[0m\n",
      "\u001b[34m12%|█▏        | 41/352 [01:17<10:10,  1.96s/it]\u001b[0m\n",
      "\u001b[34m12%|█▏        | 42/352 [01:19<10:11,  1.97s/it]\u001b[0m\n",
      "\u001b[34m12%|█▏        | 43/352 [01:20<09:16,  1.80s/it]\u001b[0m\n",
      "\u001b[34m12%|█▎        | 44/352 [01:22<09:35,  1.87s/it]\u001b[0m\n",
      "\u001b[34m13%|█▎        | 45/352 [01:24<10:17,  2.01s/it]\u001b[0m\n",
      "\u001b[34m13%|█▎        | 46/352 [01:26<10:06,  1.98s/it]\u001b[0m\n",
      "\u001b[34m13%|█▎        | 47/352 [01:28<10:07,  1.99s/it]\u001b[0m\n",
      "\u001b[34m14%|█▎        | 48/352 [01:30<09:42,  1.92s/it]\u001b[0m\n",
      "\u001b[34m14%|█▍        | 49/352 [01:32<09:31,  1.88s/it]\u001b[0m\n",
      "\u001b[34m14%|█▍        | 50/352 [01:34<09:35,  1.91s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.4416, 'learning_rate': 0.0002923238875255979, 'epoch': 0.28}\u001b[0m\n",
      "\u001b[34m14%|█▍        | 50/352 [01:34<09:35,  1.91s/it]\u001b[0m\n",
      "\u001b[34m14%|█▍        | 51/352 [01:36<09:15,  1.85s/it]\u001b[0m\n",
      "\u001b[34m15%|█▍        | 52/352 [01:38<09:43,  1.94s/it]\u001b[0m\n",
      "\u001b[34m15%|█▌        | 53/352 [01:40<10:17,  2.06s/it]\u001b[0m\n",
      "\u001b[34m15%|█▌        | 54/352 [01:42<09:39,  1.94s/it]\u001b[0m\n",
      "\u001b[34m16%|█▌        | 55/352 [01:43<09:21,  1.89s/it]\u001b[0m\n",
      "\u001b[34m16%|█▌        | 56/352 [01:45<09:05,  1.84s/it]\u001b[0m\n",
      "\u001b[34m16%|█▌        | 57/352 [01:47<08:38,  1.76s/it]\u001b[0m\n",
      "\u001b[34m16%|█▋        | 58/352 [01:48<08:22,  1.71s/it]\u001b[0m\n",
      "\u001b[34m17%|█▋        | 59/352 [01:50<08:12,  1.68s/it]\u001b[0m\n",
      "\u001b[34m17%|█▋        | 60/352 [01:52<08:35,  1.76s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.4808, 'learning_rate': 0.0002875352748222479, 'epoch': 0.34}\u001b[0m\n",
      "\u001b[34m17%|█▋        | 60/352 [01:52<08:35,  1.76s/it]\u001b[0m\n",
      "\u001b[34m17%|█▋        | 61/352 [01:54<08:27,  1.74s/it]\u001b[0m\n",
      "\u001b[34m18%|█▊        | 62/352 [01:56<08:51,  1.83s/it]\u001b[0m\n",
      "\u001b[34m18%|█▊        | 63/352 [01:58<08:51,  1.84s/it]\u001b[0m\n",
      "\u001b[34m18%|█▊        | 64/352 [01:59<08:57,  1.87s/it]\u001b[0m\n",
      "\u001b[34m18%|█▊        | 65/352 [02:01<08:54,  1.86s/it]\u001b[0m\n",
      "\u001b[34m19%|█▉        | 66/352 [02:03<09:13,  1.93s/it]\u001b[0m\n",
      "\u001b[34m19%|█▉        | 67/352 [02:05<09:08,  1.92s/it]\u001b[0m\n",
      "\u001b[34m19%|█▉        | 68/352 [02:07<09:13,  1.95s/it]\u001b[0m\n",
      "\u001b[34m20%|█▉        | 69/352 [02:09<09:16,  1.97s/it]\u001b[0m\n",
      "\u001b[34m20%|█▉        | 70/352 [02:12<09:53,  2.11s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.397, 'learning_rate': 0.0002816518484350883, 'epoch': 0.4}\u001b[0m\n",
      "\u001b[34m20%|█▉        | 70/352 [02:12<09:53,  2.11s/it]\u001b[0m\n",
      "\u001b[34m20%|██        | 71/352 [02:14<09:40,  2.07s/it]\u001b[0m\n",
      "\u001b[34m20%|██        | 72/352 [02:15<09:09,  1.96s/it]\u001b[0m\n",
      "\u001b[34m21%|██        | 73/352 [02:17<08:39,  1.86s/it]\u001b[0m\n",
      "\u001b[34m21%|██        | 74/352 [02:19<08:54,  1.92s/it]\u001b[0m\n",
      "\u001b[34m21%|██▏       | 75/352 [02:21<08:42,  1.88s/it]\u001b[0m\n",
      "\u001b[34m22%|██▏       | 76/352 [02:23<08:25,  1.83s/it]\u001b[0m\n",
      "\u001b[34m22%|██▏       | 77/352 [02:24<08:24,  1.84s/it]\u001b[0m\n",
      "\u001b[34m22%|██▏       | 78/352 [02:27<09:07,  2.00s/it]\u001b[0m\n",
      "\u001b[34m22%|██▏       | 79/352 [02:29<08:43,  1.92s/it]\u001b[0m\n",
      "\u001b[34m23%|██▎       | 80/352 [02:30<08:39,  1.91s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.4471, 'learning_rate': 0.0002747204418453818, 'epoch': 0.45}\u001b[0m\n",
      "\u001b[34m23%|██▎       | 80/352 [02:30<08:39,  1.91s/it]\u001b[0m\n",
      "\u001b[34m23%|██▎       | 81/352 [02:32<08:21,  1.85s/it]\u001b[0m\n",
      "\u001b[34m23%|██▎       | 82/352 [02:34<08:19,  1.85s/it]\u001b[0m\n",
      "\u001b[34m24%|██▎       | 83/352 [02:36<08:50,  1.97s/it]\u001b[0m\n",
      "\u001b[34m24%|██▍       | 84/352 [02:38<08:36,  1.93s/it]\u001b[0m\n",
      "\u001b[34m24%|██▍       | 85/352 [02:40<08:20,  1.88s/it]\u001b[0m\n",
      "\u001b[34m24%|██▍       | 86/352 [02:42<08:08,  1.84s/it]\u001b[0m\n",
      "\u001b[34m25%|██▍       | 87/352 [02:44<08:16,  1.87s/it]\u001b[0m\n",
      "\u001b[34m25%|██▌       | 88/352 [02:45<08:13,  1.87s/it]\u001b[0m\n",
      "\u001b[34m25%|██▌       | 89/352 [02:48<08:49,  2.01s/it]\u001b[0m\n",
      "\u001b[34m26%|██▌       | 90/352 [02:50<08:26,  1.93s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.4793, 'learning_rate': 0.00026679623070746325, 'epoch': 0.51}\u001b[0m\n",
      "\u001b[34m26%|██▌       | 90/352 [02:50<08:26,  1.93s/it]\u001b[0m\n",
      "\u001b[34m26%|██▌       | 91/352 [02:51<07:57,  1.83s/it]\u001b[0m\n",
      "\u001b[34m26%|██▌       | 92/352 [02:53<08:00,  1.85s/it]\u001b[0m\n",
      "\u001b[34m26%|██▋       | 93/352 [02:56<08:56,  2.07s/it]\u001b[0m\n",
      "\u001b[34m27%|██▋       | 94/352 [02:58<08:41,  2.02s/it]\u001b[0m\n",
      "\u001b[34m27%|██▋       | 95/352 [02:59<08:25,  1.97s/it]\u001b[0m\n",
      "\u001b[34m27%|██▋       | 96/352 [03:02<08:53,  2.08s/it]\u001b[0m\n",
      "\u001b[34m28%|██▊       | 97/352 [03:03<08:22,  1.97s/it]\u001b[0m\n",
      "\u001b[34m28%|██▊       | 98/352 [03:05<08:03,  1.90s/it]\u001b[0m\n",
      "\u001b[34m28%|██▊       | 99/352 [03:07<07:35,  1.80s/it]\u001b[0m\n",
      "\u001b[34m28%|██▊       | 100/352 [03:09<07:38,  1.82s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.4068, 'learning_rate': 0.0002579422936373344, 'epoch': 0.57}\u001b[0m\n",
      "\u001b[34m28%|██▊       | 100/352 [03:09<07:38,  1.82s/it]\u001b[0m\n",
      "\u001b[34m07/20/2023 06:11:16 - INFO - llmtuner.tuner.core.trainer - Saving model checkpoint to /tmp/ouput/checkpoint-100\u001b[0m\n",
      "\u001b[34mINFO:llmtuner.tuner.core.trainer:Saving model checkpoint to /tmp/ouput/checkpoint-100\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:11:19,520] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step100 is about to be saved!\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/torch/nn/modules/module.py:1432: UserWarning: Positional args are being deprecated, use kwargs instead. Refer to https://pytorch.org/docs/master/generated/torch.nn.Module.html#torch.nn.Module.state_dict for details.\n",
      "  warnings.warn(\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/torch/nn/modules/module.py:1432: UserWarning: Positional args are being deprecated, use kwargs instead. Refer to https://pytorch.org/docs/master/generated/torch.nn.Module.html#torch.nn.Module.state_dict for details.\n",
      "  warnings.warn(\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/torch/nn/modules/module.py:1432: UserWarning: Positional args are being deprecated, use kwargs instead. Refer to https://pytorch.org/docs/master/generated/torch.nn.Module.html#torch.nn.Module.state_dict for details.\n",
      "  warnings.warn(\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/torch/nn/modules/module.py:1432: UserWarning: Positional args are being deprecated, use kwargs instead. Refer to https://pytorch.org/docs/master/generated/torch.nn.Module.html#torch.nn.Module.state_dict for details.\n",
      "  warnings.warn(\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/torch/nn/modules/module.py:1432: UserWarning: Positional args are being deprecated, use kwargs instead. Refer to https://pytorch.org/docs/master/generated/torch.nn.Module.html#torch.nn.Module.state_dict for details.\n",
      "  warnings.warn(\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/torch/nn/modules/module.py:1432: UserWarning: Positional args are being deprecated, use kwargs instead. Refer to https://pytorch.org/docs/master/generated/torch.nn.Module.html#torch.nn.Module.state_dict for details.\n",
      "  warnings.warn(\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/torch/nn/modules/module.py:1432: UserWarning: Positional args are being deprecated, use kwargs instead. Refer to https://pytorch.org/docs/master/generated/torch.nn.Module.html#torch.nn.Module.state_dict for details.\n",
      "  warnings.warn(\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/torch/nn/modules/module.py:1432: UserWarning: Positional args are being deprecated, use kwargs instead. Refer to https://pytorch.org/docs/master/generated/torch.nn.Module.html#torch.nn.Module.state_dict for details.\n",
      "  warnings.warn(\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:11:37,784] [INFO] [logging.py:96:log_dist] [Rank 0] Saving model checkpoint: /tmp/ouput/checkpoint-100/global_step100/mp_rank_00_model_states.pt\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:11:37,785] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving /tmp/ouput/checkpoint-100/global_step100/mp_rank_00_model_states.pt...\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:12:42,715] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved /tmp/ouput/checkpoint-100/global_step100/mp_rank_00_model_states.pt.\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:12:43,562] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving /tmp/ouput/checkpoint-100/global_step100/zero_pp_rank_0_mp_rank_00_optim_states.pt...\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:12:44,137] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved /tmp/ouput/checkpoint-100/global_step100/zero_pp_rank_0_mp_rank_00_optim_states.pt.\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:12:44,138] [INFO] [engine.py:3285:_save_zero_checkpoint] zero checkpoint saved /tmp/ouput/checkpoint-100/global_step100/zero_pp_rank_0_mp_rank_00_optim_states.pt\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:12:44,138] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step100 is ready now!\u001b[0m\n",
      "\u001b[34m29%|██▊       | 101/352 [05:29<3:01:30, 43.39s/it]\u001b[0m\n",
      "\u001b[34m29%|██▉       | 102/352 [05:31<2:08:32, 30.85s/it]\u001b[0m\n",
      "\u001b[34m29%|██▉       | 103/352 [05:33<1:32:10, 22.21s/it]\u001b[0m\n",
      "\u001b[34m30%|██▉       | 104/352 [05:34<1:06:25, 16.07s/it]\u001b[0m\n",
      "\u001b[34m30%|██▉       | 105/352 [05:36<48:27, 11.77s/it]\u001b[0m\n",
      "\u001b[34m30%|███       | 106/352 [05:38<36:00,  8.78s/it]\u001b[0m\n",
      "\u001b[34m30%|███       | 107/352 [05:40<27:40,  6.78s/it]\u001b[0m\n",
      "\u001b[34m31%|███       | 108/352 [05:42<21:23,  5.26s/it]\u001b[0m\n",
      "\u001b[34m31%|███       | 109/352 [05:44<17:21,  4.29s/it]\u001b[0m\n",
      "\u001b[34m31%|███▏      | 110/352 [05:46<14:19,  3.55s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.3955, 'learning_rate': 0.00024822911009179276, 'epoch': 0.62}\u001b[0m\n",
      "\u001b[34m31%|███▏      | 110/352 [05:46<14:19,  3.55s/it]\u001b[0m\n",
      "\u001b[34m32%|███▏      | 111/352 [05:48<12:38,  3.15s/it]\u001b[0m\n",
      "\u001b[34m32%|███▏      | 112/352 [05:50<11:32,  2.89s/it]\u001b[0m\n",
      "\u001b[34m32%|███▏      | 113/352 [05:52<10:09,  2.55s/it]\u001b[0m\n",
      "\u001b[34m32%|███▏      | 114/352 [05:54<09:26,  2.38s/it]\u001b[0m\n",
      "\u001b[34m33%|███▎      | 115/352 [05:55<08:24,  2.13s/it]\u001b[0m\n",
      "\u001b[34m33%|███▎      | 116/352 [05:57<07:55,  2.01s/it]\u001b[0m\n",
      "\u001b[34m33%|███▎      | 117/352 [05:59<07:50,  2.00s/it]\u001b[0m\n",
      "\u001b[34m34%|███▎      | 118/352 [06:01<07:36,  1.95s/it]\u001b[0m\n",
      "\u001b[34m34%|███▍      | 119/352 [06:03<07:28,  1.93s/it]\u001b[0m\n",
      "\u001b[34m34%|███▍      | 120/352 [06:05<07:12,  1.87s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.4217, 'learning_rate': 0.00023773399933509794, 'epoch': 0.68}\u001b[0m\n",
      "\u001b[34m34%|███▍      | 120/352 [06:05<07:12,  1.87s/it]\u001b[0m\n",
      "\u001b[34m34%|███▍      | 121/352 [06:06<07:00,  1.82s/it]\u001b[0m\n",
      "\u001b[34m35%|███▍      | 122/352 [06:08<07:01,  1.83s/it]\u001b[0m\n",
      "\u001b[34m35%|███▍      | 123/352 [06:10<06:51,  1.80s/it]\u001b[0m\n",
      "\u001b[34m35%|███▌      | 124/352 [06:12<06:43,  1.77s/it]\u001b[0m\n",
      "\u001b[34m36%|███▌      | 125/352 [06:14<07:10,  1.90s/it]\u001b[0m\n",
      "\u001b[34m36%|███▌      | 126/352 [06:16<07:06,  1.89s/it]\u001b[0m\n",
      "\u001b[34m36%|███▌      | 127/352 [06:18<07:11,  1.92s/it]\u001b[0m\n",
      "\u001b[34m36%|███▋      | 128/352 [06:20<07:13,  1.94s/it]\u001b[0m\n",
      "\u001b[34m37%|███▋      | 129/352 [06:22<07:35,  2.04s/it]\u001b[0m\n",
      "\u001b[34m37%|███▋      | 130/352 [06:24<07:43,  2.09s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.4221, 'learning_rate': 0.00022654050495913495, 'epoch': 0.74}\u001b[0m\n",
      "\u001b[34m37%|███▋      | 130/352 [06:24<07:43,  2.09s/it]\u001b[0m\n",
      "\u001b[34m37%|███▋      | 131/352 [06:26<07:09,  1.94s/it]\u001b[0m\n",
      "\u001b[34m38%|███▊      | 132/352 [06:28<07:20,  2.00s/it]\u001b[0m\n",
      "\u001b[34m38%|███▊      | 133/352 [06:30<07:11,  1.97s/it]\u001b[0m\n",
      "\u001b[34m38%|███▊      | 134/352 [06:32<07:27,  2.05s/it]\u001b[0m\n",
      "\u001b[34m38%|███▊      | 135/352 [06:34<07:05,  1.96s/it]\u001b[0m\n",
      "\u001b[34m39%|███▊      | 136/352 [06:36<07:00,  1.95s/it]\u001b[0m\n",
      "\u001b[34m39%|███▉      | 137/352 [06:38<07:14,  2.02s/it]\u001b[0m\n",
      "\u001b[34m39%|███▉      | 138/352 [06:39<06:53,  1.93s/it]\u001b[0m\n",
      "\u001b[34m39%|███▉      | 139/352 [06:41<06:49,  1.92s/it]\u001b[0m\n",
      "\u001b[34m40%|███▉      | 140/352 [06:43<06:23,  1.81s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.3727, 'learning_rate': 0.00021473772985644004, 'epoch': 0.8}\u001b[0m\n",
      "\u001b[34m40%|███▉      | 140/352 [06:43<06:23,  1.81s/it]\u001b[0m\n",
      "\u001b[34m40%|████      | 141/352 [06:45<06:12,  1.77s/it]\u001b[0m\n",
      "\u001b[34m40%|████      | 142/352 [06:46<05:57,  1.70s/it]\u001b[0m\n",
      "\u001b[34m41%|████      | 143/352 [06:48<05:58,  1.72s/it]\u001b[0m\n",
      "\u001b[34m41%|████      | 144/352 [06:50<05:57,  1.72s/it]\u001b[0m\n",
      "\u001b[34m41%|████      | 145/352 [06:51<05:54,  1.71s/it]\u001b[0m\n",
      "\u001b[34m41%|████▏     | 146/352 [06:53<05:54,  1.72s/it]\u001b[0m\n",
      "\u001b[34m42%|████▏     | 147/352 [06:55<05:54,  1.73s/it]\u001b[0m\n",
      "\u001b[34m42%|████▏     | 148/352 [06:57<06:01,  1.77s/it]\u001b[0m\n",
      "\u001b[34m42%|████▏     | 149/352 [06:59<06:12,  1.83s/it]\u001b[0m\n",
      "\u001b[34m43%|████▎     | 150/352 [07:00<06:03,  1.80s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.4472, 'learning_rate': 0.00020241962693986476, 'epoch': 0.85}\u001b[0m\n",
      "\u001b[34m43%|████▎     | 150/352 [07:00<06:03,  1.80s/it]\u001b[0m\n",
      "\u001b[34m43%|████▎     | 151/352 [07:02<06:07,  1.83s/it]\u001b[0m\n",
      "\u001b[34m43%|████▎     | 152/352 [07:04<06:07,  1.84s/it]\u001b[0m\n",
      "\u001b[34m43%|████▎     | 153/352 [07:06<06:16,  1.89s/it]\u001b[0m\n",
      "\u001b[34m44%|████▍     | 154/352 [07:08<06:06,  1.85s/it]\u001b[0m\n",
      "\u001b[34m44%|████▍     | 155/352 [07:10<05:56,  1.81s/it]\u001b[0m\n",
      "\u001b[34m44%|████▍     | 156/352 [07:11<05:50,  1.79s/it]\u001b[0m\n",
      "\u001b[34m45%|████▍     | 157/352 [07:13<05:44,  1.77s/it]\u001b[0m\n",
      "\u001b[34m45%|████▍     | 158/352 [07:15<05:36,  1.73s/it]\u001b[0m\n",
      "\u001b[34m45%|████▌     | 159/352 [07:17<06:15,  1.95s/it]\u001b[0m\n",
      "\u001b[34m45%|████▌     | 160/352 [07:19<05:58,  1.87s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.4396, 'learning_rate': 0.00018968425125491685, 'epoch': 0.91}\u001b[0m\n",
      "\u001b[34m45%|████▌     | 160/352 [07:19<05:58,  1.87s/it]\u001b[0m\n",
      "\u001b[34m46%|████▌     | 161/352 [07:21<05:48,  1.82s/it]\u001b[0m\n",
      "\u001b[34m46%|████▌     | 162/352 [07:23<06:21,  2.01s/it]\u001b[0m\n",
      "\u001b[34m46%|████▋     | 163/352 [07:25<05:56,  1.88s/it]\u001b[0m\n",
      "\u001b[34m47%|████▋     | 164/352 [07:26<05:45,  1.84s/it]\u001b[0m\n",
      "\u001b[34m47%|████▋     | 165/352 [07:28<05:48,  1.86s/it]\u001b[0m\n",
      "\u001b[34m47%|████▋     | 166/352 [07:30<05:41,  1.83s/it]\u001b[0m\n",
      "\u001b[34m47%|████▋     | 167/352 [07:32<05:32,  1.80s/it]\u001b[0m\n",
      "\u001b[34m48%|████▊     | 168/352 [07:34<05:48,  1.89s/it]\u001b[0m\n",
      "\u001b[34m48%|████▊     | 169/352 [07:36<05:37,  1.84s/it]\u001b[0m\n",
      "\u001b[34m48%|████▊     | 170/352 [07:37<05:30,  1.82s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.4354, 'learning_rate': 0.00017663297943814552, 'epoch': 0.97}\u001b[0m\n",
      "\u001b[34m48%|████▊     | 170/352 [07:37<05:30,  1.82s/it]\u001b[0m\n",
      "\u001b[34m49%|████▊     | 171/352 [07:39<05:30,  1.83s/it]\u001b[0m\n",
      "\u001b[34m49%|████▉     | 172/352 [07:42<06:12,  2.07s/it]\u001b[0m\n",
      "\u001b[34m49%|████▉     | 173/352 [07:44<06:18,  2.12s/it]\u001b[0m\n",
      "\u001b[34m49%|████▉     | 174/352 [07:46<06:19,  2.13s/it]\u001b[0m\n",
      "\u001b[34m50%|████▉     | 175/352 [07:48<06:22,  2.16s/it]\u001b[0m\n",
      "\u001b[34m50%|█████     | 176/352 [07:50<05:57,  2.03s/it]\u001b[0m\n",
      "\u001b[34m50%|█████     | 177/352 [07:52<05:49,  1.99s/it]\u001b[0m\n",
      "\u001b[34m51%|█████     | 178/352 [07:54<06:04,  2.10s/it]\u001b[0m\n",
      "\u001b[34m51%|█████     | 179/352 [07:56<05:38,  1.96s/it]\u001b[0m\n",
      "\u001b[34m51%|█████     | 180/352 [07:58<05:16,  1.84s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.2872, 'learning_rate': 0.00016336970273486638, 'epoch': 1.02}\u001b[0m\n",
      "\u001b[34m51%|█████     | 180/352 [07:58<05:16,  1.84s/it]\u001b[0m\n",
      "\u001b[34m51%|█████▏    | 181/352 [08:00<05:22,  1.88s/it]\u001b[0m\n",
      "\u001b[34m52%|█████▏    | 182/352 [08:02<05:54,  2.09s/it]\u001b[0m\n",
      "\u001b[34m52%|█████▏    | 183/352 [08:04<05:39,  2.01s/it]\u001b[0m\n",
      "\u001b[34m52%|█████▏    | 184/352 [08:06<05:21,  1.92s/it]\u001b[0m\n",
      "\u001b[34m53%|█████▎    | 185/352 [08:07<05:05,  1.83s/it]\u001b[0m\n",
      "\u001b[34m53%|█████▎    | 186/352 [08:10<05:24,  1.96s/it]\u001b[0m\n",
      "\u001b[34m53%|█████▎    | 187/352 [08:11<05:09,  1.87s/it]\u001b[0m\n",
      "\u001b[34m53%|█████▎    | 188/352 [08:13<05:09,  1.88s/it]\u001b[0m\n",
      "\u001b[34m54%|█████▎    | 189/352 [08:15<05:29,  2.02s/it]\u001b[0m\n",
      "\u001b[34m54%|█████▍    | 190/352 [08:18<05:30,  2.04s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.1842, 'learning_rate': 0.00015, 'epoch': 1.08}\u001b[0m\n",
      "\u001b[34m54%|█████▍    | 190/352 [08:18<05:30,  2.04s/it]\u001b[0m\n",
      "\u001b[34m54%|█████▍    | 191/352 [08:19<05:03,  1.89s/it]\u001b[0m\n",
      "\u001b[34m55%|█████▍    | 192/352 [08:21<04:52,  1.83s/it]\u001b[0m\n",
      "\u001b[34m55%|█████▍    | 193/352 [08:23<04:55,  1.86s/it]\u001b[0m\n",
      "\u001b[34m55%|█████▌    | 194/352 [08:25<04:58,  1.89s/it]\u001b[0m\n",
      "\u001b[34m55%|█████▌    | 195/352 [08:27<04:59,  1.91s/it]\u001b[0m\n",
      "\u001b[34m56%|█████▌    | 196/352 [08:28<04:53,  1.88s/it]\u001b[0m\n",
      "\u001b[34m56%|█████▌    | 197/352 [08:30<04:34,  1.77s/it]\u001b[0m\n",
      "\u001b[34m56%|█████▋    | 198/352 [08:32<04:36,  1.80s/it]\u001b[0m\n",
      "\u001b[34m57%|█████▋    | 199/352 [08:34<04:43,  1.85s/it]\u001b[0m\n",
      "\u001b[34m57%|█████▋    | 200/352 [08:36<04:43,  1.87s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.1756, 'learning_rate': 0.0001366302972651336, 'epoch': 1.14}\u001b[0m\n",
      "\u001b[34m57%|█████▋    | 200/352 [08:36<04:43,  1.87s/it]\u001b[0m\n",
      "\u001b[34m07/20/2023 06:16:42 - INFO - llmtuner.tuner.core.trainer - Saving model checkpoint to /tmp/ouput/checkpoint-200\u001b[0m\n",
      "\u001b[34mINFO:llmtuner.tuner.core.trainer:Saving model checkpoint to /tmp/ouput/checkpoint-200\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:16:47,180] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step200 is about to be saved!\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:17:05,856] [INFO] [logging.py:96:log_dist] [Rank 0] Saving model checkpoint: /tmp/ouput/checkpoint-200/global_step200/mp_rank_00_model_states.pt\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:17:05,856] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving /tmp/ouput/checkpoint-200/global_step200/mp_rank_00_model_states.pt...\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:18:11,993] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved /tmp/ouput/checkpoint-200/global_step200/mp_rank_00_model_states.pt.\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:18:12,801] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving /tmp/ouput/checkpoint-200/global_step200/zero_pp_rank_0_mp_rank_00_optim_states.pt...\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:18:13,366] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved /tmp/ouput/checkpoint-200/global_step200/zero_pp_rank_0_mp_rank_00_optim_states.pt.\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:18:13,366] [INFO] [engine.py:3285:_save_zero_checkpoint] zero checkpoint saved /tmp/ouput/checkpoint-200/global_step200/zero_pp_rank_0_mp_rank_00_optim_states.pt\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:18:13,366] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step200 is ready now!\u001b[0m\n",
      "\u001b[34m57%|█████▋    | 201/352 [10:58<1:50:50, 44.04s/it]\u001b[0m\n",
      "\u001b[34m57%|█████▋    | 202/352 [11:00<1:18:48, 31.53s/it]\u001b[0m\n",
      "\u001b[34m58%|█████▊    | 203/352 [11:02<56:04, 22.58s/it]\u001b[0m\n",
      "\u001b[34m58%|█████▊    | 204/352 [11:04<40:40, 16.49s/it]\u001b[0m\n",
      "\u001b[34m58%|█████▊    | 205/352 [11:06<29:38, 12.10s/it]\u001b[0m\n",
      "\u001b[34m59%|█████▊    | 206/352 [11:08<21:50,  8.98s/it]\u001b[0m\n",
      "\u001b[34m59%|█████▉    | 207/352 [11:10<16:31,  6.84s/it]\u001b[0m\n",
      "\u001b[34m59%|█████▉    | 208/352 [11:12<12:43,  5.30s/it]\u001b[0m\n",
      "\u001b[34m59%|█████▉    | 209/352 [11:13<10:02,  4.21s/it]\u001b[0m\n",
      "\u001b[34m60%|█████▉    | 210/352 [11:15<08:12,  3.47s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.1779, 'learning_rate': 0.00012336702056185453, 'epoch': 1.19}\u001b[0m\n",
      "\u001b[34m60%|█████▉    | 210/352 [11:15<08:12,  3.47s/it]\u001b[0m\n",
      "\u001b[34m60%|█████▉    | 211/352 [11:17<06:56,  2.95s/it]\u001b[0m\n",
      "\u001b[34m60%|██████    | 212/352 [11:19<06:15,  2.69s/it]\u001b[0m\n",
      "\u001b[34m61%|██████    | 213/352 [11:21<05:50,  2.52s/it]\u001b[0m\n",
      "\u001b[34m61%|██████    | 214/352 [11:23<05:19,  2.32s/it]\u001b[0m\n",
      "\u001b[34m61%|██████    | 215/352 [11:24<04:48,  2.11s/it]\u001b[0m\n",
      "\u001b[34m61%|██████▏   | 216/352 [11:26<04:32,  2.01s/it]\u001b[0m\n",
      "\u001b[34m62%|██████▏   | 217/352 [11:28<04:20,  1.93s/it]\u001b[0m\n",
      "\u001b[34m62%|██████▏   | 218/352 [11:30<04:05,  1.84s/it]\u001b[0m\n",
      "\u001b[34m62%|██████▏   | 219/352 [11:31<03:59,  1.80s/it]\u001b[0m\n",
      "\u001b[34m62%|██████▎   | 220/352 [11:33<03:50,  1.74s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.2038, 'learning_rate': 0.00011031574874508316, 'epoch': 1.25}\u001b[0m\n",
      "\u001b[34m62%|██████▎   | 220/352 [11:33<03:50,  1.74s/it]\u001b[0m\n",
      "\u001b[34m63%|██████▎   | 221/352 [11:35<03:53,  1.79s/it]\u001b[0m\n",
      "\u001b[34m63%|██████▎   | 222/352 [11:37<04:04,  1.88s/it]\u001b[0m\n",
      "\u001b[34m63%|██████▎   | 223/352 [11:39<04:08,  1.93s/it]\u001b[0m\n",
      "\u001b[34m64%|██████▎   | 224/352 [11:41<04:04,  1.91s/it]\u001b[0m\n",
      "\u001b[34m64%|██████▍   | 225/352 [11:43<04:18,  2.04s/it]\u001b[0m\n",
      "\u001b[34m64%|██████▍   | 226/352 [11:45<04:15,  2.03s/it]\u001b[0m\n",
      "\u001b[34m64%|██████▍   | 227/352 [11:47<03:58,  1.91s/it]\u001b[0m\n",
      "\u001b[34m65%|██████▍   | 228/352 [11:49<03:53,  1.88s/it]\u001b[0m\n",
      "\u001b[34m65%|██████▌   | 229/352 [11:50<03:47,  1.85s/it]\u001b[0m\n",
      "\u001b[34m65%|██████▌   | 230/352 [11:52<03:41,  1.81s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.1642, 'learning_rate': 9.758037306013526e-05, 'epoch': 1.31}\u001b[0m\n",
      "\u001b[34m65%|██████▌   | 230/352 [11:52<03:41,  1.81s/it]\u001b[0m\n",
      "\u001b[34m66%|██████▌   | 231/352 [11:54<03:42,  1.83s/it]\u001b[0m\n",
      "\u001b[34m66%|██████▌   | 232/352 [11:56<03:43,  1.86s/it]\u001b[0m\n",
      "\u001b[34m66%|██████▌   | 233/352 [11:58<03:53,  1.96s/it]\u001b[0m\n",
      "\u001b[34m66%|██████▋   | 234/352 [12:00<03:37,  1.84s/it]\u001b[0m\n",
      "\u001b[34m67%|██████▋   | 235/352 [12:01<03:36,  1.85s/it]\u001b[0m\n",
      "\u001b[34m67%|██████▋   | 236/352 [12:04<03:42,  1.92s/it]\u001b[0m\n",
      "\u001b[34m67%|██████▋   | 237/352 [12:06<03:40,  1.92s/it]\u001b[0m\n",
      "\u001b[34m68%|██████▊   | 238/352 [12:07<03:36,  1.90s/it]\u001b[0m\n",
      "\u001b[34m68%|██████▊   | 239/352 [12:09<03:29,  1.85s/it]\u001b[0m\n",
      "\u001b[34m68%|██████▊   | 240/352 [12:11<03:43,  1.99s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.1223, 'learning_rate': 8.526227014355986e-05, 'epoch': 1.36}\u001b[0m\n",
      "\u001b[34m68%|██████▊   | 240/352 [12:11<03:43,  1.99s/it]\u001b[0m\n",
      "\u001b[34m68%|██████▊   | 241/352 [12:13<03:36,  1.95s/it]\u001b[0m\n",
      "\u001b[34m69%|██████▉   | 242/352 [12:15<03:39,  1.99s/it]\u001b[0m\n",
      "\u001b[34m69%|██████▉   | 243/352 [12:18<03:44,  2.06s/it]\u001b[0m\n",
      "\u001b[34m69%|██████▉   | 244/352 [12:20<03:44,  2.08s/it]\u001b[0m\n",
      "\u001b[34m70%|██████▉   | 245/352 [12:22<03:36,  2.03s/it]\u001b[0m\n",
      "\u001b[34m70%|██████▉   | 246/352 [12:23<03:29,  1.98s/it]\u001b[0m\n",
      "\u001b[34m70%|███████   | 247/352 [12:26<03:33,  2.03s/it]\u001b[0m\n",
      "\u001b[34m70%|███████   | 248/352 [12:28<03:31,  2.04s/it]\u001b[0m\n",
      "\u001b[34m71%|███████   | 249/352 [12:30<03:24,  1.99s/it]\u001b[0m\n",
      "\u001b[34m71%|███████   | 250/352 [12:32<03:24,  2.01s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.1241, 'learning_rate': 7.345949504086507e-05, 'epoch': 1.42}\u001b[0m\n",
      "\u001b[34m71%|███████   | 250/352 [12:32<03:24,  2.01s/it]\u001b[0m\n",
      "\u001b[34m71%|███████▏  | 251/352 [12:34<03:27,  2.06s/it]\u001b[0m\n",
      "\u001b[34m72%|███████▏  | 252/352 [12:36<03:22,  2.02s/it]\u001b[0m\n",
      "\u001b[34m72%|███████▏  | 253/352 [12:38<03:14,  1.96s/it]\u001b[0m\n",
      "\u001b[34m72%|███████▏  | 254/352 [12:40<03:23,  2.07s/it]\u001b[0m\n",
      "\u001b[34m72%|███████▏  | 255/352 [12:41<03:01,  1.88s/it]\u001b[0m\n",
      "\u001b[34m73%|███████▎  | 256/352 [12:43<02:58,  1.86s/it]\u001b[0m\n",
      "\u001b[34m73%|███████▎  | 257/352 [12:45<02:58,  1.88s/it]\u001b[0m\n",
      "\u001b[34m73%|███████▎  | 258/352 [12:47<02:48,  1.79s/it]\u001b[0m\n",
      "\u001b[34m74%|███████▎  | 259/352 [12:49<02:53,  1.86s/it]\u001b[0m\n",
      "\u001b[34m74%|███████▍  | 260/352 [12:50<02:49,  1.84s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.1565, 'learning_rate': 6.226600066490204e-05, 'epoch': 1.48}\u001b[0m\n",
      "\u001b[34m74%|███████▍  | 260/352 [12:50<02:49,  1.84s/it]\u001b[0m\n",
      "\u001b[34m74%|███████▍  | 261/352 [12:53<02:57,  1.95s/it]\u001b[0m\n",
      "\u001b[34m74%|███████▍  | 262/352 [12:55<03:13,  2.15s/it]\u001b[0m\n",
      "\u001b[34m75%|███████▍  | 263/352 [12:57<03:08,  2.11s/it]\u001b[0m\n",
      "\u001b[34m75%|███████▌  | 264/352 [12:59<02:53,  1.97s/it]\u001b[0m\n",
      "\u001b[34m75%|███████▌  | 265/352 [13:00<02:40,  1.85s/it]\u001b[0m\n",
      "\u001b[34m76%|███████▌  | 266/352 [13:02<02:40,  1.87s/it]\u001b[0m\n",
      "\u001b[34m76%|███████▌  | 267/352 [13:04<02:40,  1.89s/it]\u001b[0m\n",
      "\u001b[34m76%|███████▌  | 268/352 [13:06<02:34,  1.84s/it]\u001b[0m\n",
      "\u001b[34m76%|███████▋  | 269/352 [13:08<02:28,  1.79s/it]\u001b[0m\n",
      "\u001b[34m77%|███████▋  | 270/352 [13:10<02:27,  1.80s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.1644, 'learning_rate': 5.1770889908207245e-05, 'epoch': 1.53}\u001b[0m\n",
      "\u001b[34m77%|███████▋  | 270/352 [13:10<02:27,  1.80s/it]\u001b[0m\n",
      "\u001b[34m77%|███████▋  | 271/352 [13:11<02:27,  1.82s/it]\u001b[0m\n",
      "\u001b[34m77%|███████▋  | 272/352 [13:13<02:29,  1.87s/it]\u001b[0m\n",
      "\u001b[34m78%|███████▊  | 273/352 [13:15<02:30,  1.90s/it]\u001b[0m\n",
      "\u001b[34m78%|███████▊  | 274/352 [13:17<02:23,  1.84s/it]\u001b[0m\n",
      "\u001b[34m78%|███████▊  | 275/352 [13:19<02:20,  1.83s/it]\u001b[0m\n",
      "\u001b[34m78%|███████▊  | 276/352 [13:21<02:16,  1.80s/it]\u001b[0m\n",
      "\u001b[34m79%|███████▊  | 277/352 [13:22<02:16,  1.83s/it]\u001b[0m\n",
      "\u001b[34m79%|███████▉  | 278/352 [13:24<02:16,  1.84s/it]\u001b[0m\n",
      "\u001b[34m79%|███████▉  | 279/352 [13:26<02:17,  1.88s/it]\u001b[0m\n",
      "\u001b[34m80%|███████▉  | 280/352 [13:28<02:19,  1.93s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.1346, 'learning_rate': 4.2057706362665564e-05, 'epoch': 1.59}\u001b[0m\n",
      "\u001b[34m80%|███████▉  | 280/352 [13:28<02:19,  1.93s/it]\u001b[0m\n",
      "\u001b[34m80%|███████▉  | 281/352 [13:30<02:19,  1.96s/it]\u001b[0m\n",
      "\u001b[34m80%|████████  | 282/352 [13:32<02:09,  1.86s/it]\u001b[0m\n",
      "\u001b[34m80%|████████  | 283/352 [13:34<02:16,  1.97s/it]\u001b[0m\n",
      "\u001b[34m81%|████████  | 284/352 [13:36<02:06,  1.85s/it]\u001b[0m\n",
      "\u001b[34m81%|████████  | 285/352 [13:39<02:23,  2.13s/it]\u001b[0m\n",
      "\u001b[34m81%|████████▏ | 286/352 [13:40<02:10,  1.98s/it]\u001b[0m\n",
      "\u001b[34m82%|████████▏ | 287/352 [13:43<02:15,  2.08s/it]\u001b[0m\n",
      "\u001b[34m82%|████████▏ | 288/352 [13:44<02:08,  2.01s/it]\u001b[0m\n",
      "\u001b[34m82%|████████▏ | 289/352 [13:46<02:01,  1.92s/it]\u001b[0m\n",
      "\u001b[34m82%|████████▏ | 290/352 [13:48<01:55,  1.86s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.1717, 'learning_rate': 3.3203769292536764e-05, 'epoch': 1.65}\u001b[0m\n",
      "\u001b[34m82%|████████▏ | 290/352 [13:48<01:55,  1.86s/it]\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:21:06,387] [INFO] [loss_scaler.py:183:update_scale] [deepspeed] OVERFLOW! Rank 0 Skipping step. Attempted loss scale: 262144, reducing to 131072\u001b[0m\n",
      "\u001b[34m83%|████████▎ | 291/352 [13:50<01:50,  1.81s/it]\u001b[0m\n",
      "\u001b[34m83%|████████▎ | 292/352 [13:51<01:47,  1.79s/it]\u001b[0m\n",
      "\u001b[34m83%|████████▎ | 293/352 [13:53<01:46,  1.80s/it]\u001b[0m\n",
      "\u001b[34m84%|████████▎ | 294/352 [13:55<01:43,  1.78s/it]\u001b[0m\n",
      "\u001b[34m84%|████████▍ | 295/352 [13:57<01:41,  1.79s/it]\u001b[0m\n",
      "\u001b[34m84%|████████▍ | 296/352 [13:59<01:41,  1.81s/it]\u001b[0m\n",
      "\u001b[34m84%|████████▍ | 297/352 [14:00<01:41,  1.85s/it]\u001b[0m\n",
      "\u001b[34m85%|████████▍ | 298/352 [14:02<01:42,  1.90s/it]\u001b[0m\n",
      "\u001b[34m85%|████████▍ | 299/352 [14:04<01:35,  1.80s/it]\u001b[0m\n",
      "\u001b[34m85%|████████▌ | 300/352 [14:06<01:33,  1.80s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.1648, 'learning_rate': 2.6028283476858038e-05, 'epoch': 1.7}\u001b[0m\n",
      "\u001b[34m85%|████████▌ | 300/352 [14:06<01:33,  1.80s/it]\u001b[0m\n",
      "\u001b[34m07/20/2023 06:22:12 - INFO - llmtuner.tuner.core.trainer - Saving model checkpoint to /tmp/ouput/checkpoint-300\u001b[0m\n",
      "\u001b[34mINFO:llmtuner.tuner.core.trainer:Saving model checkpoint to /tmp/ouput/checkpoint-300\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:22:17,394] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step300 is about to be saved!\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:22:36,140] [INFO] [logging.py:96:log_dist] [Rank 0] Saving model checkpoint: /tmp/ouput/checkpoint-300/global_step300/mp_rank_00_model_states.pt\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:22:36,140] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving /tmp/ouput/checkpoint-300/global_step300/mp_rank_00_model_states.pt...\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:23:42,414] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved /tmp/ouput/checkpoint-300/global_step300/mp_rank_00_model_states.pt.\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:23:43,243] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving /tmp/ouput/checkpoint-300/global_step300/zero_pp_rank_0_mp_rank_00_optim_states.pt...\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:23:43,780] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved /tmp/ouput/checkpoint-300/global_step300/zero_pp_rank_0_mp_rank_00_optim_states.pt.\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:23:43,781] [INFO] [engine.py:3285:_save_zero_checkpoint] zero checkpoint saved /tmp/ouput/checkpoint-300/global_step300/zero_pp_rank_0_mp_rank_00_optim_states.pt\u001b[0m\n",
      "\u001b[34m[2023-07-20 06:23:43,781] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step300 is ready now!\u001b[0m\n",
      "\u001b[34m86%|████████▌ | 301/352 [16:29<37:31, 44.15s/it]\u001b[0m\n",
      "\u001b[34m86%|████████▌ | 302/352 [16:31<26:18, 31.58s/it]\u001b[0m\n",
      "\u001b[34m86%|████████▌ | 303/352 [16:33<18:28, 22.62s/it]\u001b[0m\n",
      "\u001b[34m86%|████████▋ | 304/352 [16:35<13:11, 16.49s/it]\u001b[0m\n",
      "\u001b[34m87%|████████▋ | 305/352 [16:37<09:28, 12.10s/it]\u001b[0m\n",
      "\u001b[34m87%|████████▋ | 306/352 [16:39<06:53,  8.99s/it]\u001b[0m\n",
      "\u001b[34m87%|████████▋ | 307/352 [16:40<05:06,  6.81s/it]\u001b[0m\n",
      "\u001b[34m88%|████████▊ | 308/352 [16:42<03:58,  5.42s/it]\u001b[0m\n",
      "\u001b[34m88%|████████▊ | 309/352 [16:44<03:08,  4.38s/it]\u001b[0m\n",
      "\u001b[34m88%|████████▊ | 310/352 [16:46<02:31,  3.60s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.1649, 'learning_rate': 1.8994979473690537e-05, 'epoch': 1.76}\u001b[0m\n",
      "\u001b[34m88%|████████▊ | 310/352 [16:46<02:31,  3.60s/it]\u001b[0m\n",
      "\u001b[34m88%|████████▊ | 311/352 [16:48<02:03,  3.01s/it]\u001b[0m\n",
      "\u001b[34m89%|████████▊ | 312/352 [16:50<01:51,  2.79s/it]\u001b[0m\n",
      "\u001b[34m89%|████████▉ | 313/352 [16:53<01:45,  2.70s/it]\u001b[0m\n",
      "\u001b[34m89%|████████▉ | 314/352 [16:54<01:32,  2.43s/it]\u001b[0m\n",
      "\u001b[34m89%|████████▉ | 315/352 [16:57<01:28,  2.39s/it]\u001b[0m\n",
      "\u001b[34m90%|████████▉ | 316/352 [16:58<01:19,  2.22s/it]\u001b[0m\n",
      "\u001b[34m90%|█████████ | 317/352 [17:00<01:13,  2.11s/it]\u001b[0m\n",
      "\u001b[34m90%|█████████ | 318/352 [17:03<01:13,  2.17s/it]\u001b[0m\n",
      "\u001b[34m91%|█████████ | 319/352 [17:04<01:07,  2.05s/it]\u001b[0m\n",
      "\u001b[34m91%|█████████ | 320/352 [17:06<01:04,  2.01s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.1485, 'learning_rate': 1.3004506768205226e-05, 'epoch': 1.82}\u001b[0m\n",
      "\u001b[34m91%|█████████ | 320/352 [17:06<01:04,  2.01s/it]\u001b[0m\n",
      "\u001b[34m91%|█████████ | 321/352 [17:09<01:04,  2.08s/it]\u001b[0m\n",
      "\u001b[34m91%|█████████▏| 322/352 [17:10<00:59,  1.98s/it]\u001b[0m\n",
      "\u001b[34m92%|█████████▏| 323/352 [17:12<00:56,  1.94s/it]\u001b[0m\n",
      "\u001b[34m92%|█████████▏| 324/352 [17:14<00:52,  1.88s/it]\u001b[0m\n",
      "\u001b[34m92%|█████████▏| 325/352 [17:16<00:49,  1.83s/it]\u001b[0m\n",
      "\u001b[34m93%|█████████▎| 326/352 [17:17<00:47,  1.83s/it]\u001b[0m\n",
      "\u001b[34m93%|█████████▎| 327/352 [17:19<00:45,  1.82s/it]\u001b[0m\n",
      "\u001b[34m93%|█████████▎| 328/352 [17:21<00:44,  1.83s/it]\u001b[0m\n",
      "\u001b[34m93%|█████████▎| 329/352 [17:23<00:42,  1.84s/it]\u001b[0m\n",
      "\u001b[34m94%|█████████▍| 330/352 [17:25<00:41,  1.89s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.1208, 'learning_rate': 8.104550955962469e-06, 'epoch': 1.88}\u001b[0m\n",
      "\u001b[34m94%|█████████▍| 330/352 [17:25<00:41,  1.89s/it]\u001b[0m\n",
      "\u001b[34m94%|█████████▍| 331/352 [17:27<00:39,  1.90s/it]\u001b[0m\n",
      "\u001b[34m94%|█████████▍| 332/352 [17:29<00:37,  1.90s/it]\u001b[0m\n",
      "\u001b[34m95%|█████████▍| 333/352 [17:31<00:35,  1.85s/it]\u001b[0m\n",
      "\u001b[34m95%|█████████▍| 334/352 [17:32<00:33,  1.89s/it]\u001b[0m\n",
      "\u001b[34m95%|█████████▌| 335/352 [17:35<00:34,  2.03s/it]\u001b[0m\n",
      "\u001b[34m95%|█████████▌| 336/352 [17:36<00:30,  1.90s/it]\u001b[0m\n",
      "\u001b[34m96%|█████████▌| 337/352 [17:38<00:28,  1.93s/it]\u001b[0m\n",
      "\u001b[34m96%|█████████▌| 338/352 [17:40<00:27,  1.96s/it]\u001b[0m\n",
      "\u001b[34m96%|█████████▋| 339/352 [17:42<00:24,  1.92s/it]\u001b[0m\n",
      "\u001b[34m97%|█████████▋| 340/352 [17:44<00:23,  1.94s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.1787, 'learning_rate': 4.334116857218317e-06, 'epoch': 1.93}\u001b[0m\n",
      "\u001b[34m97%|█████████▋| 340/352 [17:44<00:23,  1.94s/it]\u001b[0m\n",
      "\u001b[34m97%|█████████▋| 341/352 [17:46<00:20,  1.90s/it]\u001b[0m\n",
      "\u001b[34m97%|█████████▋| 342/352 [17:48<00:18,  1.80s/it]\u001b[0m\n",
      "\u001b[34m97%|█████████▋| 343/352 [17:50<00:16,  1.82s/it]\u001b[0m\n",
      "\u001b[34m98%|█████████▊| 344/352 [17:51<00:14,  1.76s/it]\u001b[0m\n",
      "\u001b[34m98%|█████████▊| 345/352 [17:53<00:12,  1.84s/it]\u001b[0m\n",
      "\u001b[34m98%|█████████▊| 346/352 [17:55<00:10,  1.81s/it]\u001b[0m\n",
      "\u001b[34m99%|█████████▊| 347/352 [17:57<00:08,  1.74s/it]\u001b[0m\n",
      "\u001b[34m99%|█████████▉| 348/352 [17:58<00:06,  1.70s/it]\u001b[0m\n",
      "\u001b[34m99%|█████████▉| 349/352 [18:00<00:05,  1.76s/it]\u001b[0m\n",
      "\u001b[34m99%|█████████▉| 350/352 [18:02<00:03,  1.85s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.0996, 'learning_rate': 1.7232180292259369e-06, 'epoch': 1.99}\u001b[0m\n",
      "\u001b[34m99%|█████████▉| 350/352 [18:02<00:03,  1.85s/it]\u001b[0m\n",
      "\u001b[34m100%|█████████▉| 351/352 [18:04<00:01,  1.80s/it]\u001b[0m\n",
      "\u001b[34m100%|██████████| 352/352 [18:06<00:00,  1.79s/it]\u001b[0m\n",
      "\u001b[34m[INFO|trainer.py:1934] 2023-07-20 06:25:22,416 >> \u001b[0m\n",
      "\u001b[34mTraining completed. Do not forget to share your model on huggingface.co/models =)\u001b[0m\n",
      "\u001b[34m[INFO|trainer.py:1934] 2023-07-20 06:25:22,416 >> \u001b[0m\n",
      "\u001b[34mTraining completed. Do not forget to share your model on huggingface.co/models =)\u001b[0m\n",
      "\u001b[34m{'train_runtime': 1096.9803, 'train_samples_per_second': 20.536, 'train_steps_per_second': 0.321, 'train_loss': 1.3334588557481766, 'epoch': 2.0}\u001b[0m\n",
      "\u001b[34m100%|██████████| 352/352 [18:06<00:00,  1.79s/it]\u001b[0m\n",
      "\u001b[34m100%|██████████| 352/352 [18:06<00:00,  3.09s/it]\u001b[0m\n",
      "\u001b[34m***** train metrics *****\u001b[0m\n",
      "\u001b[34mepoch                    =        2.0\u001b[0m\n",
      "\u001b[34mtrain_loss               =     1.3335\u001b[0m\n",
      "\u001b[34mtrain_runtime            = 0:18:16.98\u001b[0m\n",
      "\u001b[34mtrain_samples_per_second =     20.536\u001b[0m\n",
      "\u001b[34mtrain_steps_per_second   =      0.321\u001b[0m\n",
      "\u001b[34m07/20/2023 06:26:12 - INFO - llmtuner.tuner.core.trainer - Saving model checkpoint to /tmp/ouput/\u001b[0m\n",
      "\u001b[34mINFO:llmtuner.tuner.core.trainer:Saving model checkpoint to /tmp/ouput/\u001b[0m\n",
      "\u001b[34mFigure saved:\u001b[0m\n",
      "\u001b[34m/tmp/ouput/training_loss.png\u001b[0m\n",
      "\u001b[34m07/20/2023 06:26:17 - WARNING - llmtuner.extras.ploting - No metric eval_loss to plot.\u001b[0m\n",
      "\u001b[34mWARNING:llmtuner.extras.ploting:No metric eval_loss to plot.\u001b[0m\n",
      "\u001b[34mwandb: Waiting for W&B process to finish... (success).\u001b[0m\n",
      "\u001b[34mwandb: \u001b[0m\n",
      "\u001b[34mwandb: Run history:\u001b[0m\n",
      "\u001b[34mwandb:                    train/epoch ▁▁▁▂▂▂▂▂▃▃▃▃▃▄▄▄▄▄▅▅▅▅▆▆▆▆▆▇▇▇▇▇████\u001b[0m\n",
      "\u001b[34mwandb:              train/global_step ▁▁▁▂▂▂▂▂▃▃▃▃▃▄▄▄▄▄▅▅▅▅▆▆▆▆▆▇▇▇▇▇████\u001b[0m\n",
      "\u001b[34mwandb:            train/learning_rate ███████▇▇▇▇▇▆▆▆▅▅▅▄▄▄▄▃▃▃▂▂▂▂▂▁▁▁▁▁\u001b[0m\n",
      "\u001b[34mwandb:                     train/loss █▇▄▄▃▄▃▃▃▃▃▃▃▃▃▃▃▂▂▁▂▂▁▁▁▁▁▁▁▁▁▁▁▂▁\u001b[0m\n",
      "\u001b[34mwandb:               train/total_flos ▁\u001b[0m\n",
      "\u001b[34mwandb:               train/train_loss ▁\u001b[0m\n",
      "\u001b[34mwandb:            train/train_runtime ▁\u001b[0m\n",
      "\u001b[34mwandb: train/train_samples_per_second ▁\u001b[0m\n",
      "\u001b[34mwandb:   train/train_steps_per_second ▁\u001b[0m\n",
      "\u001b[34mwandb: \u001b[0m\n",
      "\u001b[34mwandb: Run summary:\u001b[0m\n",
      "\u001b[34mwandb:                    train/epoch 2.0\u001b[0m\n",
      "\u001b[34mwandb:              train/global_step 352\u001b[0m\n",
      "\u001b[34mwandb:            train/learning_rate 0.0\u001b[0m\n",
      "\u001b[34mwandb:                     train/loss 1.0996\u001b[0m\n",
      "\u001b[34mwandb:               train/total_flos 3.874963959442309e+17\u001b[0m\n",
      "\u001b[34mwandb:               train/train_loss 1.33346\u001b[0m\n",
      "\u001b[34mwandb:            train/train_runtime 1096.9803\u001b[0m\n",
      "\u001b[34mwandb: train/train_samples_per_second 20.536\u001b[0m\n",
      "\u001b[34mwandb:   train/train_steps_per_second 0.321\u001b[0m\n",
      "\u001b[34mwandb:\u001b[0m\n",
      "\u001b[34mwandb: 🚀 View run baichuan-finetuning-2023-07-20-05-51-37-413-sqcyuk-algo-1 at: https://wandb.ai/imtty/sm-baichuan7b-sft/runs/baichuan-finetuning-2023-07-20-05-51-37-413-sqcyuk-algo-1\u001b[0m\n",
      "\u001b[34mwandb: Synced 6 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)\u001b[0m\n",
      "\u001b[34mwandb: Find logs at: ./wandb/run-20230720_060706-baichuan-finetuning-2023-07-20-05-51-37-413-sqcyuk-algo-1/logs\u001b[0m\n",
      "\u001b[34m2023-07-20 06:26:28,840 sagemaker-training-toolkit INFO     Waiting for the process to finish and give a return code.\u001b[0m\n",
      "\u001b[34m2023-07-20 06:26:28,840 sagemaker-training-toolkit INFO     Done waiting for a return code. Received 0 from exiting process.\u001b[0m\n",
      "\u001b[34m2023-07-20 06:26:28,841 sagemaker-training-toolkit INFO     Reporting training SUCCESS\u001b[0m\n",
      "\n",
      "2023-07-20 06:26:43 Uploading - Uploading generated training model\n",
      "2023-07-20 06:26:43 Completed - Training job completed\n",
      "Training seconds: 1828\n",
      "Billable seconds: 1828\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "from sagemaker.estimator import Estimator\n",
    "\n",
    "environment = {\n",
    "              'MODEL_S3_PATH': model_s3_path # The bucket to store pretrained model and fine-tune model\n",
    "}\n",
    "\n",
    "base_job_name = 'baichuan-finetuning'\n",
    "\n",
    "# instance_type = 'ml.g5.12xlarge'\n",
    "\n",
    "# instance_type = 'ml.g5.48xlarge'\n",
    "\n",
    "instance_type = 'ml.p4d.24xlarge'\n",
    "\n",
    "\n",
    "estimator = Estimator(role=role,\n",
    "                      entry_point='train.sh',\n",
    "                      source_dir='./LLaMA-Efficient-Tuning/',\n",
    "                      base_job_name=base_job_name,\n",
    "                      instance_count=1,\n",
    "                      instance_type=instance_type,\n",
    "                      image_uri=image_uri,\n",
    "                      environment=environment,\n",
    "                      disable_profiler=True,\n",
    "                      debugger_hook_config=False,\n",
    "                      max_run=max_run,\n",
    "                      use_spot_instances=use_spot_instances,\n",
    "                      max_wait=max_wait\n",
    "                     )\n",
    "\n",
    "estimator.fit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e498053a-03e8-4847-9c34-6fc1c45edc09",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import subprocess"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "28b454d7-3e4c-4d1f-bca0-63eec96925fc",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# os.system('./s5cmd sync monitor.sh s3://sagemaker-us-west-2-960661357527/llm/models/LLM_baichuan_model/')\n",
    "\n",
    "subprocess.run('./s5cmd sync Dockerfile s3://sagemaker-us-west-2-960661357527/llm/models/LLM_baichuan_model/', shell=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1aa77f43-8aae-4621-9474-ea1797c913f0",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "89ecf7fb-6eef-49fa-bef2-01571366f37a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cb06c597-9e88-44ae-ad8c-47a1ed622cbe",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "datetime.datetime.now().strftime(\"%Y-%m-%d-%H-%M-%S\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b79f0f6a-a82d-42e5-a0e5-ad19d9cdfeb1",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!pip install accelerate"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7ab8accf-7d94-41c2-a44f-bee60aef30f7",
   "metadata": {
    "tags": []
   },
   "source": [
    "## !accelerate config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "a84a1cfd-1f2c-47f5-8cbb-d4e957d62afd",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import transformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "72fc3433-943e-40b0-843e-27a77cb0d5ca",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from transformers import AutoConfig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "2696f6b7-9c27-4599-b26d-e5e54f280f1f",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "A new version of the following files was downloaded from https://huggingface.co/baichuan-inc/Baichuan-13B-Chat:\n",
      "- configuration_baichuan.py\n",
      ". Make sure to double-check they do not contain any added malicious code. To avoid downloading new versions of the code file, you can pin a revision.\n"
     ]
    }
   ],
   "source": [
    "config = AutoConfig.from_pretrained('baichuan-inc/Baichuan-13B-Chat', trust_remote_code=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "ad62b06c-ceb6-4a90-8ab3-d24c37b6f735",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BaichuanConfig {\n",
       "  \"_from_model_config\": true,\n",
       "  \"_name_or_path\": \"baichuan-inc/Baichuan-13B-Chat\",\n",
       "  \"architectures\": [\n",
       "    \"BaichuanForCausalLM\"\n",
       "  ],\n",
       "  \"auto_map\": {\n",
       "    \"AutoConfig\": \"baichuan-inc/Baichuan-13B-Chat--configuration_baichuan.BaichuanConfig\",\n",
       "    \"AutoModelForCausalLM\": \"baichuan-inc/Baichuan-13B-Chat--modeling_baichuan.BaichuanForCausalLM\"\n",
       "  },\n",
       "  \"bos_token_id\": 1,\n",
       "  \"eos_token_id\": 2,\n",
       "  \"gradient_checkpointing\": [\n",
       "    false\n",
       "  ],\n",
       "  \"hidden_act\": \"silu\",\n",
       "  \"hidden_size\": 5120,\n",
       "  \"initializer_range\": 0.02,\n",
       "  \"intermediate_size\": 13696,\n",
       "  \"model_max_length\": 4096,\n",
       "  \"model_type\": \"baichuan\",\n",
       "  \"num_attention_heads\": 40,\n",
       "  \"num_hidden_layers\": 40,\n",
       "  \"pad_token_id\": 0,\n",
       "  \"rms_norm_eps\": 1e-06,\n",
       "  \"tie_word_embeddings\": false,\n",
       "  \"torch_dtype\": \"bfloat16\",\n",
       "  \"transformers_version\": \"4.29.0\",\n",
       "  \"use_cache\": true,\n",
       "  \"vocab_size\": 64000\n",
       "}"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "fb24dc60-084b-473b-ac57-6d93686c2400",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "128.0"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "142cd03f-e9ad-400b-b1c2-46b1069e57da",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_amazonei_pytorch_latest_p37",
   "language": "python",
   "name": "conda_amazonei_pytorch_latest_p37"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}