baichuan_fine_tune_ligtning_loRA.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "252de0de",
   "metadata": {
    "tags": []
   },
   "source": [
    "### SageMaker fine tune baichuan\n",
    "\n",
    "#### 准备\n",
    "1. 升级boto3, sagemaker python sdk  \n",
    "2. 准备requirements.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b8f2c403",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Looking in indexes: https://pypi.org/simple, https://pip.repos.neuron.amazonaws.com\n",
      "Requirement already satisfied: boto3 in /home/ec2-user/anaconda3/envs/amazonei_pytorch_latest_p37/lib/python3.7/site-packages (1.26.114)\n",
      "Collecting boto3\n",
      "  Downloading boto3-1.28.1-py3-none-any.whl (135 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m135.7/135.7 kB\u001b[0m \u001b[31m6.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting botocore<1.32.0,>=1.31.1\n",
      "  Downloading botocore-1.31.1-py3-none-any.whl (11.0 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m11.0/11.0 MB\u001b[0m \u001b[31m116.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m0:01\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: jmespath<2.0.0,>=0.7.1 in /home/ec2-user/anaconda3/envs/amazonei_pytorch_latest_p37/lib/python3.7/site-packages (from boto3) (1.0.1)\n",
      "Requirement already satisfied: s3transfer<0.7.0,>=0.6.0 in /home/ec2-user/anaconda3/envs/amazonei_pytorch_latest_p37/lib/python3.7/site-packages (from boto3) (0.6.0)\n",
      "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /home/ec2-user/anaconda3/envs/amazonei_pytorch_latest_p37/lib/python3.7/site-packages (from botocore<1.32.0,>=1.31.1->boto3) (2.8.2)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.25.4 in /home/ec2-user/anaconda3/envs/amazonei_pytorch_latest_p37/lib/python3.7/site-packages (from botocore<1.32.0,>=1.31.1->boto3) (1.26.8)\n",
      "Requirement already satisfied: six>=1.5 in /home/ec2-user/anaconda3/envs/amazonei_pytorch_latest_p37/lib/python3.7/site-packages (from python-dateutil<3.0.0,>=2.1->botocore<1.32.0,>=1.31.1->boto3) (1.16.0)\n",
      "Installing collected packages: botocore, boto3\n",
      "  Attempting uninstall: botocore\n",
      "    Found existing installation: botocore 1.29.114\n",
      "    Uninstalling botocore-1.29.114:\n",
      "      Successfully uninstalled botocore-1.29.114\n",
      "  Attempting uninstall: boto3\n",
      "    Found existing installation: boto3 1.26.114\n",
      "    Uninstalling boto3-1.26.114:\n",
      "      Successfully uninstalled boto3-1.26.114\n",
      "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
      "awscli 1.27.114 requires botocore==1.29.114, but you have botocore 1.31.1 which is incompatible.\u001b[0m\u001b[31m\n",
      "\u001b[0mSuccessfully installed boto3-1.28.1 botocore-1.31.1\n",
      "Looking in indexes: https://pypi.org/simple, https://pip.repos.neuron.amazonaws.com\n",
      "Requirement already satisfied: sagemaker in /home/ec2-user/anaconda3/envs/amazonei_pytorch_latest_p37/lib/python3.7/site-packages (2.153.0)\n",
      "Collecting sagemaker\n",
      "  Downloading sagemaker-2.171.0.tar.gz (853 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m853.5/853.5 kB\u001b[0m \u001b[31m16.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hCollecting attrs<24,>=23.1.0\n",
      "  Using cached attrs-23.1.0-py3-none-any.whl (61 kB)\n",
      "Requirement already satisfied: boto3<2.0,>=1.26.131 in /home/ec2-user/anaconda3/envs/amazonei_pytorch_latest_p37/lib/python3.7/site-packages (from sagemaker) (1.28.1)\n",
      "Requirement already satisfied: cloudpickle==2.2.1 in /home/ec2-user/anaconda3/envs/amazonei_pytorch_latest_p37/lib/python3.7/site-packages (from sagemaker) (2.2.1)\n",
      "Requirement already satisfied: google-pasta in /home/ec2-user/anaconda3/envs/amazonei_pytorch_latest_p37/lib/python3.7/site-packages (from sagemaker) (0.2.0)\n",
      "Requirement already satisfied: numpy<2.0,>=1.9.0 in /home/ec2-user/anaconda3/envs/amazonei_pytorch_latest_p37/lib/python3.7/site-packages (from sagemaker) (1.21.6)\n",
      "Requirement already satisfied: protobuf<4.0,>=3.1 in /home/ec2-user/anaconda3/envs/amazonei_pytorch_latest_p37/lib/python3.7/site-packages (from sagemaker) (3.20.3)\n",
      "Requirement already satisfied: smdebug_rulesconfig==1.0.1 in /home/ec2-user/anaconda3/envs/amazonei_pytorch_latest_p37/lib/python3.7/site-packages (from sagemaker) (1.0.1)\n",
      "Requirement already satisfied: importlib-metadata<5.0,>=1.4.0 in /home/ec2-user/anaconda3/envs/amazonei_pytorch_latest_p37/lib/python3.7/site-packages (from sagemaker) (4.11.4)\n",
      "Requirement already satisfied: packaging>=20.0 in /home/ec2-user/anaconda3/envs/amazonei_pytorch_latest_p37/lib/python3.7/site-packages (from sagemaker) (21.3)\n",
      "Requirement already satisfied: pandas in /home/ec2-user/anaconda3/envs/amazonei_pytorch_latest_p37/lib/python3.7/site-packages (from sagemaker) (1.3.5)\n",
      "Requirement already satisfied: pathos in /home/ec2-user/anaconda3/envs/amazonei_pytorch_latest_p37/lib/python3.7/site-packages (from sagemaker) (0.3.0)\n",
      "Requirement already satisfied: schema in /home/ec2-user/anaconda3/envs/amazonei_pytorch_latest_p37/lib/python3.7/site-packages (from sagemaker) (0.7.5)\n",
      "Collecting PyYAML==6.0\n",
      "  Downloading PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (596 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m596.3/596.3 kB\u001b[0m \u001b[31m66.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: jsonschema in /home/ec2-user/anaconda3/envs/amazonei_pytorch_latest_p37/lib/python3.7/site-packages (from sagemaker) (3.2.0)\n",
      "Requirement already satisfied: platformdirs in /home/ec2-user/anaconda3/envs/amazonei_pytorch_latest_p37/lib/python3.7/site-packages (from sagemaker) (3.2.0)\n",
      "Requirement already satisfied: tblib==1.7.0 in /home/ec2-user/anaconda3/envs/amazonei_pytorch_latest_p37/lib/python3.7/site-packages (from sagemaker) (1.7.0)\n",
      "Requirement already satisfied: s3transfer<0.7.0,>=0.6.0 in /home/ec2-user/anaconda3/envs/amazonei_pytorch_latest_p37/lib/python3.7/site-packages (from boto3<2.0,>=1.26.131->sagemaker) (0.6.0)\n",
      "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /home/ec2-user/anaconda3/envs/amazonei_pytorch_latest_p37/lib/python3.7/site-packages (from boto3<2.0,>=1.26.131->sagemaker) (1.0.1)\n",
      "Requirement already satisfied: botocore<1.32.0,>=1.31.1 in /home/ec2-user/anaconda3/envs/amazonei_pytorch_latest_p37/lib/python3.7/site-packages (from boto3<2.0,>=1.26.131->sagemaker) (1.31.1)\n",
      "Requirement already satisfied: typing-extensions>=3.6.4 in /home/ec2-user/anaconda3/envs/amazonei_pytorch_latest_p37/lib/python3.7/site-packages (from importlib-metadata<5.0,>=1.4.0->sagemaker) (4.5.0)\n",
      "Requirement already satisfied: zipp>=0.5 in /home/ec2-user/anaconda3/envs/amazonei_pytorch_latest_p37/lib/python3.7/site-packages (from importlib-metadata<5.0,>=1.4.0->sagemaker) (3.11.0)\n",
      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /home/ec2-user/anaconda3/envs/amazonei_pytorch_latest_p37/lib/python3.7/site-packages (from packaging>=20.0->sagemaker) (3.0.9)\n",
      "Requirement already satisfied: six in /home/ec2-user/anaconda3/envs/amazonei_pytorch_latest_p37/lib/python3.7/site-packages (from google-pasta->sagemaker) (1.16.0)\n",
      "Requirement already satisfied: setuptools in /home/ec2-user/anaconda3/envs/amazonei_pytorch_latest_p37/lib/python3.7/site-packages (from jsonschema->sagemaker) (65.6.3)\n",
      "Requirement already satisfied: pyrsistent>=0.14.0 in /home/ec2-user/anaconda3/envs/amazonei_pytorch_latest_p37/lib/python3.7/site-packages (from jsonschema->sagemaker) (0.18.1)\n",
      "Requirement already satisfied: pytz>=2017.3 in /home/ec2-user/anaconda3/envs/amazonei_pytorch_latest_p37/lib/python3.7/site-packages (from pandas->sagemaker) (2022.7)\n",
      "Requirement already satisfied: python-dateutil>=2.7.3 in /home/ec2-user/anaconda3/envs/amazonei_pytorch_latest_p37/lib/python3.7/site-packages (from pandas->sagemaker) (2.8.2)\n",
      "Requirement already satisfied: ppft>=1.7.6.6 in /home/ec2-user/anaconda3/envs/amazonei_pytorch_latest_p37/lib/python3.7/site-packages (from pathos->sagemaker) (1.7.6.6)\n",
      "Requirement already satisfied: pox>=0.3.2 in /home/ec2-user/anaconda3/envs/amazonei_pytorch_latest_p37/lib/python3.7/site-packages (from pathos->sagemaker) (0.3.2)\n",
      "Requirement already satisfied: multiprocess>=0.70.14 in /home/ec2-user/anaconda3/envs/amazonei_pytorch_latest_p37/lib/python3.7/site-packages (from pathos->sagemaker) (0.70.14)\n",
      "Requirement already satisfied: dill>=0.3.6 in /home/ec2-user/anaconda3/envs/amazonei_pytorch_latest_p37/lib/python3.7/site-packages (from pathos->sagemaker) (0.3.6)\n",
      "Requirement already satisfied: contextlib2>=0.5.5 in /home/ec2-user/anaconda3/envs/amazonei_pytorch_latest_p37/lib/python3.7/site-packages (from schema->sagemaker) (21.6.0)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.25.4 in /home/ec2-user/anaconda3/envs/amazonei_pytorch_latest_p37/lib/python3.7/site-packages (from botocore<1.32.0,>=1.31.1->boto3<2.0,>=1.26.131->sagemaker) (1.26.8)\n",
      "Building wheels for collected packages: sagemaker\n",
      "  Building wheel for sagemaker (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for sagemaker: filename=sagemaker-2.171.0-py2.py3-none-any.whl size=1160877 sha256=48bf895e5c58ad4d363b3065b1121f7e58cfc76b7f65a95942c4d431a0b6d8b8\n",
      "  Stored in directory: /home/ec2-user/.cache/pip/wheels/ad/56/36/62408964fea2c559a20348ddbdb498e9eff40886b9157c30b8\n",
      "Successfully built sagemaker\n",
      "Installing collected packages: PyYAML, attrs, sagemaker\n",
      "  Attempting uninstall: PyYAML\n",
      "    Found existing installation: PyYAML 5.4.1\n",
      "    Uninstalling PyYAML-5.4.1:\n",
      "      Successfully uninstalled PyYAML-5.4.1\n",
      "  Attempting uninstall: attrs\n",
      "    Found existing installation: attrs 22.2.0\n",
      "    Uninstalling attrs-22.2.0:\n",
      "      Successfully uninstalled attrs-22.2.0\n",
      "  Attempting uninstall: sagemaker\n",
      "    Found existing installation: sagemaker 2.153.0\n",
      "    Uninstalling sagemaker-2.153.0:\n",
      "      Successfully uninstalled sagemaker-2.153.0\n",
      "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
      "docker-compose 1.29.2 requires PyYAML<6,>=3.10, but you have pyyaml 6.0 which is incompatible.\n",
      "awscli 1.27.114 requires botocore==1.29.114, but you have botocore 1.31.1 which is incompatible.\n",
      "awscli 1.27.114 requires PyYAML<5.5,>=3.10, but you have pyyaml 6.0 which is incompatible.\u001b[0m\u001b[31m\n",
      "\u001b[0mSuccessfully installed PyYAML-6.0 attrs-23.1.0 sagemaker-2.171.0\n"
     ]
    }
   ],
   "source": [
    "!pip install --upgrade boto3\n",
    "!pip install --upgrade sagemaker"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "a4a30f3a",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ec2-user/anaconda3/envs/amazonei_pytorch_latest_p37/lib/python3.7/site-packages/boto3/compat.py:82: PythonDeprecationWarning: Boto3 will no longer support Python 3.7 starting December 13, 2023. To continue receiving service updates, bug fixes, and security updates please upgrade to Python 3.8 or later. More information can be found here: https://aws.amazon.com/blogs/developer/python-support-policy-updates-for-aws-sdks-and-tools/\n",
      "  warnings.warn(warning, PythonDeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "arn:aws:iam::687912291502:role/service-role/AmazonSageMaker-ExecutionRole-20211013T113123\n",
      "sagemaker-us-west-2-687912291502\n"
     ]
    }
   ],
   "source": [
    "import boto3\n",
    "import sagemaker\n",
    "\n",
    "account = boto3.client('sts').get_caller_identity().get('Account')\n",
    "region = boto3.session.Session().region_name\n",
    "\n",
    "sagemaker_session = sagemaker.Session()\n",
    "bucket = sagemaker_session.default_bucket()\n",
    "role = sagemaker.get_execution_role()\n",
    "\n",
    "print(role)\n",
    "print(bucket)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f6f59e3f",
   "metadata": {
    "tags": []
   },
   "source": [
    "### baichuan fine tune \n",
    "1:使用pytorch ligtening框架  \n",
    "2:deepspeed+QLoRA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "9db2f288-0610-4b18-9409-c8ef5ee91ca0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cloning into 'LLaMA-Efficient-Tuning'...\n",
      "remote: Enumerating objects: 730, done.\u001b[K\n",
      "remote: Counting objects: 100% (303/303), done.\u001b[K\n",
      "remote: Compressing objects: 100% (45/45), done.\u001b[K\n",
      "remote: Total 730 (delta 275), reused 259 (delta 258), pack-reused 427\u001b[K\n",
      "Receiving objects: 100% (730/730), 72.12 MiB | 16.24 MiB/s, done.\n",
      "Resolving deltas: 100% (515/515), done.\n",
      "Updating files: 100% (47/47), done.\n"
     ]
    }
   ],
   "source": [
    "#!rm -rf ./baichuan_finetuning\n",
    "#!git clone https://github.com/ssbuild/baichuan_finetuning.git(弃用）\n",
    "#!cp ./s5cmd ./baichuan_finetuning/\n",
    "!rm -rf ./LLaMA-Efficient-Tuning\n",
    "!git clone https://github.com/hiyouga/LLaMA-Efficient-Tuning.git\n",
    "!cp ./s5cmd ./LLaMA-Efficient-Tuning/"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b42e2794-2ed4-49f9-b1b0-054f1a0af54f",
   "metadata": {},
   "source": [
    "## prepare docker images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8aceb270-324f-445e-ae84-0da1d1b98adf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting Dockerfile\n"
     ]
    }
   ],
   "source": [
    "%%writefile Dockerfile\n",
    "## You should change below region code to the region you used, here sample is use us-west-2\n",
    "From 763104351884.dkr.ecr.us-west-2.amazonaws.com/huggingface-pytorch-training:1.13.1-transformers4.26.0-gpu-py39-cu117-ubuntu20.04 \n",
    "#From pytorch/pytorch:1.5-cuda10.1-cudnn7-runtime\n",
    "\n",
    "ENV LANG=C.UTF-8\n",
    "ENV PYTHONUNBUFFERED=TRUE\n",
    "ENV PYTHONDONTWRITEBYTECODE=TRUE\n",
    "\n",
    "#RUN pip install -U git+https://github.com/ssbuild/deep_training.git\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "3b8ee553",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING! Your password will be stored unencrypted in /home/ec2-user/.docker/config.json.\n",
      "Configure a credential helper to remove this warning. See\n",
      "https://docs.docker.com/engine/reference/commandline/login/#credentials-store\n",
      "\n",
      "Login Succeeded\n"
     ]
    }
   ],
   "source": [
    "## You should change below region code to the region you used, here sample is use us-west-2\n",
    "!aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin 763104351884.dkr.ecr.us-west-2.amazonaws.com"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c00b23c6-6e88-48f2-96dd-99140c147be5",
   "metadata": {},
   "source": [
    "**Build image and push to ECR.**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "fa970133-14f1-40d4-963f-895154a43f94",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "## define repo name, should contain *sagemaker* in the name\n",
    "repo_name = \"sagemaker-baichuan_finetuning\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "51c2d43e-ff0f-4718-b772-555c95d6aaaf",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Login Succeeded\n",
      "Sending build context to Docker daemon  29.04GB\n",
      "Step 1/4 : From 763104351884.dkr.ecr.us-west-2.amazonaws.com/huggingface-pytorch-training:1.13.1-transformers4.26.0-gpu-py39-cu117-ubuntu20.04\n",
      " ---> c5a6ef695006\n",
      "Step 2/4 : ENV LANG=C.UTF-8\n",
      " ---> Using cache\n",
      " ---> af49cfa7feae\n",
      "Step 3/4 : ENV PYTHONUNBUFFERED=TRUE\n",
      " ---> Using cache\n",
      " ---> 287106637dc6\n",
      "Step 4/4 : ENV PYTHONDONTWRITEBYTECODE=TRUE\n",
      " ---> Using cache\n",
      " ---> 773b4cf30c90\n",
      "Successfully built 773b4cf30c90\n",
      "Successfully tagged sagemaker-baichuan_finetuning:latest\n",
      "The push refers to repository [687912291502.dkr.ecr.us-west-2.amazonaws.com/sagemaker-baichuan_finetuning]\n",
      "f8dae5c3df1e: Preparing\n",
      "e3221f18601a: Preparing\n",
      "b6f286626882: Preparing\n",
      "76fe97d80cdb: Preparing\n",
      "f5f76489fff8: Preparing\n",
      "621c3f07daa7: Preparing\n",
      "9b484bb42e11: Preparing\n",
      "54c7c0b58471: Preparing\n",
      "c34adc3ab668: Preparing\n",
      "bbf651e48b84: Preparing\n",
      "f61045791108: Preparing\n",
      "4e2ac0cda74a: Preparing\n",
      "658a33d555eb: Preparing\n",
      "bd16d9a61a98: Preparing\n",
      "f0c0cd2accfa: Preparing\n",
      "1275469c066c: Preparing\n",
      "b802dd3babf4: Preparing\n",
      "a3834ec63558: Preparing\n",
      "63edcef6dedf: Preparing\n",
      "0154e84cc2dd: Preparing\n",
      "7085d1c151f6: Preparing\n",
      "a77a2104cfb6: Preparing\n",
      "9b484bb42e11: Waiting\n",
      "6808e7f9da2f: Preparing\n",
      "3bc059a9dec6: Preparing\n",
      "de783f3fec23: Preparing\n",
      "18ca52d74b2f: Preparing\n",
      "54c7c0b58471: Waiting\n",
      "73df6ccd636c: Preparing\n",
      "6738b73ff7a8: Preparing\n",
      "2a8292d9bfcc: Preparing\n",
      "5b75a5ef32a7: Preparing\n",
      "25a5f55a11f0: Preparing\n",
      "707f484816ae: Preparing\n",
      "0430aa1e47d4: Preparing\n",
      "65448e793131: Preparing\n",
      "15af6e2d42ba: Preparing\n",
      "b46caef92993: Preparing\n",
      "53ce33a12646: Preparing\n",
      "aad68760f4ce: Preparing\n",
      "658a33d555eb: Waiting\n",
      "323d67ab1719: Preparing\n",
      "3bc059a9dec6: Waiting\n",
      "e72743a0fdfe: Preparing\n",
      "3996353f5820: Preparing\n",
      "ea87e0b9c30f: Preparing\n",
      "bbf651e48b84: Waiting\n",
      "bd16d9a61a98: Waiting\n",
      "af18356cdf10: Preparing\n",
      "c34adc3ab668: Waiting\n",
      "f6e30dd4497e: Preparing\n",
      "f61045791108: Waiting\n",
      "99832d04a153: Preparing\n",
      "f0c0cd2accfa: Waiting\n",
      "de783f3fec23: Waiting\n",
      "a5981ed7a378: Preparing\n",
      "6738b73ff7a8: Waiting\n",
      "250519a2f830: Preparing\n",
      "6cadbde53f94: Preparing\n",
      "0002c93bdb37: Preparing\n",
      "ea87e0b9c30f: Waiting\n",
      "af18356cdf10: Waiting\n",
      "f6e30dd4497e: Waiting\n",
      "1275469c066c: Waiting\n",
      "99832d04a153: Waiting\n",
      "a5981ed7a378: Waiting\n",
      "e72743a0fdfe: Waiting\n",
      "18ca52d74b2f: Waiting\n",
      "2a8292d9bfcc: Waiting\n",
      "73df6ccd636c: Waiting\n",
      "3996353f5820: Waiting\n",
      "0002c93bdb37: Waiting\n",
      "250519a2f830: Waiting\n",
      "707f484816ae: Waiting\n",
      "25a5f55a11f0: Waiting\n",
      "65448e793131: Waiting\n",
      "323d67ab1719: Waiting\n",
      "53ce33a12646: Waiting\n",
      "a3834ec63558: Waiting\n",
      "63edcef6dedf: Waiting\n",
      "7085d1c151f6: Waiting\n",
      "6808e7f9da2f: Waiting\n",
      "b46caef92993: Waiting\n",
      "0154e84cc2dd: Waiting\n",
      "76fe97d80cdb: Layer already exists\n",
      "e3221f18601a: Layer already exists\n",
      "b6f286626882: Layer already exists\n",
      "f5f76489fff8: Layer already exists\n",
      "f8dae5c3df1e: Layer already exists\n",
      "621c3f07daa7: Layer already exists\n",
      "9b484bb42e11: Layer already exists\n",
      "bbf651e48b84: Layer already exists\n",
      "54c7c0b58471: Layer already exists\n",
      "c34adc3ab668: Layer already exists\n",
      "f61045791108: Layer already exists\n",
      "4e2ac0cda74a: Layer already exists\n",
      "658a33d555eb: Layer already exists\n",
      "bd16d9a61a98: Layer already exists\n",
      "f0c0cd2accfa: Layer already exists\n",
      "1275469c066c: Layer already exists\n",
      "a3834ec63558: Layer already exists\n",
      "b802dd3babf4: Layer already exists\n",
      "0154e84cc2dd: Layer already exists\n",
      "63edcef6dedf: Layer already exists\n",
      "7085d1c151f6: Layer already exists\n",
      "a77a2104cfb6: Layer already exists\n",
      "6808e7f9da2f: Layer already exists\n",
      "de783f3fec23: Layer already exists\n",
      "3bc059a9dec6: Layer already exists\n",
      "18ca52d74b2f: Layer already exists\n",
      "73df6ccd636c: Layer already exists\n",
      "6738b73ff7a8: Layer already exists\n",
      "2a8292d9bfcc: Layer already exists\n",
      "5b75a5ef32a7: Layer already exists\n",
      "25a5f55a11f0: Layer already exists\n",
      "707f484816ae: Layer already exists\n",
      "0430aa1e47d4: Layer already exists\n",
      "15af6e2d42ba: Layer already exists\n",
      "65448e793131: Layer already exists\n",
      "b46caef92993: Layer already exists\n",
      "aad68760f4ce: Layer already exists\n",
      "53ce33a12646: Layer already exists\n",
      "323d67ab1719: Layer already exists\n",
      "e72743a0fdfe: Layer already exists\n",
      "3996353f5820: Layer already exists\n",
      "ea87e0b9c30f: Layer already exists\n",
      "af18356cdf10: Layer already exists\n",
      "99832d04a153: Layer already exists\n",
      "f6e30dd4497e: Layer already exists\n",
      "a5981ed7a378: Layer already exists\n",
      "250519a2f830: Layer already exists\n",
      "6cadbde53f94: Layer already exists\n",
      "0002c93bdb37: Layer already exists\n",
      "latest: digest: sha256:ba67f2d6b6ee7fac7939b42bbca91915c0136eadee93cbc2df6ac5f1d8c98a30 size: 10621\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING! Your password will be stored unencrypted in /home/ec2-user/.docker/config.json.\n",
      "Configure a credential helper to remove this warning. See\n",
      "https://docs.docker.com/engine/reference/commandline/login/#credentials-store\n",
      "\n"
     ]
    }
   ],
   "source": [
    "%%script env repo_name=$repo_name bash\n",
    "\n",
    "#!/usr/bin/env bash\n",
    "\n",
    "# This script shows how to build the Docker image and push it to ECR to be ready for use\n",
    "# by SageMaker.\n",
    "\n",
    "# The argument to this script is the image name. This will be used as the image on the local\n",
    "# machine and combined with the account and region to form the repository name for ECR.\n",
    "# The name of our algorithm\n",
    "algorithm_name=${repo_name}\n",
    "\n",
    "account=$(aws sts get-caller-identity --query Account --output text)\n",
    "\n",
    "# Get the region defined in the current configuration (default to us-west-2 if none defined)\n",
    "region=$(aws configure get region)\n",
    "region=${region:-us-west-2}\n",
    "\n",
    "fullname=\"${account}.dkr.ecr.${region}.amazonaws.com/${algorithm_name}:latest\"\n",
    "\n",
    "# If the repository doesn't exist in ECR, create it.\n",
    "aws ecr describe-repositories --repository-names \"${algorithm_name}\" > /dev/null 2>&1\n",
    "\n",
    "if [ $? -ne 0 ]\n",
    "then\n",
    "    aws ecr create-repository --repository-name \"${algorithm_name}\" > /dev/null\n",
    "fi\n",
    "\n",
    "# Get the login command from ECR and execute it directly\n",
    "aws ecr get-login-password --region ${region}|docker login --username AWS --password-stdin ${fullname}\n",
    "\n",
    "# Build the docker image locally with the image name and then push it to ECR\n",
    "# with the full name.\n",
    "\n",
    "docker build -t ${algorithm_name} .\n",
    "docker tag ${algorithm_name} ${fullname}\n",
    "\n",
    "docker push ${fullname}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5dcb4cf6-4413-42b8-9ec1-9dcb9dcde793",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "## Train ligtning baichuan!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "43bf2350-66cd-4902-b7fe-45933bdcc87a",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting ./baichuan_finetuning/config/main.py\n"
     ]
    }
   ],
   "source": [
    "%%writefile ./baichuan_finetuning/config/main.py\n",
    "\n",
    "\n",
    "# -*- coding: utf-8 -*-\n",
    "# @Author  : ssbuild\n",
    "# @Time    : 2023/5/31 14:43\n",
    "\n",
    "import json\n",
    "import os\n",
    "import torch\n",
    "import torchvision\n",
    "torch.__version__\n",
    "torchvision.__version__\n",
    "\n",
    "train_info_args = {\n",
    "    'devices': 4,\n",
    "}\n",
    "\n",
    "global_args = {\n",
    "          \"load_in_8bit\": False, # lora 如果显卡支持int8 可以开启 ， 需安装依赖 pip install bitsandbytes\n",
    "          \"num_layers\": -1, # 是否使用骨干网络的全部层数 ， -1 表示全层, 否则只用只用N层\n",
    "          \"num_layers_key\":  \"num_layers\"\n",
    "}\n",
    "\n",
    "lora_info_args = {\n",
    "               'with_lora': True,  # 是否启用lora模块\n",
    "               'r': 8,\n",
    "               'target_modules': ['query_key_value'],\n",
    "               'target_dtype': None,\n",
    "               'lora_alpha': 32,\n",
    "               'lora_dropout': 0.1,\n",
    "               'bias': 'none',  # Bias type for Lora. Can be 'none', 'all' or 'lora_only'\"\n",
    "               'modules_to_save' : None, # \"help\": \"List of modules apart from LoRA layers to be set as trainable and saved in the final checkpoint. \"\n",
    "}\n",
    "\n",
    "\n",
    "# 模块配置， 默认启用lora\n",
    "enable_deepspeed = True\n",
    "enable_ptv2 = False\n",
    "enable_lora = True\n",
    "load_in_bit = 0  # 4 load_in_4bit, 8 load_in_8bit  other  0\n",
    "\n",
    "\n",
    "\n",
    "if enable_lora:\n",
    "    from config.sft_config_lora import *\n",
    "elif enable_ptv2:\n",
    "    from config.sft_config_ptv2 import *\n",
    "else:\n",
    "    from config.sft_config import *\n",
    "\n",
    "\n",
    "\n",
    "if enable_lora:\n",
    "    enable_ptv2 = False\n",
    "    global_args['load_in_4bit'] = load_in_bit == 4\n",
    "    global_args['load_in_8bit'] = load_in_bit == 8\n",
    "\n",
    "    if global_args['load_in_4bit']:\n",
    "        global_args['quantization_config'] = None\n",
    "\n",
    "    #检查lora adalora是否开启\n",
    "    if 'lora' not in train_info_args and 'adalora' not in train_info_args:\n",
    "        raise ValueError('please config lora or adalora')\n",
    "    if train_info_args.get('lora',{}).get('with_lora',False) and train_info_args.get('adalora',{}).get('with_lora',False):\n",
    "        raise Exception('lora and adalora can set one at same time !')\n",
    "\n",
    "    train_info_args.pop('prompt', None)\n",
    "elif enable_ptv2:\n",
    "    enable_lora = False\n",
    "    global_args['load_in_4bit'] = False\n",
    "    global_args['load_in_8bit'] = False\n",
    "    train_info_args.pop('lora', None)\n",
    "    train_info_args.pop('adalora', None)\n",
    "else:\n",
    "    # global_args['load_in_4bit'] = False\n",
    "    # global_args['load_in_8bit'] = False\n",
    "    train_info_args.pop('lora',None)\n",
    "    train_info_args.pop('adalora', None)\n",
    "    train_info_args.pop('prompt', None)\n",
    "\n",
    "#预处理\n",
    "if 'rwkv' in train_info_args['tokenizer_name'].lower():\n",
    "    train_info_args['use_fast_tokenizer'] = True\n",
    "\n",
    "\n",
    "def get_deepspeed_config():\n",
    "    '''\n",
    "        lora prompt finetuning 使用 deepspeed_offload.json\n",
    "        普通finetuning 使用deepspeed.json\n",
    "    '''\n",
    "    # 是否开启deepspeed\n",
    "    if not enable_deepspeed:\n",
    "        return None\n",
    "\n",
    "    # 选择 deepspeed 配置文件\n",
    "    is_need_update_config = False\n",
    "    if enable_lora:\n",
    "        is_need_update_config = True\n",
    "        filename = os.path.join(os.path.dirname(__file__), 'deepspeed_offload.json')\n",
    "    else:\n",
    "        filename = os.path.join(os.path.dirname(__file__), 'deepspeed.json')\n",
    "\n",
    "\n",
    "    with open(filename, mode='r', encoding='utf-8') as f:\n",
    "        deepspeed_config = json.loads(f.read())\n",
    "\n",
    "    #lora offload 同步优化器配置\n",
    "    if is_need_update_config:\n",
    "        optimizer = deepspeed_config.get('optimizer',None)\n",
    "        if optimizer:\n",
    "            optimizer['params']['betas'] = train_info_args.get('optimizer_betas', (0.9, 0.999))\n",
    "            optimizer['params']['lr'] = train_info_args.get('learning_rate', 2e-5)\n",
    "            optimizer['params']['eps'] = train_info_args.get('adam_epsilon', 1e-8)\n",
    "    return deepspeed_config\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "55cedcb2",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting ./baichuan_finetuning/train.sh\n"
     ]
    }
   ],
   "source": [
    "%%writefile ./baichuan_finetuning/train.sh\n",
    "#!/bin/bash\n",
    "\n",
    "chmod +x ./s5cmd\n",
    "pip install -U -r requirements.txt\n",
    "\n",
    "#制作数据\n",
    "python data_utils.py\n",
    "    \n",
    "#训练\n",
    "python train.py\n",
    "\n",
    "./s5cmd sync ./best_ckpt s3://$MODEL_S3_BUCKET/models/baichuan_finetuning/output/$(date +%Y-%m-%d-%H-%M-%S)/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "91dc3c13-ba5c-41b9-9682-11dd61cc8382",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'687912291502.dkr.ecr.us-west-2.amazonaws.com/sagemaker-baichuan_finetuning:latest'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## The image uri which is build and pushed above\n",
    "image_uri = \"{}.dkr.ecr.{}.amazonaws.com/{}:latest\".format(account, region, repo_name)\n",
    "image_uri"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8ab36100",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "import time\n",
    "from sagemaker.estimator import Estimator\n",
    "\n",
    "environment = {\n",
    "              'MODEL_S3_BUCKET': bucket # The bucket to store pretrained model and fine-tune model\n",
    "}\n",
    "\n",
    "base_job_name = 'baichuan-finetuning'\n",
    "\n",
    "instance_type = 'ml.g5.12xlarge'\n",
    "\n",
    "estimator = Estimator(role=role,\n",
    "                      entry_point='train.sh',\n",
    "                      source_dir='./baichuan_finetuning/',\n",
    "                      base_job_name=base_job_name,\n",
    "                      instance_count=1,\n",
    "                      instance_type=instance_type,\n",
    "                      image_uri=image_uri,\n",
    "                      environment=environment,\n",
    "                      disable_profiler=True,\n",
    "                      debugger_hook_config=False,\n",
    "                      max_run=24*60*60*2)\n",
    "\n",
    "estimator.fit()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "97efab5a-0fd4-4bb6-bf08-809c1a699050",
   "metadata": {
    "tags": []
   },
   "source": [
    "## train GPT4 baichuan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5187530f-a175-43e6-988a-c89cbf0c4b8d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "##基础模型\n",
    "!aws s3 ls s3://sagemaker-us-west-2-687912291502/llm/models/LLM_baichuan_model/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7037b9c7-d422-4f9c-a302-115862bd650f",
   "metadata": {},
   "outputs": [],
   "source": [
    "## 合并基础模型及LoRA权重\n",
    "#python src/export_model.py --checkpoint_dir path_to_checkpoint --output_dir path_to_save_model --model_name_or_path model_name_or_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "46f0ea3e-bce4-499b-b665-c3c013f9eda5",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "####下载的baichuan model s3路径 ###########\n",
    "model_s3_path=\"s3://sagemaker-us-west-2-687912291502/llm/models/LLM_baichuan_model\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "bb3a0c64-1e8d-402f-8d58-1b8327b970b2",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'687912291502.dkr.ecr.us-west-2.amazonaws.com/sagemaker-baichuan_finetuning:latest'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## The image uri which is build and pushed above\n",
    "image_uri = \"{}.dkr.ecr.{}.amazonaws.com/{}:latest\".format(account, region, repo_name)\n",
    "image_uri"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "d43edef7-e129-4755-b2a0-185e416755f8",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting ./LLaMA-Efficient-Tuning/requirements.txt\n"
     ]
    }
   ],
   "source": [
    "%%writefile ./LLaMA-Efficient-Tuning/requirements.txt\n",
    "transformers>=4.29.1\n",
    "datasets>=2.12.0\n",
    "accelerate>=0.19.0\n",
    "peft>=0.3.0\n",
    "trl>=0.4.4\n",
    "sentencepiece\n",
    "jieba\n",
    "rouge-chinese\n",
    "nltk\n",
    "gradio\n",
    "mdtex2html\n",
    "uvicorn\n",
    "fastapi\n",
    "sse-starlette"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "1c00bd60-67c4-413a-a68a-f25c02ca1e58",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting ./LLaMA-Efficient-Tuning/train.sh\n"
     ]
    }
   ],
   "source": [
    "%%writefile ./LLaMA-Efficient-Tuning/train.sh\n",
    "#!/bin/bash\n",
    "\n",
    "chmod +x ./s5cmd\n",
    "pip install -U -r requirements.txt\n",
    "\n",
    "#./s5cmd sync $MODEL_S3_PATH/* /tmp/baichun-7b/\n",
    "\n",
    "\n",
    "export CUDA_VISIBLE_DEVICES=0\n",
    "python src/train_sft.py \\\n",
    "    --model_name_or_path \"fireballoon/baichuan-vicuna-7b\" \\\n",
    "    --do_train \\\n",
    "    --dataset alpaca_zh \\\n",
    "    --finetuning_type lora \\\n",
    "    --output_dir /tmp/ouput/ \\\n",
    "    --overwrite_cache \\\n",
    "    --per_device_train_batch_size 4 \\\n",
    "    --gradient_accumulation_steps 4 \\\n",
    "    --lr_scheduler_type cosine \\\n",
    "    --logging_steps 10 \\\n",
    "    --save_steps 1000 \\\n",
    "    --learning_rate 5e-5 \\\n",
    "    --num_train_epochs 1.0 \\\n",
    "    --plot_loss \\\n",
    "    --fp16\n",
    "\n",
    "./s5cmd sync /tmp/ouput/ $MODEL_S3_PATH/models/baichuan_finetuning/output/$(date +%Y-%m-%d-%H-%M-%S)/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "2eb4f842-4da7-4284-8ecd-579ee9644b5d",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Writing ./LLaMA-Efficient-Tuning/train_distribute.sh\n"
     ]
    }
   ],
   "source": [
    "%%writefile ./LLaMA-Efficient-Tuning/train_distribute.sh\n",
    "chmod +x ./s5cmd\n",
    "pip install -U -r requirements.txt\n",
    "\n",
    "#./s5cmd sync $MODEL_S3_PATH/* /tmp/baichun-7b/\n",
    "\n",
    "\n",
    "export CUDA_VISIBLE_DEVICES=0,1,2,3\n",
    "\n",
    "accelerate launch --multi_gpu src/train_sft.py \\\n",
    "    --model_name_or_path \"fireballoon/baichuan-vicuna-7b\" \\\n",
    "    --do_train \\\n",
    "    --dataset alpaca_zh \\\n",
    "    --finetuning_type lora \\\n",
    "    --output_dir /tmp/ouput/ \\\n",
    "    --overwrite_cache \\\n",
    "    --per_device_train_batch_size 4 \\\n",
    "    --gradient_accumulation_steps 4 \\\n",
    "    --lr_scheduler_type cosine \\\n",
    "    --logging_steps 10 \\\n",
    "    --save_steps 1000 \\\n",
    "    --learning_rate 5e-5 \\\n",
    "    --num_train_epochs 1.0 \\\n",
    "    --plot_loss \\\n",
    "    --fp16\n",
    "\n",
    "./s5cmd sync /tmp/ouput/ $MODEL_S3_PATH/models/baichuan_finetuning/output/$(date +%Y-%m-%d-%H-%M-%S)/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "05074911-7b33-4d96-b7e6-5cfa655b8672",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using provided s3_resource\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:sagemaker:Creating training-job with name: baichuan-finetuning-2023-07-17-23-27-27-327\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2023-07-17 23:27:49 Starting - Starting the training job......\n",
      "2023-07-17 23:28:43 Starting - Preparing the instances for training......\n",
      "2023-07-17 23:29:26 Downloading - Downloading input data......\n",
      "2023-07-17 23:30:28 Training - Downloading the training image..................\n",
      "2023-07-17 23:33:34 Training - Training image download completed. Training in progress......\u001b[34mbash: cannot set terminal process group (-1): Inappropriate ioctl for device\u001b[0m\n",
      "\u001b[34mbash: no job control in this shell\u001b[0m\n",
      "\u001b[34m2023-07-17 23:34:29,114 sagemaker-training-toolkit INFO     Imported framework sagemaker_pytorch_container.training\u001b[0m\n",
      "\u001b[34m2023-07-17 23:34:29,147 sagemaker-training-toolkit INFO     No Neurons detected (normal if no neurons installed)\u001b[0m\n",
      "\u001b[34m2023-07-17 23:34:29,158 sagemaker_pytorch_container.training INFO     Block until all host DNS lookups succeed.\u001b[0m\n",
      "\u001b[34m2023-07-17 23:34:29,160 sagemaker_pytorch_container.training INFO     Invoking user training script.\u001b[0m\n",
      "\u001b[34m2023-07-17 23:34:31,982 sagemaker-training-toolkit INFO     No Neurons detected (normal if no neurons installed)\u001b[0m\n",
      "\u001b[34m2023-07-17 23:34:32,029 sagemaker-training-toolkit INFO     No Neurons detected (normal if no neurons installed)\u001b[0m\n",
      "\u001b[34m2023-07-17 23:34:32,076 sagemaker-training-toolkit INFO     No Neurons detected (normal if no neurons installed)\u001b[0m\n",
      "\u001b[34m2023-07-17 23:34:32,088 sagemaker-training-toolkit INFO     Invoking user script\u001b[0m\n",
      "\u001b[34mTraining Env:\u001b[0m\n",
      "\u001b[34m{\n",
      "    \"additional_framework_parameters\": {},\n",
      "    \"channel_input_dirs\": {},\n",
      "    \"current_host\": \"algo-1\",\n",
      "    \"current_instance_group\": \"homogeneousCluster\",\n",
      "    \"current_instance_group_hosts\": [\n",
      "        \"algo-1\"\n",
      "    ],\n",
      "    \"current_instance_type\": \"ml.g5.24xlarge\",\n",
      "    \"distribution_hosts\": [],\n",
      "    \"distribution_instance_groups\": [],\n",
      "    \"framework_module\": \"sagemaker_pytorch_container.training:main\",\n",
      "    \"hosts\": [\n",
      "        \"algo-1\"\n",
      "    ],\n",
      "    \"hyperparameters\": {},\n",
      "    \"input_config_dir\": \"/opt/ml/input/config\",\n",
      "    \"input_data_config\": {},\n",
      "    \"input_dir\": \"/opt/ml/input\",\n",
      "    \"instance_groups\": [\n",
      "        \"homogeneousCluster\"\n",
      "    ],\n",
      "    \"instance_groups_dict\": {\n",
      "        \"homogeneousCluster\": {\n",
      "            \"instance_group_name\": \"homogeneousCluster\",\n",
      "            \"instance_type\": \"ml.g5.24xlarge\",\n",
      "            \"hosts\": [\n",
      "                \"algo-1\"\n",
      "            ]\n",
      "        }\n",
      "    },\n",
      "    \"is_hetero\": false,\n",
      "    \"is_master\": true,\n",
      "    \"is_modelparallel_enabled\": null,\n",
      "    \"is_smddpmprun_installed\": true,\n",
      "    \"job_name\": \"baichuan-finetuning-2023-07-17-23-27-27-327\",\n",
      "    \"log_level\": 20,\n",
      "    \"master_hostname\": \"algo-1\",\n",
      "    \"model_dir\": \"/opt/ml/model\",\n",
      "    \"module_dir\": \"s3://sagemaker-us-west-2-687912291502/baichuan-finetuning-2023-07-17-23-27-27-327/source/sourcedir.tar.gz\",\n",
      "    \"module_name\": \"train_distribute.sh\",\n",
      "    \"network_interface_name\": \"eth0\",\n",
      "    \"num_cpus\": 96,\n",
      "    \"num_gpus\": 4,\n",
      "    \"num_neurons\": 0,\n",
      "    \"output_data_dir\": \"/opt/ml/output/data\",\n",
      "    \"output_dir\": \"/opt/ml/output\",\n",
      "    \"output_intermediate_dir\": \"/opt/ml/output/intermediate\",\n",
      "    \"resource_config\": {\n",
      "        \"current_host\": \"algo-1\",\n",
      "        \"current_instance_type\": \"ml.g5.24xlarge\",\n",
      "        \"current_group_name\": \"homogeneousCluster\",\n",
      "        \"hosts\": [\n",
      "            \"algo-1\"\n",
      "        ],\n",
      "        \"instance_groups\": [\n",
      "            {\n",
      "                \"instance_group_name\": \"homogeneousCluster\",\n",
      "                \"instance_type\": \"ml.g5.24xlarge\",\n",
      "                \"hosts\": [\n",
      "                    \"algo-1\"\n",
      "                ]\n",
      "            }\n",
      "        ],\n",
      "        \"network_interface_name\": \"eth0\"\n",
      "    },\n",
      "    \"user_entry_point\": \"train_distribute.sh\"\u001b[0m\n",
      "\u001b[34m}\u001b[0m\n",
      "\u001b[34mEnvironment variables:\u001b[0m\n",
      "\u001b[34mSM_HOSTS=[\"algo-1\"]\u001b[0m\n",
      "\u001b[34mSM_NETWORK_INTERFACE_NAME=eth0\u001b[0m\n",
      "\u001b[34mSM_HPS={}\u001b[0m\n",
      "\u001b[34mSM_USER_ENTRY_POINT=train_distribute.sh\u001b[0m\n",
      "\u001b[34mSM_FRAMEWORK_PARAMS={}\u001b[0m\n",
      "\u001b[34mSM_RESOURCE_CONFIG={\"current_group_name\":\"homogeneousCluster\",\"current_host\":\"algo-1\",\"current_instance_type\":\"ml.g5.24xlarge\",\"hosts\":[\"algo-1\"],\"instance_groups\":[{\"hosts\":[\"algo-1\"],\"instance_group_name\":\"homogeneousCluster\",\"instance_type\":\"ml.g5.24xlarge\"}],\"network_interface_name\":\"eth0\"}\u001b[0m\n",
      "\u001b[34mSM_INPUT_DATA_CONFIG={}\u001b[0m\n",
      "\u001b[34mSM_OUTPUT_DATA_DIR=/opt/ml/output/data\u001b[0m\n",
      "\u001b[34mSM_CHANNELS=[]\u001b[0m\n",
      "\u001b[34mSM_CURRENT_HOST=algo-1\u001b[0m\n",
      "\u001b[34mSM_CURRENT_INSTANCE_TYPE=ml.g5.24xlarge\u001b[0m\n",
      "\u001b[34mSM_CURRENT_INSTANCE_GROUP=homogeneousCluster\u001b[0m\n",
      "\u001b[34mSM_CURRENT_INSTANCE_GROUP_HOSTS=[\"algo-1\"]\u001b[0m\n",
      "\u001b[34mSM_INSTANCE_GROUPS=[\"homogeneousCluster\"]\u001b[0m\n",
      "\u001b[34mSM_INSTANCE_GROUPS_DICT={\"homogeneousCluster\":{\"hosts\":[\"algo-1\"],\"instance_group_name\":\"homogeneousCluster\",\"instance_type\":\"ml.g5.24xlarge\"}}\u001b[0m\n",
      "\u001b[34mSM_DISTRIBUTION_INSTANCE_GROUPS=[]\u001b[0m\n",
      "\u001b[34mSM_IS_HETERO=false\u001b[0m\n",
      "\u001b[34mSM_MODULE_NAME=train_distribute.sh\u001b[0m\n",
      "\u001b[34mSM_LOG_LEVEL=20\u001b[0m\n",
      "\u001b[34mSM_FRAMEWORK_MODULE=sagemaker_pytorch_container.training:main\u001b[0m\n",
      "\u001b[34mSM_INPUT_DIR=/opt/ml/input\u001b[0m\n",
      "\u001b[34mSM_INPUT_CONFIG_DIR=/opt/ml/input/config\u001b[0m\n",
      "\u001b[34mSM_OUTPUT_DIR=/opt/ml/output\u001b[0m\n",
      "\u001b[34mSM_NUM_CPUS=96\u001b[0m\n",
      "\u001b[34mSM_NUM_GPUS=4\u001b[0m\n",
      "\u001b[34mSM_NUM_NEURONS=0\u001b[0m\n",
      "\u001b[34mSM_MODEL_DIR=/opt/ml/model\u001b[0m\n",
      "\u001b[34mSM_MODULE_DIR=s3://sagemaker-us-west-2-687912291502/baichuan-finetuning-2023-07-17-23-27-27-327/source/sourcedir.tar.gz\u001b[0m\n",
      "\u001b[34mSM_TRAINING_ENV={\"additional_framework_parameters\":{},\"channel_input_dirs\":{},\"current_host\":\"algo-1\",\"current_instance_group\":\"homogeneousCluster\",\"current_instance_group_hosts\":[\"algo-1\"],\"current_instance_type\":\"ml.g5.24xlarge\",\"distribution_hosts\":[],\"distribution_instance_groups\":[],\"framework_module\":\"sagemaker_pytorch_container.training:main\",\"hosts\":[\"algo-1\"],\"hyperparameters\":{},\"input_config_dir\":\"/opt/ml/input/config\",\"input_data_config\":{},\"input_dir\":\"/opt/ml/input\",\"instance_groups\":[\"homogeneousCluster\"],\"instance_groups_dict\":{\"homogeneousCluster\":{\"hosts\":[\"algo-1\"],\"instance_group_name\":\"homogeneousCluster\",\"instance_type\":\"ml.g5.24xlarge\"}},\"is_hetero\":false,\"is_master\":true,\"is_modelparallel_enabled\":null,\"is_smddpmprun_installed\":true,\"job_name\":\"baichuan-finetuning-2023-07-17-23-27-27-327\",\"log_level\":20,\"master_hostname\":\"algo-1\",\"model_dir\":\"/opt/ml/model\",\"module_dir\":\"s3://sagemaker-us-west-2-687912291502/baichuan-finetuning-2023-07-17-23-27-27-327/source/sourcedir.tar.gz\",\"module_name\":\"train_distribute.sh\",\"network_interface_name\":\"eth0\",\"num_cpus\":96,\"num_gpus\":4,\"num_neurons\":0,\"output_data_dir\":\"/opt/ml/output/data\",\"output_dir\":\"/opt/ml/output\",\"output_intermediate_dir\":\"/opt/ml/output/intermediate\",\"resource_config\":{\"current_group_name\":\"homogeneousCluster\",\"current_host\":\"algo-1\",\"current_instance_type\":\"ml.g5.24xlarge\",\"hosts\":[\"algo-1\"],\"instance_groups\":[{\"hosts\":[\"algo-1\"],\"instance_group_name\":\"homogeneousCluster\",\"instance_type\":\"ml.g5.24xlarge\"}],\"network_interface_name\":\"eth0\"},\"user_entry_point\":\"train_distribute.sh\"}\u001b[0m\n",
      "\u001b[34mSM_USER_ARGS=[]\u001b[0m\n",
      "\u001b[34mSM_OUTPUT_INTERMEDIATE_DIR=/opt/ml/output/intermediate\u001b[0m\n",
      "\u001b[34mPYTHONPATH=/opt/ml/code:/opt/conda/bin:/opt/conda/lib/python39.zip:/opt/conda/lib/python3.9:/opt/conda/lib/python3.9/lib-dynload:/opt/conda/lib/python3.9/site-packages\u001b[0m\n",
      "\u001b[34mInvoking script with the following command:\u001b[0m\n",
      "\u001b[34m/bin/sh -c \"./train_distribute.sh \"\u001b[0m\n",
      "\u001b[34m[2023-07-17 23:34:33.691: W smdistributed/modelparallel/torch/nn/predefined_hooks.py:78] Found unsupported HuggingFace version 4.26.0 for automated tensor parallelism. HuggingFace modules will not be automatically distributed. You can use smp.tp_register_with_module API to register desired modules for tensor parallelism, or directly instantiate an smp.nn.DistributedModule. Supported HuggingFace transformers versions for automated tensor parallelism: ['4.17.0', '4.20.1', '4.21.0']\u001b[0m\n",
      "\u001b[34m2023-07-17 23:34:33,695 root         INFO     Using NamedTuple = typing._NamedTuple instead.\u001b[0m\n",
      "\u001b[34m2023-07-17 23:34:33,716 sagemaker-training-toolkit INFO     Exceptions not imported for SageMaker TF as Tensorflow is not installed.\u001b[0m\n",
      "\u001b[34mCollecting transformers>=4.29.1\u001b[0m\n",
      "\u001b[34mDownloading transformers-4.30.2-py3-none-any.whl (7.2 MB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 7.2/7.2 MB 21.1 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting datasets>=2.12.0\u001b[0m\n",
      "\u001b[34mDownloading datasets-2.13.1-py3-none-any.whl (486 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 486.2/486.2 kB 34.0 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting accelerate>=0.19.0\u001b[0m\n",
      "\u001b[34mDownloading accelerate-0.21.0-py3-none-any.whl (244 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 244.2/244.2 kB 45.0 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting peft>=0.3.0\u001b[0m\n",
      "\u001b[34mDownloading peft-0.4.0-py3-none-any.whl (72 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 72.9/72.9 kB 25.8 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting trl>=0.4.4\u001b[0m\n",
      "\u001b[34mDownloading trl-0.4.7-py3-none-any.whl (77 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 77.4/77.4 kB 26.5 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: sentencepiece in /opt/conda/lib/python3.9/site-packages (from -r requirements.txt (line 6)) (0.1.97)\u001b[0m\n",
      "\u001b[34mCollecting sentencepiece\u001b[0m\n",
      "\u001b[34mDownloading sentencepiece-0.1.99-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.3/1.3 MB 45.3 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting jieba\u001b[0m\n",
      "\u001b[34mDownloading jieba-0.42.1.tar.gz (19.2 MB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 19.2/19.2 MB 73.8 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mPreparing metadata (setup.py): started\u001b[0m\n",
      "\u001b[34mPreparing metadata (setup.py): finished with status 'done'\u001b[0m\n",
      "\u001b[34mCollecting rouge-chinese\u001b[0m\n",
      "\u001b[34mDownloading rouge_chinese-1.0.3-py3-none-any.whl (21 kB)\u001b[0m\n",
      "\u001b[34mCollecting nltk\u001b[0m\n",
      "\u001b[34mDownloading nltk-3.8.1-py3-none-any.whl (1.5 MB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.5/1.5 MB 90.8 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting gradio\u001b[0m\n",
      "\u001b[34mDownloading gradio-3.37.0-py3-none-any.whl (19.8 MB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 19.8/19.8 MB 62.2 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting mdtex2html\u001b[0m\n",
      "\u001b[34mDownloading mdtex2html-1.2.0-py3-none-any.whl (13 kB)\u001b[0m\n",
      "\u001b[34mCollecting uvicorn\u001b[0m\n",
      "\u001b[34mDownloading uvicorn-0.23.0-py3-none-any.whl (59 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 59.6/59.6 kB 19.8 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting fastapi\u001b[0m\n",
      "\u001b[34mDownloading fastapi-0.100.0-py3-none-any.whl (65 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 65.7/65.7 kB 16.0 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting sse-starlette\u001b[0m\n",
      "\u001b[34mDownloading sse_starlette-1.6.1-py3-none-any.whl (9.6 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: regex!=2019.12.17 in /opt/conda/lib/python3.9/site-packages (from transformers>=4.29.1->-r requirements.txt (line 1)) (2022.10.31)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: requests in /opt/conda/lib/python3.9/site-packages (from transformers>=4.29.1->-r requirements.txt (line 1)) (2.28.2)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.9/site-packages (from transformers>=4.29.1->-r requirements.txt (line 1)) (23.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: tqdm>=4.27 in /opt/conda/lib/python3.9/site-packages (from transformers>=4.29.1->-r requirements.txt (line 1)) (4.64.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /opt/conda/lib/python3.9/site-packages (from transformers>=4.29.1->-r requirements.txt (line 1)) (0.13.2)\u001b[0m\n",
      "\u001b[34mCollecting huggingface-hub<1.0,>=0.14.1\u001b[0m\n",
      "\u001b[34mDownloading huggingface_hub-0.16.4-py3-none-any.whl (268 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 268.8/268.8 kB 62.1 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.9/site-packages (from transformers>=4.29.1->-r requirements.txt (line 1)) (1.23.5)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: filelock in /opt/conda/lib/python3.9/site-packages (from transformers>=4.29.1->-r requirements.txt (line 1)) (3.9.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.9/site-packages (from transformers>=4.29.1->-r requirements.txt (line 1)) (5.4.1)\u001b[0m\n",
      "\u001b[34mCollecting safetensors>=0.3.1\u001b[0m\n",
      "\u001b[34mDownloading safetensors-0.3.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.3/1.3 MB 95.9 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: dill<0.3.7,>=0.3.0 in /opt/conda/lib/python3.9/site-packages (from datasets>=2.12.0->-r requirements.txt (line 2)) (0.3.6)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: xxhash in /opt/conda/lib/python3.9/site-packages (from datasets>=2.12.0->-r requirements.txt (line 2)) (3.2.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pyarrow>=8.0.0 in /opt/conda/lib/python3.9/site-packages (from datasets>=2.12.0->-r requirements.txt (line 2)) (11.0.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: fsspec[http]>=2021.11.1 in /opt/conda/lib/python3.9/site-packages (from datasets>=2.12.0->-r requirements.txt (line 2)) (2023.1.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: multiprocess in /opt/conda/lib/python3.9/site-packages (from datasets>=2.12.0->-r requirements.txt (line 2)) (0.70.14)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pandas in /opt/conda/lib/python3.9/site-packages (from datasets>=2.12.0->-r requirements.txt (line 2)) (1.5.3)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: aiohttp in /opt/conda/lib/python3.9/site-packages (from datasets>=2.12.0->-r requirements.txt (line 2)) (3.8.4)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: psutil in /opt/conda/lib/python3.9/site-packages (from accelerate>=0.19.0->-r requirements.txt (line 3)) (5.9.4)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: torch>=1.10.0 in /opt/conda/lib/python3.9/site-packages (from accelerate>=0.19.0->-r requirements.txt (line 3)) (1.13.1+cu117)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: six in /opt/conda/lib/python3.9/site-packages (from rouge-chinese->-r requirements.txt (line 8)) (1.16.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: joblib in /opt/conda/lib/python3.9/site-packages (from nltk->-r requirements.txt (line 9)) (1.2.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: click in /opt/conda/lib/python3.9/site-packages (from nltk->-r requirements.txt (line 9)) (8.1.2)\u001b[0m\n",
      "\u001b[34mCollecting markdown-it-py[linkify]>=2.0.0\u001b[0m\n",
      "\u001b[34mDownloading markdown_it_py-3.0.0-py3-none-any.whl (87 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 87.5/87.5 kB 27.2 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting aiofiles<24.0,>=22.0\u001b[0m\n",
      "\u001b[34mDownloading aiofiles-23.1.0-py3-none-any.whl (14 kB)\u001b[0m\n",
      "\u001b[34mCollecting mdit-py-plugins<=0.3.3\u001b[0m\n",
      "\u001b[34mDownloading mdit_py_plugins-0.3.3-py3-none-any.whl (50 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 50.5/50.5 kB 16.6 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pillow<11.0,>=8.0 in /opt/conda/lib/python3.9/site-packages (from gradio->-r requirements.txt (line 10)) (9.4.0)\u001b[0m\n",
      "\u001b[34mCollecting pydub\u001b[0m\n",
      "\u001b[34mDownloading pydub-0.25.1-py2.py3-none-any.whl (32 kB)\u001b[0m\n",
      "\u001b[34mCollecting websockets<12.0,>=10.0\u001b[0m\n",
      "\u001b[34mDownloading websockets-11.0.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (129 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 129.7/129.7 kB 27.2 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting orjson~=3.0\u001b[0m\n",
      "\u001b[34mDownloading orjson-3.9.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (138 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 138.6/138.6 kB 37.3 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting python-multipart\u001b[0m\n",
      "\u001b[34mDownloading python_multipart-0.0.6-py3-none-any.whl (45 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 45.7/45.7 kB 13.9 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pydantic!=1.8,!=1.8.1,!=2.0.0,!=2.0.1,<3.0.0,>=1.7.4 in /opt/conda/lib/python3.9/site-packages (from gradio->-r requirements.txt (line 10)) (1.10.4)\u001b[0m\n",
      "\u001b[34mCollecting semantic-version~=2.0\u001b[0m\n",
      "\u001b[34mDownloading semantic_version-2.10.0-py2.py3-none-any.whl (15 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: matplotlib~=3.0 in /opt/conda/lib/python3.9/site-packages (from gradio->-r requirements.txt (line 10)) (3.6.3)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: markupsafe~=2.0 in /opt/conda/lib/python3.9/site-packages (from gradio->-r requirements.txt (line 10)) (2.1.2)\u001b[0m\n",
      "\u001b[34mCollecting httpx\u001b[0m\n",
      "\u001b[34mDownloading httpx-0.24.1-py3-none-any.whl (75 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 75.4/75.4 kB 24.4 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting gradio-client>=0.2.10\u001b[0m\n",
      "\u001b[34mDownloading gradio_client-0.2.10-py3-none-any.whl (288 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 289.0/289.0 kB 56.0 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: typing-extensions~=4.0 in /opt/conda/lib/python3.9/site-packages (from gradio->-r requirements.txt (line 10)) (4.4.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: jinja2<4.0 in /opt/conda/lib/python3.9/site-packages (from gradio->-r requirements.txt (line 10)) (3.1.2)\u001b[0m\n",
      "\u001b[34mCollecting altair<6.0,>=4.2.0\u001b[0m\n",
      "\u001b[34mDownloading altair-5.0.1-py3-none-any.whl (471 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 471.5/471.5 kB 84.1 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting ffmpy\u001b[0m\n",
      "\u001b[34mDownloading ffmpy-0.3.1.tar.gz (5.5 kB)\u001b[0m\n",
      "\u001b[34mPreparing metadata (setup.py): started\u001b[0m\n",
      "\u001b[34mPreparing metadata (setup.py): finished with status 'done'\u001b[0m\n",
      "\u001b[34mCollecting latex2mathml\u001b[0m\n",
      "\u001b[34mDownloading latex2mathml-3.76.0-py3-none-any.whl (73 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 73.4/73.4 kB 25.1 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: markdown in /opt/conda/lib/python3.9/site-packages (from mdtex2html->-r requirements.txt (line 11)) (3.4.1)\u001b[0m\n",
      "\u001b[34mCollecting h11>=0.8\u001b[0m\n",
      "\u001b[34mDownloading h11-0.14.0-py3-none-any.whl (58 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 58.3/58.3 kB 20.9 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting typing-extensions~=4.0\u001b[0m\n",
      "\u001b[34mDownloading typing_extensions-4.7.1-py3-none-any.whl (33 kB)\u001b[0m\n",
      "\u001b[34mCollecting starlette<0.28.0,>=0.27.0\u001b[0m\n",
      "\u001b[34mDownloading starlette-0.27.0-py3-none-any.whl (66 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 67.0/67.0 kB 18.0 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: multidict<7.0,>=4.5 in /opt/conda/lib/python3.9/site-packages (from aiohttp->datasets>=2.12.0->-r requirements.txt (line 2)) (6.0.4)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: aiosignal>=1.1.2 in /opt/conda/lib/python3.9/site-packages (from aiohttp->datasets>=2.12.0->-r requirements.txt (line 2)) (1.3.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: yarl<2.0,>=1.0 in /opt/conda/lib/python3.9/site-packages (from aiohttp->datasets>=2.12.0->-r requirements.txt (line 2)) (1.8.2)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: charset-normalizer<4.0,>=2.0 in /opt/conda/lib/python3.9/site-packages (from aiohttp->datasets>=2.12.0->-r requirements.txt (line 2)) (2.1.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: frozenlist>=1.1.1 in /opt/conda/lib/python3.9/site-packages (from aiohttp->datasets>=2.12.0->-r requirements.txt (line 2)) (1.3.3)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /opt/conda/lib/python3.9/site-packages (from aiohttp->datasets>=2.12.0->-r requirements.txt (line 2)) (4.0.2)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: attrs>=17.3.0 in /opt/conda/lib/python3.9/site-packages (from aiohttp->datasets>=2.12.0->-r requirements.txt (line 2)) (22.2.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: toolz in /opt/conda/lib/python3.9/site-packages (from altair<6.0,>=4.2.0->gradio->-r requirements.txt (line 10)) (0.12.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: jsonschema>=3.0 in /opt/conda/lib/python3.9/site-packages (from altair<6.0,>=4.2.0->gradio->-r requirements.txt (line 10)) (4.17.3)\u001b[0m\n",
      "\u001b[34mCollecting mdurl~=0.1\u001b[0m\n",
      "\u001b[34mDownloading mdurl-0.1.2-py3-none-any.whl (10.0 kB)\u001b[0m\n",
      "\u001b[34mCollecting linkify-it-py<3,>=1\u001b[0m\n",
      "\u001b[34mDownloading linkify_it_py-2.0.2-py3-none-any.whl (19 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: contourpy>=1.0.1 in /opt/conda/lib/python3.9/site-packages (from matplotlib~=3.0->gradio->-r requirements.txt (line 10)) (1.0.7)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: cycler>=0.10 in /opt/conda/lib/python3.9/site-packages (from matplotlib~=3.0->gradio->-r requirements.txt (line 10)) (0.11.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pyparsing>=2.2.1 in /opt/conda/lib/python3.9/site-packages (from matplotlib~=3.0->gradio->-r requirements.txt (line 10)) (3.0.9)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: kiwisolver>=1.0.1 in /opt/conda/lib/python3.9/site-packages (from matplotlib~=3.0->gradio->-r requirements.txt (line 10)) (1.4.4)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: python-dateutil>=2.7 in /opt/conda/lib/python3.9/site-packages (from matplotlib~=3.0->gradio->-r requirements.txt (line 10)) (2.8.2)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: fonttools>=4.22.0 in /opt/conda/lib/python3.9/site-packages (from matplotlib~=3.0->gradio->-r requirements.txt (line 10)) (4.38.0)\u001b[0m\n",
      "\u001b[34mCollecting mdit-py-plugins<=0.3.3\u001b[0m\n",
      "\u001b[34mDownloading mdit_py_plugins-0.3.2-py3-none-any.whl (50 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 50.4/50.4 kB 16.3 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading mdit_py_plugins-0.3.1-py3-none-any.whl (46 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 46.5/46.5 kB 16.9 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading mdit_py_plugins-0.3.0-py3-none-any.whl (43 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 43.7/43.7 kB 15.0 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading mdit_py_plugins-0.2.8-py3-none-any.whl (41 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 41.0/41.0 kB 14.1 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading mdit_py_plugins-0.2.7-py3-none-any.whl (41 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 41.0/41.0 kB 14.7 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading mdit_py_plugins-0.2.6-py3-none-any.whl (39 kB)\u001b[0m\n",
      "\u001b[34mDownloading mdit_py_plugins-0.2.5-py3-none-any.whl (39 kB)\u001b[0m\n",
      "\u001b[34mDownloading mdit_py_plugins-0.2.4-py3-none-any.whl (39 kB)\u001b[0m\n",
      "\u001b[34mDownloading mdit_py_plugins-0.2.3-py3-none-any.whl (39 kB)\u001b[0m\n",
      "\u001b[34mDownloading mdit_py_plugins-0.2.2-py3-none-any.whl (39 kB)\u001b[0m\n",
      "\u001b[34mDownloading mdit_py_plugins-0.2.1-py3-none-any.whl (38 kB)\u001b[0m\n",
      "\u001b[34mDownloading mdit_py_plugins-0.2.0-py3-none-any.whl (38 kB)\u001b[0m\n",
      "\u001b[34mDownloading mdit_py_plugins-0.1.0-py3-none-any.whl (37 kB)\u001b[0m\n",
      "\u001b[34mINFO: pip is looking at multiple versions of matplotlib to determine which version is compatible with other requirements. This could take a while.\u001b[0m\n",
      "\u001b[34mCollecting matplotlib~=3.0\u001b[0m\n",
      "\u001b[34mDownloading matplotlib-3.7.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (11.6 MB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 11.6/11.6 MB 74.8 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mINFO: pip is looking at multiple versions of markupsafe to determine which version is compatible with other requirements. This could take a while.\u001b[0m\n",
      "\u001b[34mCollecting markupsafe~=2.0\u001b[0m\n",
      "\u001b[34mDownloading MarkupSafe-2.1.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (25 kB)\u001b[0m\n",
      "\u001b[34mINFO: pip is looking at multiple versions of markdown-it-py[linkify] to determine which version is compatible with other requirements. This could take a while.\u001b[0m\n",
      "\u001b[34mCollecting markdown-it-py[linkify]>=2.0.0\u001b[0m\n",
      "\u001b[34mDownloading markdown_it_py-2.2.0-py3-none-any.whl (84 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 84.5/84.5 kB 27.2 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pytz>=2020.1 in /opt/conda/lib/python3.9/site-packages (from pandas->datasets>=2.12.0->-r requirements.txt (line 2)) (2022.7.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.9/site-packages (from requests->transformers>=4.29.1->-r requirements.txt (line 1)) (3.4)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.9/site-packages (from requests->transformers>=4.29.1->-r requirements.txt (line 1)) (2022.12.7)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.9/site-packages (from requests->transformers>=4.29.1->-r requirements.txt (line 1)) (1.26.14)\u001b[0m\n",
      "\u001b[34mCollecting anyio<5,>=3.4.0\u001b[0m\n",
      "\u001b[34mDownloading anyio-3.7.1-py3-none-any.whl (80 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 80.9/80.9 kB 22.2 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting httpcore<0.18.0,>=0.15.0\u001b[0m\n",
      "\u001b[34mDownloading httpcore-0.17.3-py3-none-any.whl (74 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 74.5/74.5 kB 26.5 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting sniffio\u001b[0m\n",
      "\u001b[34mDownloading sniffio-1.3.0-py3-none-any.whl (10 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: importlib-metadata>=4.4 in /opt/conda/lib/python3.9/site-packages (from markdown->mdtex2html->-r requirements.txt (line 11)) (4.13.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: exceptiongroup in /opt/conda/lib/python3.9/site-packages (from anyio<5,>=3.4.0->starlette<0.28.0,>=0.27.0->fastapi->-r requirements.txt (line 13)) (1.1.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: zipp>=0.5 in /opt/conda/lib/python3.9/site-packages (from importlib-metadata>=4.4->markdown->mdtex2html->-r requirements.txt (line 11)) (3.13.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pyrsistent!=0.17.0,!=0.17.1,!=0.17.2,>=0.14.0 in /opt/conda/lib/python3.9/site-packages (from jsonschema>=3.0->altair<6.0,>=4.2.0->gradio->-r requirements.txt (line 10)) (0.19.3)\u001b[0m\n",
      "\u001b[34mCollecting uc-micro-py\u001b[0m\n",
      "\u001b[34mDownloading uc_micro_py-1.0.2-py3-none-any.whl (6.2 kB)\u001b[0m\n",
      "\u001b[34mBuilding wheels for collected packages: jieba, ffmpy\u001b[0m\n",
      "\u001b[34mBuilding wheel for jieba (setup.py): started\u001b[0m\n",
      "\u001b[34mBuilding wheel for jieba (setup.py): finished with status 'done'\u001b[0m\n",
      "\u001b[34mCreated wheel for jieba: filename=jieba-0.42.1-py3-none-any.whl size=19314458 sha256=067c5385e030cc7d1d077018afade5a28e09dad1cfa34e2a1e5e1ef48018ecb1\u001b[0m\n",
      "\u001b[34mStored in directory: /root/.cache/pip/wheels/7d/74/cf/08c94db4b784e2c1ef675a600b7b5b281fd25240dcb954ee7e\u001b[0m\n",
      "\u001b[34mBuilding wheel for ffmpy (setup.py): started\u001b[0m\n",
      "\u001b[34mBuilding wheel for ffmpy (setup.py): finished with status 'done'\u001b[0m\n",
      "\u001b[34mCreated wheel for ffmpy: filename=ffmpy-0.3.1-py3-none-any.whl size=5580 sha256=5b3399bb6662ee8001c44b675a24725a9610488480ddb72d2aa5ab26310328f5\u001b[0m\n",
      "\u001b[34mStored in directory: /root/.cache/pip/wheels/1f/f1/8d/367922b023b526b7c2ced5db30932def7b18cf39d7ac6e8572\u001b[0m\n",
      "\u001b[34mSuccessfully built jieba ffmpy\u001b[0m\n",
      "\u001b[34mInstalling collected packages: sentencepiece, safetensors, pydub, jieba, ffmpy, websockets, uc-micro-py, typing-extensions, sniffio, semantic-version, rouge-chinese, python-multipart, orjson, nltk, mdurl, latex2mathml, h11, aiofiles, uvicorn, markdown-it-py, linkify-it-py, huggingface-hub, anyio, transformers, starlette, mdtex2html, mdit-py-plugins, httpcore, altair, accelerate, sse-starlette, peft, httpx, fastapi, datasets, trl, gradio-client, gradio\u001b[0m\n",
      "\u001b[34mAttempting uninstall: sentencepiece\u001b[0m\n",
      "\u001b[34mFound existing installation: sentencepiece 0.1.97\u001b[0m\n",
      "\u001b[34mUninstalling sentencepiece-0.1.97:\u001b[0m\n",
      "\u001b[34mSuccessfully uninstalled sentencepiece-0.1.97\u001b[0m\n",
      "\u001b[34mAttempting uninstall: typing-extensions\u001b[0m\n",
      "\u001b[34mFound existing installation: typing_extensions 4.4.0\u001b[0m\n",
      "\u001b[34mUninstalling typing_extensions-4.4.0:\u001b[0m\n",
      "\u001b[34mSuccessfully uninstalled typing_extensions-4.4.0\u001b[0m\n",
      "\u001b[34mAttempting uninstall: huggingface-hub\u001b[0m\n",
      "\u001b[34mFound existing installation: huggingface-hub 0.12.0\u001b[0m\n",
      "\u001b[34mUninstalling huggingface-hub-0.12.0:\u001b[0m\n",
      "\u001b[34mSuccessfully uninstalled huggingface-hub-0.12.0\u001b[0m\n",
      "\u001b[34mAttempting uninstall: transformers\u001b[0m\n",
      "\u001b[34mFound existing installation: transformers 4.26.0\u001b[0m\n",
      "\u001b[34mUninstalling transformers-4.26.0:\u001b[0m\n",
      "\u001b[34mSuccessfully uninstalled transformers-4.26.0\u001b[0m\n",
      "\u001b[34mAttempting uninstall: accelerate\u001b[0m\n",
      "\u001b[34mFound existing installation: accelerate 0.16.0\u001b[0m\n",
      "\u001b[34mUninstalling accelerate-0.16.0:\u001b[0m\n",
      "\u001b[34mSuccessfully uninstalled accelerate-0.16.0\u001b[0m\n",
      "\u001b[34mAttempting uninstall: datasets\u001b[0m\n",
      "\u001b[34mFound existing installation: datasets 2.9.0\u001b[0m\n",
      "\u001b[34mUninstalling datasets-2.9.0:\u001b[0m\n",
      "\u001b[34mSuccessfully uninstalled datasets-2.9.0\u001b[0m\n",
      "\u001b[34mSuccessfully installed accelerate-0.21.0 aiofiles-23.1.0 altair-5.0.1 anyio-3.7.1 datasets-2.13.1 fastapi-0.100.0 ffmpy-0.3.1 gradio-3.37.0 gradio-client-0.2.10 h11-0.14.0 httpcore-0.17.3 httpx-0.24.1 huggingface-hub-0.16.4 jieba-0.42.1 latex2mathml-3.76.0 linkify-it-py-2.0.2 markdown-it-py-2.2.0 mdit-py-plugins-0.3.3 mdtex2html-1.2.0 mdurl-0.1.2 nltk-3.8.1 orjson-3.9.2 peft-0.4.0 pydub-0.25.1 python-multipart-0.0.6 rouge-chinese-1.0.3 safetensors-0.3.1 semantic-version-2.10.0 sentencepiece-0.1.99 sniffio-1.3.0 sse-starlette-1.6.1 starlette-0.27.0 transformers-4.30.2 trl-0.4.7 typing-extensions-4.7.1 uc-micro-py-1.0.2 uvicorn-0.23.0 websockets-11.0.3\u001b[0m\n",
      "\u001b[34mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\n",
      "\u001b[34m[notice] A new release of pip is available: 23.0 -> 23.2\u001b[0m\n",
      "\u001b[34m[notice] To update, run: pip install --upgrade pip\u001b[0m\n",
      "\u001b[34mThe following values were not passed to `accelerate launch` and had defaults used instead:\u001b[0m\n",
      "\u001b[34m#011`--num_processes` was set to a value of `4`\u001b[0m\n",
      "\u001b[34m#011#011More than one GPU was found, enabling multi-GPU training.\u001b[0m\n",
      "\u001b[34m#011#011If this was unintended please pass in `--num_processes=1`.\u001b[0m\n",
      "\u001b[34m#011`--num_machines` was set to a value of `1`\u001b[0m\n",
      "\u001b[34m#011`--mixed_precision` was set to a value of `'no'`\u001b[0m\n",
      "\u001b[34m#011`--dynamo_backend` was set to a value of `'no'`\u001b[0m\n",
      "\u001b[34mTo avoid this warning pass in values for each of the problematic parameters or run `accelerate config`.\u001b[0m\n",
      "\u001b[34m07/17/2023 23:35:00 - INFO - torch.distributed.distributed_c10d - Added key: store_based_barrier_key:1 to store for rank: 0\u001b[0m\n",
      "\u001b[34m07/17/2023 23:35:00 - INFO - torch.distributed.distributed_c10d - Added key: store_based_barrier_key:1 to store for rank: 1\u001b[0m\n",
      "\u001b[34m07/17/2023 23:35:00 - INFO - torch.distributed.distributed_c10d - Added key: store_based_barrier_key:1 to store for rank: 3\u001b[0m\n",
      "\u001b[34m07/17/2023 23:35:00 - INFO - torch.distributed.distributed_c10d - Added key: store_based_barrier_key:1 to store for rank: 2\u001b[0m\n",
      "\u001b[34m07/17/2023 23:35:00 - INFO - torch.distributed.distributed_c10d - Rank 2: Completed store-based barrier for key:store_based_barrier_key:1 with 4 nodes.\u001b[0m\n",
      "\u001b[34m07/17/2023 23:35:00 - INFO - torch.distributed.distributed_c10d - Rank 1: Completed store-based barrier for key:store_based_barrier_key:1 with 4 nodes.\u001b[0m\n",
      "\u001b[34m07/17/2023 23:35:00 - INFO - torch.distributed.distributed_c10d - Rank 0: Completed store-based barrier for key:store_based_barrier_key:1 with 4 nodes.\u001b[0m\n",
      "\u001b[34m07/17/2023 23:35:00 - INFO - torch.distributed.distributed_c10d - Rank 3: Completed store-based barrier for key:store_based_barrier_key:1 with 4 nodes.\u001b[0m\n",
      "\u001b[34m07/17/2023 23:35:00 - WARNING - utils.common - Please specify `prompt_template` if you are using other pre-trained models.\u001b[0m\n",
      "\u001b[34m07/17/2023 23:35:00 - WARNING - utils.common - `ddp_find_unused_parameters` needs to be set as False in DDP training.\u001b[0m\n",
      "\u001b[34m07/17/2023 23:35:00 - INFO - utils.common - Process rank: 2, device: cuda:2, n_gpu: 1\n",
      "  distributed training: True, 16-bits training: True\u001b[0m\n",
      "\u001b[34m07/17/2023 23:35:00 - INFO - utils.common - Training/evaluation parameters Seq2SeqTrainingArguments(\u001b[0m\n",
      "\u001b[34m_n_gpu=1,\u001b[0m\n",
      "\u001b[34madafactor=False,\u001b[0m\n",
      "\u001b[34madam_beta1=0.9,\u001b[0m\n",
      "\u001b[34madam_beta2=0.999,\u001b[0m\n",
      "\u001b[34madam_epsilon=1e-08,\u001b[0m\n",
      "\u001b[34mauto_find_batch_size=False,\u001b[0m\n",
      "\u001b[34mbf16=False,\u001b[0m\n",
      "\u001b[34mbf16_full_eval=False,\u001b[0m\n",
      "\u001b[34mdata_seed=None,\u001b[0m\n",
      "\u001b[34mdataloader_drop_last=False,\u001b[0m\n",
      "\u001b[34mdataloader_num_workers=0,\u001b[0m\n",
      "\u001b[34mdataloader_pin_memory=True,\u001b[0m\n",
      "\u001b[34mddp_backend=None,\u001b[0m\n",
      "\u001b[34mddp_bucket_cap_mb=None,\u001b[0m\n",
      "\u001b[34mddp_find_unused_parameters=False,\u001b[0m\n",
      "\u001b[34mddp_timeout=1800,\u001b[0m\n",
      "\u001b[34mdebug=[],\u001b[0m\n",
      "\u001b[34mdeepspeed=None,\u001b[0m\n",
      "\u001b[34mdisable_tqdm=False,\u001b[0m\n",
      "\u001b[34mdo_eval=False,\u001b[0m\n",
      "\u001b[34mdo_predict=False,\u001b[0m\n",
      "\u001b[34mdo_train=True,\u001b[0m\n",
      "\u001b[34meval_accumulation_steps=None,\u001b[0m\n",
      "\u001b[34meval_delay=0,\u001b[0m\n",
      "\u001b[34meval_steps=None,\u001b[0m\n",
      "\u001b[34mevaluation_strategy=no,\u001b[0m\n",
      "\u001b[34mfp16=True,\u001b[0m\n",
      "\u001b[34mfp16_backend=auto,\u001b[0m\n",
      "\u001b[34mfp16_full_eval=False,\u001b[0m\n",
      "\u001b[34mfp16_opt_level=O1,\u001b[0m\n",
      "\u001b[34mfsdp=[],\u001b[0m\n",
      "\u001b[34mfsdp_config={'fsdp_min_num_params': 0, 'xla': False, 'xla_fsdp_grad_ckpt': False},\u001b[0m\n",
      "\u001b[34mfsdp_min_num_params=0,\u001b[0m\n",
      "\u001b[34mfsdp_transformer_layer_cls_to_wrap=None,\u001b[0m\n",
      "\u001b[34mfull_determinism=False,\u001b[0m\n",
      "\u001b[34mgeneration_config=None,\u001b[0m\n",
      "\u001b[34mgeneration_max_length=None,\u001b[0m\n",
      "\u001b[34mgeneration_num_beams=None,\u001b[0m\n",
      "\u001b[34mgradient_accumulation_steps=4,\u001b[0m\n",
      "\u001b[34mgradient_checkpointing=False,\u001b[0m\n",
      "\u001b[34mgreater_is_better=None,\u001b[0m\n",
      "\u001b[34mgroup_by_length=False,\u001b[0m\n",
      "\u001b[34mhalf_precision_backend=auto,\u001b[0m\n",
      "\u001b[34mhub_model_id=None,\u001b[0m\n",
      "\u001b[34mhub_private_repo=False,\u001b[0m\n",
      "\u001b[34mhub_strategy=every_save,\u001b[0m\n",
      "\u001b[34mhub_token=<HUB_TOKEN>,\u001b[0m\n",
      "\u001b[34mignore_data_skip=False,\u001b[0m\n",
      "\u001b[34minclude_inputs_for_metrics=False,\u001b[0m\n",
      "\u001b[34mjit_mode_eval=False,\u001b[0m\n",
      "\u001b[34mlabel_names=None,\u001b[0m\n",
      "\u001b[34mlabel_smoothing_factor=0.0,\u001b[0m\n",
      "\u001b[34mlearning_rate=5e-05,\u001b[0m\n",
      "\u001b[34mlength_column_name=length,\u001b[0m\n",
      "\u001b[34mload_best_model_at_end=False,\u001b[0m\n",
      "\u001b[34mlocal_rank=2,\u001b[0m\n",
      "\u001b[34mlog_level=passive,\u001b[0m\n",
      "\u001b[34mlog_level_replica=warning,\u001b[0m\n",
      "\u001b[34mlog_on_each_node=True,\u001b[0m\n",
      "\u001b[34mlogging_dir=/tmp/ouput/runs/Jul17_23-35-00_algo-1,\u001b[0m\n",
      "\u001b[34mlogging_first_step=False,\u001b[0m\n",
      "\u001b[34mlogging_nan_inf_filter=True,\u001b[0m\n",
      "\u001b[34mlogging_steps=10,\u001b[0m\n",
      "\u001b[34mlogging_strategy=steps,\u001b[0m\n",
      "\u001b[34mlr_scheduler_type=cosine,\u001b[0m\n",
      "\u001b[34mmax_grad_norm=1.0,\u001b[0m\n",
      "\u001b[34mmax_steps=-1,\u001b[0m\n",
      "\u001b[34mmetric_for_best_model=None,\u001b[0m\n",
      "\u001b[34mmp_parameters=,\u001b[0m\n",
      "\u001b[34mno_cuda=False,\u001b[0m\n",
      "\u001b[34mnum_train_epochs=1.0,\u001b[0m\n",
      "\u001b[34moptim=adamw_torch,\u001b[0m\n",
      "\u001b[34moptim_args=None,\u001b[0m\n",
      "\u001b[34moutput_dir=/tmp/ouput/,\u001b[0m\n",
      "\u001b[34moverwrite_output_dir=False,\u001b[0m\n",
      "\u001b[34mpast_index=-1,\u001b[0m\n",
      "\u001b[34mper_device_eval_batch_size=8,\u001b[0m\n",
      "\u001b[34mper_device_train_batch_size=4,\u001b[0m\n",
      "\u001b[34mpredict_with_generate=False,\u001b[0m\n",
      "\u001b[34mprediction_loss_only=False,\u001b[0m\n",
      "\u001b[34mpush_to_hub=False,\u001b[0m\n",
      "\u001b[34mpush_to_hub_model_id=None,\u001b[0m\n",
      "\u001b[34mpush_to_hub_organization=None,\u001b[0m\n",
      "\u001b[34mpush_to_hub_token=<PUSH_TO_HUB_TOKEN>,\u001b[0m\n",
      "\u001b[34mray_scope=last,\u001b[0m\n",
      "\u001b[34mremove_unused_columns=True,\u001b[0m\n",
      "\u001b[34mreport_to=[],\u001b[0m\n",
      "\u001b[34mresume_from_checkpoint=None,\u001b[0m\n",
      "\u001b[34mrun_name=/tmp/ouput/,\u001b[0m\n",
      "\u001b[34msave_on_each_node=False,\u001b[0m\n",
      "\u001b[34msave_safetensors=False,\u001b[0m\n",
      "\u001b[34msave_steps=1000,\u001b[0m\n",
      "\u001b[34msave_strategy=steps,\u001b[0m\n",
      "\u001b[34msave_total_limit=None,\u001b[0m\n",
      "\u001b[34mseed=42,\u001b[0m\n",
      "\u001b[34msharded_ddp=[],\u001b[0m\n",
      "\u001b[34mskip_memory_metrics=True,\u001b[0m\n",
      "\u001b[34msortish_sampler=False,\u001b[0m\n",
      "\u001b[34mtf32=None,\u001b[0m\n",
      "\u001b[34mtorch_compile=False,\u001b[0m\n",
      "\u001b[34mtorch_compile_backend=None,\u001b[0m\n",
      "\u001b[34mtorch_compile_mode=None,\u001b[0m\n",
      "\u001b[34mtorchdynamo=None,\u001b[0m\n",
      "\u001b[34mtpu_metrics_debug=False,\u001b[0m\n",
      "\u001b[34mtpu_num_cores=None,\u001b[0m\n",
      "\u001b[34muse_ipex=False,\u001b[0m\n",
      "\u001b[34muse_legacy_prediction_loop=False,\u001b[0m\n",
      "\u001b[34muse_mps_device=False,\u001b[0m\n",
      "\u001b[34mwarmup_ratio=0.0,\u001b[0m\n",
      "\u001b[34mwarmup_steps=0,\u001b[0m\n",
      "\u001b[34mweight_decay=0.0,\u001b[0m\n",
      "\u001b[34mxpu_backend=None,\u001b[0m\n",
      "\u001b[34m)\u001b[0m\n",
      "\u001b[34m07/17/2023 23:35:00 - INFO - utils.common - Loading dataset alpaca_data_zh_51k.json...\u001b[0m\n",
      "\u001b[34m07/17/2023 23:35:00 - WARNING - utils.common - Please specify `prompt_template` if you are using other pre-trained models.\u001b[0m\n",
      "\u001b[34m07/17/2023 23:35:00 - WARNING - utils.common - `ddp_find_unused_parameters` needs to be set as False in DDP training.\u001b[0m\n",
      "\u001b[34m07/17/2023 23:35:00 - INFO - utils.common - Process rank: 1, device: cuda:1, n_gpu: 1\n",
      "  distributed training: True, 16-bits training: True\u001b[0m\n",
      "\u001b[34m07/17/2023 23:35:00 - INFO - utils.common - Training/evaluation parameters Seq2SeqTrainingArguments(\u001b[0m\n",
      "\u001b[34m_n_gpu=1,\u001b[0m\n",
      "\u001b[34madafactor=False,\u001b[0m\n",
      "\u001b[34madam_beta1=0.9,\u001b[0m\n",
      "\u001b[34madam_beta2=0.999,\u001b[0m\n",
      "\u001b[34madam_epsilon=1e-08,\u001b[0m\n",
      "\u001b[34mauto_find_batch_size=False,\u001b[0m\n",
      "\u001b[34mbf16=False,\u001b[0m\n",
      "\u001b[34mbf16_full_eval=False,\u001b[0m\n",
      "\u001b[34mdata_seed=None,\u001b[0m\n",
      "\u001b[34mdataloader_drop_last=False,\u001b[0m\n",
      "\u001b[34mdataloader_num_workers=0,\u001b[0m\n",
      "\u001b[34mdataloader_pin_memory=True,\u001b[0m\n",
      "\u001b[34mddp_backend=None,\u001b[0m\n",
      "\u001b[34mddp_bucket_cap_mb=None,\u001b[0m\n",
      "\u001b[34mddp_find_unused_parameters=False,\u001b[0m\n",
      "\u001b[34mddp_timeout=1800,\u001b[0m\n",
      "\u001b[34mdebug=[],\u001b[0m\n",
      "\u001b[34mdeepspeed=None,\u001b[0m\n",
      "\u001b[34mdisable_tqdm=False,\u001b[0m\n",
      "\u001b[34mdo_eval=False,\u001b[0m\n",
      "\u001b[34mdo_predict=False,\u001b[0m\n",
      "\u001b[34mdo_train=True,\u001b[0m\n",
      "\u001b[34meval_accumulation_steps=None,\u001b[0m\n",
      "\u001b[34meval_delay=0,\u001b[0m\n",
      "\u001b[34meval_steps=None,\u001b[0m\n",
      "\u001b[34mevaluation_strategy=no,\u001b[0m\n",
      "\u001b[34mfp16=True,\u001b[0m\n",
      "\u001b[34mfp16_backend=auto,\u001b[0m\n",
      "\u001b[34mfp16_full_eval=False,\u001b[0m\n",
      "\u001b[34mfp16_opt_level=O1,\u001b[0m\n",
      "\u001b[34mfsdp=[],\u001b[0m\n",
      "\u001b[34mfsdp_config={'fsdp_min_num_params': 0, 'xla': False, 'xla_fsdp_grad_ckpt': False},\u001b[0m\n",
      "\u001b[34mfsdp_min_num_params=0,\u001b[0m\n",
      "\u001b[34mfsdp_transformer_layer_cls_to_wrap=None,\u001b[0m\n",
      "\u001b[34mfull_determinism=False,\u001b[0m\n",
      "\u001b[34mgeneration_config=None,\u001b[0m\n",
      "\u001b[34mgeneration_max_length=None,\u001b[0m\n",
      "\u001b[34mgeneration_num_beams=None,\u001b[0m\n",
      "\u001b[34mgradient_accumulation_steps=4,\u001b[0m\n",
      "\u001b[34mgradient_checkpointing=False,\u001b[0m\n",
      "\u001b[34mgreater_is_better=None,\u001b[0m\n",
      "\u001b[34mgroup_by_length=False,\u001b[0m\n",
      "\u001b[34mhalf_precision_backend=auto,\u001b[0m\n",
      "\u001b[34mhub_model_id=None,\u001b[0m\n",
      "\u001b[34mhub_private_repo=False,\u001b[0m\n",
      "\u001b[34mhub_strategy=every_save,\u001b[0m\n",
      "\u001b[34mhub_token=<HUB_TOKEN>,\u001b[0m\n",
      "\u001b[34mignore_data_skip=False,\u001b[0m\n",
      "\u001b[34minclude_inputs_for_metrics=False,\u001b[0m\n",
      "\u001b[34mjit_mode_eval=False,\u001b[0m\n",
      "\u001b[34mlabel_names=None,\u001b[0m\n",
      "\u001b[34mlabel_smoothing_factor=0.0,\u001b[0m\n",
      "\u001b[34mlearning_rate=5e-05,\u001b[0m\n",
      "\u001b[34mlength_column_name=length,\u001b[0m\n",
      "\u001b[34mload_best_model_at_end=False,\u001b[0m\n",
      "\u001b[34mlocal_rank=1,\u001b[0m\n",
      "\u001b[34mlog_level=passive,\u001b[0m\n",
      "\u001b[34mlog_level_replica=warning,\u001b[0m\n",
      "\u001b[34mlog_on_each_node=True,\u001b[0m\n",
      "\u001b[34mlogging_dir=/tmp/ouput/runs/Jul17_23-35-00_algo-1,\u001b[0m\n",
      "\u001b[34mlogging_first_step=False,\u001b[0m\n",
      "\u001b[34mlogging_nan_inf_filter=True,\u001b[0m\n",
      "\u001b[34mlogging_steps=10,\u001b[0m\n",
      "\u001b[34mlogging_strategy=steps,\u001b[0m\n",
      "\u001b[34mlr_scheduler_type=cosine,\u001b[0m\n",
      "\u001b[34mmax_grad_norm=1.0,\u001b[0m\n",
      "\u001b[34mmax_steps=-1,\u001b[0m\n",
      "\u001b[34mmetric_for_best_model=None,\u001b[0m\n",
      "\u001b[34mmp_parameters=,\u001b[0m\n",
      "\u001b[34mno_cuda=False,\u001b[0m\n",
      "\u001b[34mnum_train_epochs=1.0,\u001b[0m\n",
      "\u001b[34moptim=adamw_torch,\u001b[0m\n",
      "\u001b[34moptim_args=None,\u001b[0m\n",
      "\u001b[34moutput_dir=/tmp/ouput/,\u001b[0m\n",
      "\u001b[34moverwrite_output_dir=False,\u001b[0m\n",
      "\u001b[34mpast_index=-1,\u001b[0m\n",
      "\u001b[34mper_device_eval_batch_size=8,\u001b[0m\n",
      "\u001b[34mper_device_train_batch_size=4,\u001b[0m\n",
      "\u001b[34mpredict_with_generate=False,\u001b[0m\n",
      "\u001b[34mprediction_loss_only=False,\u001b[0m\n",
      "\u001b[34mpush_to_hub=False,\u001b[0m\n",
      "\u001b[34mpush_to_hub_model_id=None,\u001b[0m\n",
      "\u001b[34mpush_to_hub_organization=None,\u001b[0m\n",
      "\u001b[34mpush_to_hub_token=<PUSH_TO_HUB_TOKEN>,\u001b[0m\n",
      "\u001b[34mray_scope=last,\u001b[0m\n",
      "\u001b[34mremove_unused_columns=True,\u001b[0m\n",
      "\u001b[34mreport_to=[],\u001b[0m\n",
      "\u001b[34mresume_from_checkpoint=None,\u001b[0m\n",
      "\u001b[34mrun_name=/tmp/ouput/,\u001b[0m\n",
      "\u001b[34msave_on_each_node=False,\u001b[0m\n",
      "\u001b[34msave_safetensors=False,\u001b[0m\n",
      "\u001b[34msave_steps=1000,\u001b[0m\n",
      "\u001b[34msave_strategy=steps,\u001b[0m\n",
      "\u001b[34msave_total_limit=None,\u001b[0m\n",
      "\u001b[34mseed=42,\u001b[0m\n",
      "\u001b[34msharded_ddp=[],\u001b[0m\n",
      "\u001b[34mskip_memory_metrics=True,\u001b[0m\n",
      "\u001b[34msortish_sampler=False,\u001b[0m\n",
      "\u001b[34mtf32=None,\u001b[0m\n",
      "\u001b[34mtorch_compile=False,\u001b[0m\n",
      "\u001b[34mtorch_compile_backend=None,\u001b[0m\n",
      "\u001b[34mtorch_compile_mode=None,\u001b[0m\n",
      "\u001b[34mtorchdynamo=None,\u001b[0m\n",
      "\u001b[34mtpu_metrics_debug=False,\u001b[0m\n",
      "\u001b[34mtpu_num_cores=None,\u001b[0m\n",
      "\u001b[34muse_ipex=False,\u001b[0m\n",
      "\u001b[34muse_legacy_prediction_loop=False,\u001b[0m\n",
      "\u001b[34muse_mps_device=False,\u001b[0m\n",
      "\u001b[34mwarmup_ratio=0.0,\u001b[0m\n",
      "\u001b[34mwarmup_steps=0,\u001b[0m\n",
      "\u001b[34mweight_decay=0.0,\u001b[0m\n",
      "\u001b[34mxpu_backend=None,\u001b[0m\n",
      "\u001b[34m)\u001b[0m\n",
      "\u001b[34m07/17/2023 23:35:00 - INFO - utils.common - Loading dataset alpaca_data_zh_51k.json...\u001b[0m\n",
      "\u001b[34m07/17/2023 23:35:00 - WARNING - utils.common - Please specify `prompt_template` if you are using other pre-trained models.\u001b[0m\n",
      "\u001b[34m07/17/2023 23:35:00 - WARNING - utils.common - `ddp_find_unused_parameters` needs to be set as False in DDP training.\u001b[0m\n",
      "\u001b[34m07/17/2023 23:35:00 - WARNING - utils.common - Please specify `prompt_template` if you are using other pre-trained models.\u001b[0m\n",
      "\u001b[34m07/17/2023 23:35:00 - WARNING - utils.common - `ddp_find_unused_parameters` needs to be set as False in DDP training.\u001b[0m\n",
      "\u001b[34m07/17/2023 23:35:00 - INFO - utils.common - Process rank: 0, device: cuda:0, n_gpu: 1\n",
      "  distributed training: True, 16-bits training: True\u001b[0m\n",
      "\u001b[34m07/17/2023 23:35:00 - INFO - utils.common - Process rank: 3, device: cuda:3, n_gpu: 1\n",
      "  distributed training: True, 16-bits training: True\u001b[0m\n",
      "\u001b[34m07/17/2023 23:35:00 - INFO - utils.common - Training/evaluation parameters Seq2SeqTrainingArguments(\u001b[0m\n",
      "\u001b[34m_n_gpu=1,\u001b[0m\n",
      "\u001b[34madafactor=False,\u001b[0m\n",
      "\u001b[34madam_beta1=0.9,\u001b[0m\n",
      "\u001b[34madam_beta2=0.999,\u001b[0m\n",
      "\u001b[34madam_epsilon=1e-08,\u001b[0m\n",
      "\u001b[34mauto_find_batch_size=False,\u001b[0m\n",
      "\u001b[34mbf16=False,\u001b[0m\n",
      "\u001b[34mbf16_full_eval=False,\u001b[0m\n",
      "\u001b[34mdata_seed=None,\u001b[0m\n",
      "\u001b[34mdataloader_drop_last=False,\u001b[0m\n",
      "\u001b[34mdataloader_num_workers=0,\u001b[0m\n",
      "\u001b[34mdataloader_pin_memory=True,\u001b[0m\n",
      "\u001b[34mddp_backend=None,\u001b[0m\n",
      "\u001b[34mddp_bucket_cap_mb=None,\u001b[0m\n",
      "\u001b[34mddp_find_unused_parameters=False,\u001b[0m\n",
      "\u001b[34mddp_timeout=1800,\u001b[0m\n",
      "\u001b[34mdebug=[],\u001b[0m\n",
      "\u001b[34mdeepspeed=None,\u001b[0m\n",
      "\u001b[34mdisable_tqdm=False,\u001b[0m\n",
      "\u001b[34mdo_eval=False,\u001b[0m\n",
      "\u001b[34mdo_predict=False,\u001b[0m\n",
      "\u001b[34mdo_train=True,\u001b[0m\n",
      "\u001b[34meval_accumulation_steps=None,\u001b[0m\n",
      "\u001b[34meval_delay=0,\u001b[0m\n",
      "\u001b[34meval_steps=None,\u001b[0m\n",
      "\u001b[34mevaluation_strategy=no,\u001b[0m\n",
      "\u001b[34mfp16=True,\u001b[0m\n",
      "\u001b[34mfp16_backend=auto,\u001b[0m\n",
      "\u001b[34mfp16_full_eval=False,\u001b[0m\n",
      "\u001b[34mfp16_opt_level=O1,\u001b[0m\n",
      "\u001b[34mfsdp=[],\u001b[0m\n",
      "\u001b[34mfsdp_config={'fsdp_min_num_params': 0, 'xla': False, 'xla_fsdp_grad_ckpt': False},\u001b[0m\n",
      "\u001b[34mfsdp_min_num_params=0,\u001b[0m\n",
      "\u001b[34mfsdp_transformer_layer_cls_to_wrap=None,\u001b[0m\n",
      "\u001b[34mfull_determinism=False,\u001b[0m\n",
      "\u001b[34mgeneration_config=None,\u001b[0m\n",
      "\u001b[34mgeneration_max_length=None,\u001b[0m\n",
      "\u001b[34mgeneration_num_beams=None,\u001b[0m\n",
      "\u001b[34mgradient_accumulation_steps=4,\u001b[0m\n",
      "\u001b[34mgradient_checkpointing=False,\u001b[0m\n",
      "\u001b[34mgreater_is_better=None,\u001b[0m\n",
      "\u001b[34mgroup_by_length=False,\u001b[0m\n",
      "\u001b[34mhalf_precision_backend=auto,\u001b[0m\n",
      "\u001b[34mhub_model_id=None,\u001b[0m\n",
      "\u001b[34mhub_private_repo=False,\u001b[0m\n",
      "\u001b[34mhub_strategy=every_save,\u001b[0m\n",
      "\u001b[34mhub_token=<HUB_TOKEN>,\u001b[0m\n",
      "\u001b[34mignore_data_skip=False,\u001b[0m\n",
      "\u001b[34minclude_inputs_for_metrics=False,\u001b[0m\n",
      "\u001b[34mjit_mode_eval=False,\u001b[0m\n",
      "\u001b[34mlabel_names=None,\u001b[0m\n",
      "\u001b[34mlabel_smoothing_factor=0.0,\u001b[0m\n",
      "\u001b[34mlearning_rate=5e-05,\u001b[0m\n",
      "\u001b[34mlength_column_name=length,\u001b[0m\n",
      "\u001b[34mload_best_model_at_end=False,\u001b[0m\n",
      "\u001b[34mlocal_rank=0,\u001b[0m\n",
      "\u001b[34mlog_level=passive,\u001b[0m\n",
      "\u001b[34mlog_level_replica=warning,\u001b[0m\n",
      "\u001b[34mlog_on_each_node=True,\u001b[0m\n",
      "\u001b[34mlogging_dir=/tmp/ouput/runs/Jul17_23-35-00_algo-1,\u001b[0m\n",
      "\u001b[34mlogging_first_step=False,\u001b[0m\n",
      "\u001b[34mlogging_nan_inf_filter=True,\u001b[0m\n",
      "\u001b[34mlogging_steps=10,\u001b[0m\n",
      "\u001b[34mlogging_strategy=steps,\u001b[0m\n",
      "\u001b[34mlr_scheduler_type=cosine,\u001b[0m\n",
      "\u001b[34mmax_grad_norm=1.0,\u001b[0m\n",
      "\u001b[34mmax_steps=-1,\u001b[0m\n",
      "\u001b[34mmetric_for_best_model=None,\u001b[0m\n",
      "\u001b[34mmp_parameters=,\u001b[0m\n",
      "\u001b[34mno_cuda=False,\u001b[0m\n",
      "\u001b[34mnum_train_epochs=1.0,\u001b[0m\n",
      "\u001b[34moptim=adamw_torch,\u001b[0m\n",
      "\u001b[34moptim_args=None,\u001b[0m\n",
      "\u001b[34moutput_dir=/tmp/ouput/,\u001b[0m\n",
      "\u001b[34moverwrite_output_dir=False,\u001b[0m\n",
      "\u001b[34mpast_index=-1,\u001b[0m\n",
      "\u001b[34mper_device_eval_batch_size=8,\u001b[0m\n",
      "\u001b[34mper_device_train_batch_size=4,\u001b[0m\n",
      "\u001b[34mpredict_with_generate=False,\u001b[0m\n",
      "\u001b[34mprediction_loss_only=False,\u001b[0m\n",
      "\u001b[34mpush_to_hub=False,\u001b[0m\n",
      "\u001b[34mpush_to_hub_model_id=None,\u001b[0m\n",
      "\u001b[34mpush_to_hub_organization=None,\u001b[0m\n",
      "\u001b[34mpush_to_hub_token=<PUSH_TO_HUB_TOKEN>,\u001b[0m\n",
      "\u001b[34mray_scope=last,\u001b[0m\n",
      "\u001b[34mremove_unused_columns=True,\u001b[0m\n",
      "\u001b[34mreport_to=[],\u001b[0m\n",
      "\u001b[34mresume_from_checkpoint=None,\u001b[0m\n",
      "\u001b[34mrun_name=/tmp/ouput/,\u001b[0m\n",
      "\u001b[34msave_on_each_node=False,\u001b[0m\n",
      "\u001b[34msave_safetensors=False,\u001b[0m\n",
      "\u001b[34msave_steps=1000,\u001b[0m\n",
      "\u001b[34msave_strategy=steps,\u001b[0m\n",
      "\u001b[34msave_total_limit=None,\u001b[0m\n",
      "\u001b[34mseed=42,\u001b[0m\n",
      "\u001b[34msharded_ddp=[],\u001b[0m\n",
      "\u001b[34mskip_memory_metrics=True,\u001b[0m\n",
      "\u001b[34msortish_sampler=False,\u001b[0m\n",
      "\u001b[34mtf32=None,\u001b[0m\n",
      "\u001b[34mtorch_compile=False,\u001b[0m\n",
      "\u001b[34mtorch_compile_backend=None,\u001b[0m\n",
      "\u001b[34mtorch_compile_mode=None,\u001b[0m\n",
      "\u001b[34mtorchdynamo=None,\u001b[0m\n",
      "\u001b[34mtpu_metrics_debug=False,\u001b[0m\n",
      "\u001b[34mtpu_num_cores=None,\u001b[0m\n",
      "\u001b[34muse_ipex=False,\u001b[0m\n",
      "\u001b[34muse_legacy_prediction_loop=False,\u001b[0m\n",
      "\u001b[34muse_mps_device=False,\u001b[0m\n",
      "\u001b[34mwarmup_ratio=0.0,\u001b[0m\n",
      "\u001b[34mwarmup_steps=0,\u001b[0m\n",
      "\u001b[34mweight_decay=0.0,\u001b[0m\n",
      "\u001b[34mxpu_backend=None,\u001b[0m\n",
      "\u001b[34m)\u001b[0m\n",
      "\u001b[34m07/17/2023 23:35:00 - INFO - utils.common - Training/evaluation parameters Seq2SeqTrainingArguments(\u001b[0m\n",
      "\u001b[34m_n_gpu=1,\u001b[0m\n",
      "\u001b[34madafactor=False,\u001b[0m\n",
      "\u001b[34madam_beta1=0.9,\u001b[0m\n",
      "\u001b[34madam_beta2=0.999,\u001b[0m\n",
      "\u001b[34madam_epsilon=1e-08,\u001b[0m\n",
      "\u001b[34mauto_find_batch_size=False,\u001b[0m\n",
      "\u001b[34mbf16=False,\u001b[0m\n",
      "\u001b[34mbf16_full_eval=False,\u001b[0m\n",
      "\u001b[34mdata_seed=None,\u001b[0m\n",
      "\u001b[34mdataloader_drop_last=False,\u001b[0m\n",
      "\u001b[34mdataloader_num_workers=0,\u001b[0m\n",
      "\u001b[34mdataloader_pin_memory=True,\u001b[0m\n",
      "\u001b[34mddp_backend=None,\u001b[0m\n",
      "\u001b[34mddp_bucket_cap_mb=None,\u001b[0m\n",
      "\u001b[34mddp_find_unused_parameters=False,\u001b[0m\n",
      "\u001b[34mddp_timeout=1800,\u001b[0m\n",
      "\u001b[34mdebug=[],\u001b[0m\n",
      "\u001b[34mdeepspeed=None,\u001b[0m\n",
      "\u001b[34mdisable_tqdm=False,\u001b[0m\n",
      "\u001b[34mdo_eval=False,\u001b[0m\n",
      "\u001b[34mdo_predict=False,\u001b[0m\n",
      "\u001b[34mdo_train=True,\u001b[0m\n",
      "\u001b[34meval_accumulation_steps=None,\u001b[0m\n",
      "\u001b[34meval_delay=0,\u001b[0m\n",
      "\u001b[34meval_steps=None,\u001b[0m\n",
      "\u001b[34mevaluation_strategy=no,\u001b[0m\n",
      "\u001b[34mfp16=True,\u001b[0m\n",
      "\u001b[34mfp16_backend=auto,\u001b[0m\n",
      "\u001b[34mfp16_full_eval=False,\u001b[0m\n",
      "\u001b[34mfp16_opt_level=O1,\u001b[0m\n",
      "\u001b[34mfsdp=[],\u001b[0m\n",
      "\u001b[34mfsdp_config={'fsdp_min_num_params': 0, 'xla': False, 'xla_fsdp_grad_ckpt': False},\u001b[0m\n",
      "\u001b[34mfsdp_min_num_params=0,\u001b[0m\n",
      "\u001b[34mfsdp_transformer_layer_cls_to_wrap=None,\u001b[0m\n",
      "\u001b[34mfull_determinism=False,\u001b[0m\n",
      "\u001b[34mgeneration_config=None,\u001b[0m\n",
      "\u001b[34mgeneration_max_length=None,\u001b[0m\n",
      "\u001b[34mgeneration_num_beams=None,\u001b[0m\n",
      "\u001b[34mgradient_accumulation_steps=4,\u001b[0m\n",
      "\u001b[34mgradient_checkpointing=False,\u001b[0m\n",
      "\u001b[34mgreater_is_better=None,\u001b[0m\n",
      "\u001b[34mgroup_by_length=False,\u001b[0m\n",
      "\u001b[34mhalf_precision_backend=auto,\u001b[0m\n",
      "\u001b[34mhub_model_id=None,\u001b[0m\n",
      "\u001b[34mhub_private_repo=False,\u001b[0m\n",
      "\u001b[34mhub_strategy=every_save,\u001b[0m\n",
      "\u001b[34mhub_token=<HUB_TOKEN>,\u001b[0m\n",
      "\u001b[34mignore_data_skip=False,\u001b[0m\n",
      "\u001b[34minclude_inputs_for_metrics=False,\u001b[0m\n",
      "\u001b[34mjit_mode_eval=False,\u001b[0m\n",
      "\u001b[34mlabel_names=None,\u001b[0m\n",
      "\u001b[34mlabel_smoothing_factor=0.0,\u001b[0m\n",
      "\u001b[34mlearning_rate=5e-05,\u001b[0m\n",
      "\u001b[34mlength_column_name=length,\u001b[0m\n",
      "\u001b[34mload_best_model_at_end=False,\u001b[0m\n",
      "\u001b[34mlocal_rank=3,\u001b[0m\n",
      "\u001b[34mlog_level=passive,\u001b[0m\n",
      "\u001b[34mlog_level_replica=warning,\u001b[0m\n",
      "\u001b[34mlog_on_each_node=True,\u001b[0m\n",
      "\u001b[34mlogging_dir=/tmp/ouput/runs/Jul17_23-35-00_algo-1,\u001b[0m\n",
      "\u001b[34mlogging_first_step=False,\u001b[0m\n",
      "\u001b[34mlogging_nan_inf_filter=True,\u001b[0m\n",
      "\u001b[34mlogging_steps=10,\u001b[0m\n",
      "\u001b[34mlogging_strategy=steps,\u001b[0m\n",
      "\u001b[34mlr_scheduler_type=cosine,\u001b[0m\n",
      "\u001b[34mmax_grad_norm=1.0,\u001b[0m\n",
      "\u001b[34mmax_steps=-1,\u001b[0m\n",
      "\u001b[34mmetric_for_best_model=None,\u001b[0m\n",
      "\u001b[34mmp_parameters=,\u001b[0m\n",
      "\u001b[34mno_cuda=False,\u001b[0m\n",
      "\u001b[34mnum_train_epochs=1.0,\u001b[0m\n",
      "\u001b[34moptim=adamw_torch,\u001b[0m\n",
      "\u001b[34moptim_args=None,\u001b[0m\n",
      "\u001b[34moutput_dir=/tmp/ouput/,\u001b[0m\n",
      "\u001b[34moverwrite_output_dir=False,\u001b[0m\n",
      "\u001b[34mpast_index=-1,\u001b[0m\n",
      "\u001b[34mper_device_eval_batch_size=8,\u001b[0m\n",
      "\u001b[34mper_device_train_batch_size=4,\u001b[0m\n",
      "\u001b[34mpredict_with_generate=False,\u001b[0m\n",
      "\u001b[34mprediction_loss_only=False,\u001b[0m\n",
      "\u001b[34mpush_to_hub=False,\u001b[0m\n",
      "\u001b[34mpush_to_hub_model_id=None,\u001b[0m\n",
      "\u001b[34mpush_to_hub_organization=None,\u001b[0m\n",
      "\u001b[34mpush_to_hub_token=<PUSH_TO_HUB_TOKEN>,\u001b[0m\n",
      "\u001b[34mray_scope=last,\u001b[0m\n",
      "\u001b[34mremove_unused_columns=True,\u001b[0m\n",
      "\u001b[34mreport_to=[],\u001b[0m\n",
      "\u001b[34mresume_from_checkpoint=None,\u001b[0m\n",
      "\u001b[34mrun_name=/tmp/ouput/,\u001b[0m\n",
      "\u001b[34msave_on_each_node=False,\u001b[0m\n",
      "\u001b[34msave_safetensors=False,\u001b[0m\n",
      "\u001b[34msave_steps=1000,\u001b[0m\n",
      "\u001b[34msave_strategy=steps,\u001b[0m\n",
      "\u001b[34msave_total_limit=None,\u001b[0m\n",
      "\u001b[34mseed=42,\u001b[0m\n",
      "\u001b[34msharded_ddp=[],\u001b[0m\n",
      "\u001b[34mskip_memory_metrics=True,\u001b[0m\n",
      "\u001b[34msortish_sampler=False,\u001b[0m\n",
      "\u001b[34mtf32=None,\u001b[0m\n",
      "\u001b[34mtorch_compile=False,\u001b[0m\n",
      "\u001b[34mtorch_compile_backend=None,\u001b[0m\n",
      "\u001b[34mtorch_compile_mode=None,\u001b[0m\n",
      "\u001b[34mtorchdynamo=None,\u001b[0m\n",
      "\u001b[34mtpu_metrics_debug=False,\u001b[0m\n",
      "\u001b[34mtpu_num_cores=None,\u001b[0m\n",
      "\u001b[34muse_ipex=False,\u001b[0m\n",
      "\u001b[34muse_legacy_prediction_loop=False,\u001b[0m\n",
      "\u001b[34muse_mps_device=False,\u001b[0m\n",
      "\u001b[34mwarmup_ratio=0.0,\u001b[0m\n",
      "\u001b[34mwarmup_steps=0,\u001b[0m\n",
      "\u001b[34mweight_decay=0.0,\u001b[0m\n",
      "\u001b[34mxpu_backend=None,\u001b[0m\n",
      "\u001b[34m)\u001b[0m\n",
      "\u001b[34m07/17/2023 23:35:00 - INFO - utils.common - Loading dataset alpaca_data_zh_51k.json...\u001b[0m\n",
      "\u001b[34m07/17/2023 23:35:00 - INFO - utils.common - Loading dataset alpaca_data_zh_51k.json...\u001b[0m\n",
      "\u001b[34mDownloading and preparing dataset json/default to /root/.cache/huggingface/datasets/json/default-a8e55b3e8ea81aac/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96...\u001b[0m\n",
      "\u001b[34mDownloading data files:   0%|          | 0/1 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[34mDownloading data files: 100%|██████████| 1/1 [00:00<00:00, 14614.30it/s]\u001b[0m\n",
      "\u001b[34m07/17/2023 23:35:00 - INFO - datasets.builder - Using custom data configuration default-a8e55b3e8ea81aac\u001b[0m\n",
      "\u001b[34m07/17/2023 23:35:00 - INFO - datasets.info - Loading Dataset Infos from /opt/conda/lib/python3.9/site-packages/datasets/packaged_modules/json\u001b[0m\n",
      "\u001b[34mExtracting data files:   0%|          | 0/1 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[34mExtracting data files: 100%|██████████| 1/1 [00:00<00:00, 1910.84it/s]\u001b[0m\n",
      "\u001b[34mGenerating train split: 0 examples [00:00, ? examples/s]\u001b[0m\n",
      "\u001b[34mGenerating train split: 51461 examples [00:00, 140282.55 examples/s]\u001b[0m\n",
      "\u001b[34mDataset json downloaded and prepared to /root/.cache/huggingface/datasets/json/default-a8e55b3e8ea81aac/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96. Subsequent calls will reuse this data.\u001b[0m\n",
      "\u001b[34m0%|          | 0/1 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[34m100%|██████████| 1/1 [00:00<00:00, 516.79it/s]\u001b[0m\n",
      "\u001b[34m07/17/2023 23:35:01 - WARNING - datasets.builder - Found cached dataset json (/root/.cache/huggingface/datasets/json/default-a8e55b3e8ea81aac/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)\u001b[0m\n",
      "\u001b[34m0%|          | 0/1 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[34m07/17/2023 23:35:01 - WARNING - datasets.builder - Found cached dataset json (/root/.cache/huggingface/datasets/json/default-a8e55b3e8ea81aac/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)\u001b[0m\n",
      "\u001b[34m07/17/2023 23:35:01 - INFO - datasets.info - Loading Dataset info from /root/.cache/huggingface/datasets/json/default-a8e55b3e8ea81aac/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96\u001b[0m\n",
      "\u001b[34m0%|          | 0/1 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[34m100%|██████████| 1/1 [00:00<00:00, 488.56it/s]\u001b[0m\n",
      "\u001b[34m100%|██████████| 1/1 [00:00<00:00, 499.02it/s]\u001b[0m\n",
      "\u001b[34m07/17/2023 23:35:01 - WARNING - datasets.builder - Found cached dataset json (/root/.cache/huggingface/datasets/json/default-a8e55b3e8ea81aac/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)\u001b[0m\n",
      "\u001b[34m0%|          | 0/1 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[34m100%|██████████| 1/1 [00:00<00:00, 506.44it/s]\u001b[0m\n",
      "\u001b[34mDownloading (…)okenizer_config.json:   0%|          | 0.00/872 [00:00<?, ?B/s]\u001b[0m\n",
      "\u001b[34mDownloading (…)okenizer_config.json: 100%|██████████| 872/872 [00:00<00:00, 636kB/s]\u001b[0m\n",
      "\u001b[34mDownloading tokenizer.model:   0%|          | 0.00/2.10M [00:00<?, ?B/s]\u001b[0m\n",
      "\u001b[34mDownloading tokenizer.model: 100%|██████████| 2.10M/2.10M [00:00<00:00, 62.5MB/s]\u001b[0m\n",
      "\u001b[34mDownloading (…)in/added_tokens.json:   0%|          | 0.00/21.0 [00:00<?, ?B/s]\u001b[0m\n",
      "\u001b[34mDownloading (…)in/added_tokens.json: 100%|██████████| 21.0/21.0 [00:00<00:00, 8.81kB/s]\u001b[0m\n",
      "\u001b[34mDownloading (…)cial_tokens_map.json:   0%|          | 0.00/548 [00:00<?, ?B/s]\u001b[0m\n",
      "\u001b[34mDownloading (…)cial_tokens_map.json: 100%|██████████| 548/548 [00:00<00:00, 761kB/s]\u001b[0m\n",
      "\u001b[34m[INFO|tokenization_utils_base.py:1823] 2023-07-17 23:35:02,231 >> loading file tokenizer.model from cache at /root/.cache/huggingface/hub/models--fireballoon--baichuan-vicuna-7b/snapshots/d3b83cf408270757cc8b6e6335d7feb45f868164/tokenizer.model\u001b[0m\n",
      "\u001b[34m[INFO|tokenization_utils_base.py:1823] 2023-07-17 23:35:02,231 >> loading file tokenizer.model from cache at /root/.cache/huggingface/hub/models--fireballoon--baichuan-vicuna-7b/snapshots/d3b83cf408270757cc8b6e6335d7feb45f868164/tokenizer.model\u001b[0m\n",
      "\u001b[34m[INFO|tokenization_utils_base.py:1823] 2023-07-17 23:35:02,231 >> loading file added_tokens.json from cache at /root/.cache/huggingface/hub/models--fireballoon--baichuan-vicuna-7b/snapshots/d3b83cf408270757cc8b6e6335d7feb45f868164/added_tokens.json\u001b[0m\n",
      "\u001b[34m[INFO|tokenization_utils_base.py:1823] 2023-07-17 23:35:02,231 >> loading file special_tokens_map.json from cache at /root/.cache/huggingface/hub/models--fireballoon--baichuan-vicuna-7b/snapshots/d3b83cf408270757cc8b6e6335d7feb45f868164/special_tokens_map.json\u001b[0m\n",
      "\u001b[34m[INFO|tokenization_utils_base.py:1823] 2023-07-17 23:35:02,231 >> loading file tokenizer_config.json from cache at /root/.cache/huggingface/hub/models--fireballoon--baichuan-vicuna-7b/snapshots/d3b83cf408270757cc8b6e6335d7feb45f868164/tokenizer_config.json\u001b[0m\n",
      "\u001b[34m[INFO|tokenization_utils_base.py:1823] 2023-07-17 23:35:02,231 >> loading file added_tokens.json from cache at /root/.cache/huggingface/hub/models--fireballoon--baichuan-vicuna-7b/snapshots/d3b83cf408270757cc8b6e6335d7feb45f868164/added_tokens.json\u001b[0m\n",
      "\u001b[34m[INFO|tokenization_utils_base.py:1823] 2023-07-17 23:35:02,231 >> loading file special_tokens_map.json from cache at /root/.cache/huggingface/hub/models--fireballoon--baichuan-vicuna-7b/snapshots/d3b83cf408270757cc8b6e6335d7feb45f868164/special_tokens_map.json\u001b[0m\n",
      "\u001b[34m[INFO|tokenization_utils_base.py:1823] 2023-07-17 23:35:02,231 >> loading file tokenizer_config.json from cache at /root/.cache/huggingface/hub/models--fireballoon--baichuan-vicuna-7b/snapshots/d3b83cf408270757cc8b6e6335d7feb45f868164/tokenizer_config.json\u001b[0m\n",
      "\u001b[34m[INFO|tokenization_utils.py:426] 2023-07-17 23:35:02,259 >> Adding <pad> to the vocabulary\u001b[0m\n",
      "\u001b[34m[INFO|tokenization_utils.py:426] 2023-07-17 23:35:02,259 >> Adding <pad> to the vocabulary\u001b[0m\n",
      "\u001b[34mDownloading (…)lve/main/config.json:   0%|          | 0.00/581 [00:00<?, ?B/s]\u001b[0m\n",
      "\u001b[34mDownloading (…)lve/main/config.json: 100%|██████████| 581/581 [00:00<00:00, 229kB/s]\u001b[0m\n",
      "\u001b[34m[INFO|configuration_utils.py:669] 2023-07-17 23:35:02,479 >> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--fireballoon--baichuan-vicuna-7b/snapshots/d3b83cf408270757cc8b6e6335d7feb45f868164/config.json\u001b[0m\n",
      "\u001b[34m[INFO|configuration_utils.py:669] 2023-07-17 23:35:02,479 >> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--fireballoon--baichuan-vicuna-7b/snapshots/d3b83cf408270757cc8b6e6335d7feb45f868164/config.json\u001b[0m\n",
      "\u001b[34m[INFO|configuration_utils.py:725] 2023-07-17 23:35:02,480 >> Model config LlamaConfig {\n",
      "  \"_name_or_path\": \"fireballoon/baichuan-vicuna-7b\",\n",
      "  \"architectures\": [\n",
      "    \"LlamaForCausalLM\"\n",
      "  ],\n",
      "  \"bos_token_id\": 1,\n",
      "  \"eos_token_id\": 2,\n",
      "  \"hidden_act\": \"silu\",\n",
      "  \"hidden_size\": 4096,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 11008,\n",
      "  \"max_position_embeddings\": 4096,\n",
      "  \"max_sequence_length\": 4096,\n",
      "  \"model_type\": \"llama\",\n",
      "  \"num_attention_heads\": 32,\n",
      "  \"num_hidden_layers\": 32,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"rms_norm_eps\": 1e-06,\n",
      "  \"tie_word_embeddings\": false,\n",
      "  \"torch_dtype\": \"float16\",\n",
      "  \"transformers_version\": \"4.30.2\",\n",
      "  \"use_cache\": true,\n",
      "  \"vocab_size\": 64000\u001b[0m\n",
      "\u001b[34m}\u001b[0m\n",
      "\u001b[34m[INFO|configuration_utils.py:725] 2023-07-17 23:35:02,480 >> Model config LlamaConfig {\n",
      "  \"_name_or_path\": \"fireballoon/baichuan-vicuna-7b\",\n",
      "  \"architectures\": [\n",
      "    \"LlamaForCausalLM\"\n",
      "  ],\n",
      "  \"bos_token_id\": 1,\n",
      "  \"eos_token_id\": 2,\n",
      "  \"hidden_act\": \"silu\",\n",
      "  \"hidden_size\": 4096,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 11008,\n",
      "  \"max_position_embeddings\": 4096,\n",
      "  \"max_sequence_length\": 4096,\n",
      "  \"model_type\": \"llama\",\n",
      "  \"num_attention_heads\": 32,\n",
      "  \"num_hidden_layers\": 32,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"rms_norm_eps\": 1e-06,\n",
      "  \"tie_word_embeddings\": false,\n",
      "  \"torch_dtype\": \"float16\",\n",
      "  \"transformers_version\": \"4.30.2\",\n",
      "  \"use_cache\": true,\n",
      "  \"vocab_size\": 64000\u001b[0m\n",
      "\u001b[34m}\u001b[0m\n",
      "\u001b[34mDownloading (…)model.bin.index.json:   0%|          | 0.00/26.8k [00:00<?, ?B/s]\u001b[0m\n",
      "\u001b[34mDownloading (…)model.bin.index.json: 100%|██████████| 26.8k/26.8k [00:00<00:00, 10.1MB/s]\u001b[0m\n",
      "\u001b[34mDownloading shards:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[34mDownloading shards:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[34mDownloading shards:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[34m[INFO|modeling_utils.py:2578] 2023-07-17 23:35:03,018 >> loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--fireballoon--baichuan-vicuna-7b/snapshots/d3b83cf408270757cc8b6e6335d7feb45f868164/pytorch_model.bin.index.json\u001b[0m\n",
      "\u001b[34m[INFO|modeling_utils.py:2578] 2023-07-17 23:35:03,018 >> loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--fireballoon--baichuan-vicuna-7b/snapshots/d3b83cf408270757cc8b6e6335d7feb45f868164/pytorch_model.bin.index.json\u001b[0m\n",
      "\u001b[34mDownloading shards:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:   0%|          | 0.00/9.97G [00:00<?, ?B/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:   0%|          | 21.0M/9.97G [00:00<00:58, 169MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:   1%|          | 52.4M/9.97G [00:00<00:41, 238MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:   1%|          | 83.9M/9.97G [00:00<00:41, 239MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:   1%|          | 115M/9.97G [00:00<00:42, 233MB/s] #033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:   2%|▏         | 157M/9.97G [00:00<00:35, 273MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:   2%|▏         | 189M/9.97G [00:00<00:39, 248MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:   2%|▏         | 231M/9.97G [00:00<00:33, 290MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:   3%|▎         | 273M/9.97G [00:00<00:30, 314MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:   3%|▎         | 315M/9.97G [00:01<00:33, 290MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:   4%|▎         | 367M/9.97G [00:01<00:29, 330MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:   4%|▍         | 409M/9.97G [00:01<00:29, 325MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:   5%|▍         | 451M/9.97G [00:01<00:30, 312MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:   5%|▍         | 493M/9.97G [00:01<00:29, 322MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:   5%|▌         | 535M/9.97G [00:01<00:29, 325MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:   6%|▌         | 577M/9.97G [00:01<00:27, 344MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:   6%|▋         | 629M/9.97G [00:02<00:25, 367MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:   7%|▋         | 671M/9.97G [00:02<00:25, 369MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:   7%|▋         | 713M/9.97G [00:02<00:27, 341MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:   8%|▊         | 755M/9.97G [00:02<00:27, 340MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:   8%|▊         | 797M/9.97G [00:02<00:26, 341MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:   8%|▊         | 839M/9.97G [00:02<00:28, 324MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:   9%|▉         | 881M/9.97G [00:02<00:27, 330MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:   9%|▉         | 923M/9.97G [00:02<00:26, 341MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  10%|▉         | 965M/9.97G [00:03<00:24, 360MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  10%|█         | 1.01G/9.97G [00:03<00:24, 359MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  11%|█         | 1.05G/9.97G [00:03<00:26, 341MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  11%|█         | 1.09G/9.97G [00:03<00:26, 339MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  11%|█▏        | 1.13G/9.97G [00:03<00:25, 345MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  12%|█▏        | 1.17G/9.97G [00:03<00:27, 317MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  12%|█▏        | 1.22G/9.97G [00:03<00:30, 290MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  13%|█▎        | 1.25G/9.97G [00:04<00:33, 261MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  13%|█▎        | 1.29G/9.97G [00:04<00:29, 289MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  13%|█▎        | 1.33G/9.97G [00:04<00:27, 312MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  14%|█▍        | 1.37G/9.97G [00:04<00:27, 309MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  14%|█▍        | 1.42G/9.97G [00:04<00:27, 306MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  15%|█▍        | 1.46G/9.97G [00:04<00:25, 327MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  15%|█▌        | 1.50G/9.97G [00:04<00:25, 336MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  15%|█▌        | 1.54G/9.97G [00:04<00:27, 309MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  16%|█▌        | 1.58G/9.97G [00:05<00:28, 297MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  16%|█▋        | 1.63G/9.97G [00:05<00:26, 314MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  17%|█▋        | 1.68G/9.97G [00:05<00:23, 349MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  17%|█▋        | 1.73G/9.97G [00:05<00:22, 362MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  18%|█▊        | 1.77G/9.97G [00:05<00:22, 363MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  18%|█▊        | 1.81G/9.97G [00:05<00:22, 366MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  19%|█▊        | 1.86G/9.97G [00:05<00:22, 361MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  19%|█▉        | 1.90G/9.97G [00:05<00:21, 367MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  20%|█▉        | 1.95G/9.97G [00:05<00:20, 388MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  20%|██        | 2.00G/9.97G [00:06<00:19, 401MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  21%|██        | 2.06G/9.97G [00:06<00:19, 413MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  21%|██        | 2.10G/9.97G [00:06<00:19, 400MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  21%|██▏       | 2.14G/9.97G [00:06<00:19, 400MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  22%|██▏       | 2.18G/9.97G [00:06<00:19, 401MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  22%|██▏       | 2.22G/9.97G [00:06<00:19, 403MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  23%|██▎       | 2.26G/9.97G [00:06<00:20, 381MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  23%|██▎       | 2.31G/9.97G [00:06<00:20, 365MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  24%|██▎       | 2.35G/9.97G [00:07<00:20, 369MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  24%|██▍       | 2.40G/9.97G [00:07<00:20, 369MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  25%|██▍       | 2.44G/9.97G [00:07<00:22, 333MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  25%|██▍       | 2.49G/9.97G [00:07<00:21, 346MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  25%|██▌       | 2.53G/9.97G [00:07<00:21, 341MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  26%|██▌       | 2.57G/9.97G [00:07<00:22, 324MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  26%|██▌       | 2.61G/9.97G [00:07<00:22, 325MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  27%|██▋       | 2.65G/9.97G [00:07<00:22, 320MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  27%|██▋       | 2.69G/9.97G [00:08<00:22, 323MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  27%|██▋       | 2.74G/9.97G [00:08<00:23, 310MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  28%|██▊       | 2.77G/9.97G [00:08<00:25, 283MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  28%|██▊       | 2.80G/9.97G [00:08<00:24, 290MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  29%|██▊       | 2.84G/9.97G [00:08<00:22, 315MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  29%|██▉       | 2.88G/9.97G [00:08<00:22, 316MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  29%|██▉       | 2.93G/9.97G [00:08<00:22, 310MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  30%|██▉       | 2.97G/9.97G [00:08<00:21, 324MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  30%|███       | 3.01G/9.97G [00:09<00:20, 334MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  31%|███       | 3.05G/9.97G [00:09<00:21, 321MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  31%|███       | 3.09G/9.97G [00:09<00:23, 288MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  31%|███▏      | 3.14G/9.97G [00:09<00:22, 310MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  32%|███▏      | 3.18G/9.97G [00:09<00:20, 332MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  32%|███▏      | 3.22G/9.97G [00:09<00:20, 333MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  33%|███▎      | 3.26G/9.97G [00:09<00:19, 341MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  33%|███▎      | 3.30G/9.97G [00:10<00:26, 249MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  33%|███▎      | 3.33G/9.97G [00:10<00:51, 128MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  34%|███▍      | 3.37G/9.97G [00:11<01:08, 96.4MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  34%|███▍      | 3.41G/9.97G [00:11<00:51, 127MB/s] #033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  35%|███▍      | 3.44G/9.97G [00:11<00:53, 123MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  35%|███▍      | 3.47G/9.97G [00:11<00:44, 147MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  35%|███▌      | 3.50G/9.97G [00:11<00:37, 172MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  35%|███▌      | 3.53G/9.97G [00:12<00:33, 195MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  36%|███▌      | 3.58G/9.97G [00:12<00:26, 238MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  36%|███▌      | 3.61G/9.97G [00:12<00:28, 227MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  37%|███▋      | 3.64G/9.97G [00:12<00:51, 122MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  37%|███▋      | 3.66G/9.97G [00:13<01:07, 93.1MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  37%|███▋      | 3.68G/9.97G [00:13<01:18, 79.6MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  37%|███▋      | 3.70G/9.97G [00:14<01:24, 74.4MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  37%|███▋      | 3.72G/9.97G [00:14<01:30, 69.2MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  37%|███▋      | 3.73G/9.97G [00:14<01:29, 69.7MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  38%|███▊      | 3.74G/9.97G [00:14<01:41, 61.6MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  38%|███▊      | 3.75G/9.97G [00:14<01:41, 61.0MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  38%|███▊      | 3.76G/9.97G [00:15<01:36, 64.0MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  38%|███▊      | 3.77G/9.97G [00:15<01:33, 66.0MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  38%|███▊      | 3.79G/9.97G [00:15<01:28, 69.5MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  38%|███▊      | 3.80G/9.97G [00:15<01:33, 66.1MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  38%|███▊      | 3.81G/9.97G [00:15<01:32, 66.7MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  38%|███▊      | 3.82G/9.97G [00:15<01:36, 63.4MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  38%|███▊      | 3.83G/9.97G [00:16<01:40, 61.1MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  39%|███▊      | 3.84G/9.97G [00:16<01:38, 62.0MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  39%|███▊      | 3.85G/9.97G [00:16<01:31, 66.7MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  39%|███▊      | 3.86G/9.97G [00:16<01:37, 62.7MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  39%|███▉      | 3.87G/9.97G [00:16<01:37, 62.4MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  39%|███▉      | 3.90G/9.97G [00:16<01:10, 86.6MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  39%|███▉      | 3.91G/9.97G [00:17<01:20, 74.8MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  39%|███▉      | 3.92G/9.97G [00:17<01:24, 71.6MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  39%|███▉      | 3.93G/9.97G [00:17<01:27, 68.7MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  40%|███▉      | 3.94G/9.97G [00:17<01:24, 71.4MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  40%|███▉      | 3.95G/9.97G [00:17<01:25, 70.6MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  40%|███▉      | 3.96G/9.97G [00:17<01:29, 67.2MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  40%|████      | 4.01G/9.97G [00:18<00:44, 135MB/s] #033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  41%|████      | 4.05G/9.97G [00:18<00:30, 196MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  41%|████      | 4.09G/9.97G [00:18<00:26, 223MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  41%|████▏     | 4.12G/9.97G [00:18<00:24, 239MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  42%|████▏     | 4.15G/9.97G [00:18<00:23, 248MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  42%|████▏     | 4.19G/9.97G [00:18<00:19, 290MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  42%|████▏     | 4.24G/9.97G [00:18<00:17, 324MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  43%|████▎     | 4.28G/9.97G [00:18<00:16, 341MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  43%|████▎     | 4.32G/9.97G [00:18<00:15, 361MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  44%|████▍     | 4.36G/9.97G [00:19<00:14, 375MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  44%|████▍     | 4.40G/9.97G [00:19<00:14, 385MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  45%|████▍     | 4.45G/9.97G [00:19<00:14, 389MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  45%|████▌     | 4.49G/9.97G [00:19<00:15, 348MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  45%|████▌     | 4.53G/9.97G [00:19<00:19, 285MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  46%|████▌     | 4.56G/9.97G [00:19<00:18, 290MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  46%|████▌     | 4.59G/9.97G [00:19<00:18, 286MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  46%|████▋     | 4.62G/9.97G [00:20<00:19, 269MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  47%|████▋     | 4.67G/9.97G [00:20<00:18, 285MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  47%|████▋     | 4.70G/9.97G [00:20<00:18, 284MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  47%|████▋     | 4.73G/9.97G [00:20<00:18, 289MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  48%|████▊     | 4.77G/9.97G [00:20<00:17, 295MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  48%|████▊     | 4.80G/9.97G [00:20<00:17, 293MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  49%|████▊     | 4.84G/9.97G [00:20<00:16, 303MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  49%|████▉     | 4.88G/9.97G [00:20<00:17, 297MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  49%|████▉     | 4.92G/9.97G [00:20<00:16, 312MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  50%|████▉     | 4.96G/9.97G [00:21<00:15, 331MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  50%|█████     | 5.00G/9.97G [00:21<00:16, 307MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  51%|█████     | 5.05G/9.97G [00:21<00:14, 339MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  51%|█████     | 5.10G/9.97G [00:21<00:14, 346MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  52%|█████▏    | 5.14G/9.97G [00:21<00:14, 341MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  52%|█████▏    | 5.18G/9.97G [00:21<00:13, 345MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  52%|█████▏    | 5.22G/9.97G [00:21<00:13, 358MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  53%|█████▎    | 5.26G/9.97G [00:21<00:12, 370MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  53%|█████▎    | 5.31G/9.97G [00:22<00:12, 379MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  54%|█████▎    | 5.35G/9.97G [00:22<00:13, 351MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  54%|█████▍    | 5.39G/9.97G [00:22<00:13, 336MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  54%|█████▍    | 5.43G/9.97G [00:22<00:15, 295MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  55%|█████▍    | 5.46G/9.97G [00:22<00:15, 290MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  55%|█████▌    | 5.51G/9.97G [00:22<00:14, 303MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  56%|█████▌    | 5.55G/9.97G [00:22<00:14, 313MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  56%|█████▌    | 5.59G/9.97G [00:22<00:13, 332MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  56%|█████▋    | 5.63G/9.97G [00:23<00:12, 345MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  57%|█████▋    | 5.67G/9.97G [00:23<00:11, 364MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  57%|█████▋    | 5.71G/9.97G [00:23<00:12, 336MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  58%|█████▊    | 5.76G/9.97G [00:23<00:12, 330MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  58%|█████▊    | 5.80G/9.97G [00:23<00:12, 332MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  59%|█████▊    | 5.84G/9.97G [00:23<00:11, 353MB/s]\u001b[0m\n",
      "\u001b[34m#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  59%|█████▉    | 5.88G/9.97G [00:23<00:11, 363MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  59%|█████▉    | 5.92G/9.97G [00:23<00:11, 360MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  60%|█████▉    | 5.97G/9.97G [00:24<00:10, 367MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  60%|██████    | 6.01G/9.97G [00:24<00:10, 371MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  61%|██████    | 6.05G/9.97G [00:24<00:11, 354MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  61%|██████    | 6.09G/9.97G [00:24<00:12, 321MB/s]\u001b[0m\n",
      "\u001b[34m#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  62%|██████▏   | 6.13G/9.97G [00:24<00:11, 332MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  62%|██████▏   | 6.18G/9.97G [00:24<00:12, 297MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  62%|██████▏   | 6.22G/9.97G [00:24<00:11, 313MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  63%|██████▎   | 6.26G/9.97G [00:24<00:11, 316MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  63%|██████▎   | 6.30G/9.97G [00:25<00:11, 316MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  64%|██████▎   | 6.34G/9.97G [00:25<00:10, 339MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  64%|██████▍   | 6.39G/9.97G [00:25<00:09, 360MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  65%|██████▍   | 6.44G/9.97G [00:25<00:09, 381MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  65%|██████▌   | 6.48G/9.97G [00:25<00:08, 389MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  66%|██████▌   | 6.53G/9.97G [00:25<00:08, 400MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  66%|██████▌   | 6.57G/9.97G [00:25<00:09, 377MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  66%|██████▋   | 6.62G/9.97G [00:25<00:09, 339MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  67%|██████▋   | 6.66G/9.97G [00:26<00:09, 343MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  67%|██████▋   | 6.70G/9.97G [00:26<00:10, 325MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  68%|██████▊   | 6.75G/9.97G [00:26<00:09, 347MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  68%|██████▊   | 6.79G/9.97G [00:26<00:09, 345MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  69%|██████▊   | 6.84G/9.97G [00:26<00:08, 354MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  69%|██████▉   | 6.88G/9.97G [00:26<00:08, 362MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  69%|██████▉   | 6.92G/9.97G [00:26<00:08, 343MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  70%|██████▉   | 6.96G/9.97G [00:26<00:08, 360MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  70%|███████   | 7.00G/9.97G [00:27<00:08, 341MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  71%|███████   | 7.05G/9.97G [00:27<00:08, 354MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  71%|███████   | 7.09G/9.97G [00:27<00:08, 341MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  72%|███████▏  | 7.13G/9.97G [00:27<00:08, 339MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  72%|███████▏  | 7.17G/9.97G [00:27<00:08, 320MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  72%|███████▏  | 7.21G/9.97G [00:27<00:08, 332MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  73%|███████▎  | 7.26G/9.97G [00:27<00:07, 352MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  73%|███████▎  | 7.30G/9.97G [00:27<00:07, 346MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  74%|███████▎  | 7.34G/9.97G [00:28<00:08, 322MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  74%|███████▍  | 7.38G/9.97G [00:28<00:07, 340MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  74%|███████▍  | 7.42G/9.97G [00:28<00:07, 359MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  75%|███████▍  | 7.47G/9.97G [00:28<00:06, 367MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  75%|███████▌  | 7.51G/9.97G [00:28<00:06, 354MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  76%|███████▌  | 7.55G/9.97G [00:28<00:06, 352MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  76%|███████▋  | 7.60G/9.97G [00:28<00:06, 365MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  77%|███████▋  | 7.64G/9.97G [00:29<00:09, 244MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  77%|███████▋  | 7.68G/9.97G [00:29<00:14, 158MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  77%|███████▋  | 7.71G/9.97G [00:29<00:14, 159MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  78%|███████▊  | 7.75G/9.97G [00:29<00:11, 194MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  78%|███████▊  | 7.79G/9.97G [00:29<00:09, 232MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  78%|███████▊  | 7.82G/9.97G [00:30<00:08, 241MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  79%|███████▉  | 7.85G/9.97G [00:30<00:08, 239MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  79%|███████▉  | 7.89G/9.97G [00:30<00:08, 232MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  79%|███████▉  | 7.92G/9.97G [00:30<00:09, 210MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  80%|███████▉  | 7.95G/9.97G [00:30<00:14, 142MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  80%|████████  | 7.98G/9.97G [00:31<00:11, 168MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  80%|████████  | 8.01G/9.97G [00:31<00:10, 191MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  81%|████████  | 8.05G/9.97G [00:31<00:08, 226MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  81%|████████  | 8.10G/9.97G [00:31<00:07, 257MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  82%|████████▏ | 8.13G/9.97G [00:31<00:06, 268MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  82%|████████▏ | 8.16G/9.97G [00:31<00:06, 279MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  82%|████████▏ | 8.19G/9.97G [00:31<00:06, 262MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  82%|████████▏ | 8.22G/9.97G [00:31<00:06, 273MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  83%|████████▎ | 8.25G/9.97G [00:32<00:07, 223MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  83%|████████▎ | 8.28G/9.97G [00:32<00:08, 201MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  83%|████████▎ | 8.32G/9.97G [00:32<00:07, 221MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  84%|████████▍ | 8.36G/9.97G [00:32<00:06, 247MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  84%|████████▍ | 8.41G/9.97G [00:32<00:05, 295MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  85%|████████▍ | 8.45G/9.97G [00:32<00:05, 294MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  85%|████████▌ | 8.48G/9.97G [00:32<00:05, 269MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  85%|████████▌ | 8.51G/9.97G [00:33<00:05, 252MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  86%|████████▌ | 8.56G/9.97G [00:33<00:05, 279MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  86%|████████▋ | 8.60G/9.97G [00:33<00:04, 298MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  87%|████████▋ | 8.64G/9.97G [00:33<00:04, 314MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  87%|████████▋ | 8.68G/9.97G [00:33<00:04, 298MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  88%|████████▊ | 8.72G/9.97G [00:33<00:04, 309MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  88%|████████▊ | 8.77G/9.97G [00:33<00:03, 314MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  88%|████████▊ | 8.81G/9.97G [00:33<00:03, 292MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  89%|████████▊ | 8.84G/9.97G [00:34<00:04, 272MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  89%|████████▉ | 8.87G/9.97G [00:34<00:06, 175MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  89%|████████▉ | 8.90G/9.97G [00:34<00:08, 128MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  90%|████████▉ | 8.92G/9.97G [00:35<00:07, 131MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  90%|████████▉ | 8.94G/9.97G [00:35<00:08, 126MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  90%|████████▉ | 8.97G/9.97G [00:35<00:08, 115MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  90%|█████████ | 8.99G/9.97G [00:35<00:08, 118MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  91%|█████████ | 9.03G/9.97G [00:35<00:05, 168MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  91%|█████████ | 9.06G/9.97G [00:35<00:04, 188MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  91%|█████████▏| 9.10G/9.97G [00:35<00:03, 227MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  92%|█████████▏| 9.15G/9.97G [00:36<00:02, 275MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  92%|█████████▏| 9.20G/9.97G [00:36<00:02, 290MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  93%|█████████▎| 9.23G/9.97G [00:36<00:02, 253MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  93%|█████████▎| 9.27G/9.97G [00:36<00:02, 241MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  94%|█████████▎| 9.32G/9.97G [00:36<00:02, 284MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  94%|█████████▍| 9.36G/9.97G [00:36<00:01, 303MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  94%|█████████▍| 9.41G/9.97G [00:36<00:01, 318MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  95%|█████████▍| 9.46G/9.97G [00:37<00:01, 350MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  95%|█████████▌| 9.50G/9.97G [00:37<00:01, 361MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  96%|█████████▌| 9.54G/9.97G [00:37<00:01, 292MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  96%|█████████▌| 9.58G/9.97G [00:37<00:01, 304MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  97%|█████████▋| 9.63G/9.97G [00:37<00:01, 303MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  97%|█████████▋| 9.67G/9.97G [00:37<00:00, 325MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  97%|█████████▋| 9.71G/9.97G [00:37<00:00, 340MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  98%|█████████▊| 9.75G/9.97G [00:38<00:00, 297MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  98%|█████████▊| 9.79G/9.97G [00:38<00:00, 317MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  99%|█████████▊| 9.84G/9.97G [00:38<00:00, 330MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin:  99%|█████████▉| 9.88G/9.97G [00:38<00:00, 317MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin: 100%|█████████▉| 9.92G/9.97G [00:38<00:00, 172MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin: 100%|█████████▉| 9.95G/9.97G [00:39<00:00, 140MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00001-of-00002.bin: 100%|██████████| 9.97G/9.97G [00:39<00:00, 254MB/s]\u001b[0m\n",
      "\u001b[34mDownloading shards:  50%|█████     | 1/2 [00:39<00:39, 39.42s/it]\u001b[0m\n",
      "\u001b[34mDownloading shards:  50%|█████     | 1/2 [00:39<00:39, 39.42s/it]\u001b[0m\n",
      "\u001b[34mDownloading shards:  50%|█████     | 1/2 [00:39<00:39, 39.41s/it]\u001b[0m\n",
      "\u001b[34mDownloading shards:  50%|█████     | 1/2 [00:39<00:39, 39.46s/it]\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:   0%|          | 0.00/4.03G [00:00<?, ?B/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:   1%|          | 21.0M/4.03G [00:00<00:29, 136MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:   1%|▏         | 52.4M/4.03G [00:00<00:19, 205MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:   2%|▏         | 94.4M/4.03G [00:00<00:13, 285MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:   3%|▎         | 136M/4.03G [00:00<00:12, 317MB/s] #033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:   4%|▍         | 178M/4.03G [00:00<00:11, 334MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:   5%|▌         | 220M/4.03G [00:00<00:12, 304MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:   6%|▌         | 252M/4.03G [00:00<00:13, 273MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:   7%|▋         | 283M/4.03G [00:01<00:13, 269MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:   8%|▊         | 315M/4.03G [00:01<00:14, 264MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:   9%|▊         | 346M/4.03G [00:01<00:13, 271MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:   9%|▉         | 377M/4.03G [00:01<00:13, 270MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  10%|█         | 409M/4.03G [00:01<00:13, 266MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  11%|█         | 440M/4.03G [00:01<00:13, 266MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  12%|█▏        | 472M/4.03G [00:01<00:19, 185MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  13%|█▎        | 514M/4.03G [00:02<00:15, 230MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  14%|█▍        | 556M/4.03G [00:02<00:12, 271MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  15%|█▍        | 598M/4.03G [00:02<00:11, 288MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  16%|█▌        | 640M/4.03G [00:02<00:12, 265MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  17%|█▋        | 671M/4.03G [00:02<00:12, 259MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  17%|█▋        | 703M/4.03G [00:02<00:13, 242MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  18%|█▊        | 734M/4.03G [00:02<00:14, 235MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  19%|█▉        | 765M/4.03G [00:02<00:13, 247MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  20%|█▉        | 797M/4.03G [00:03<00:13, 233MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  21%|██        | 828M/4.03G [00:03<00:14, 224MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  21%|██▏       | 860M/4.03G [00:03<00:13, 234MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  22%|██▏       | 902M/4.03G [00:03<00:12, 245MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  23%|██▎       | 933M/4.03G [00:03<00:12, 254MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  24%|██▍       | 975M/4.03G [00:03<00:10, 286MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  25%|██▍       | 1.01G/4.03G [00:03<00:10, 278MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  26%|██▌       | 1.04G/4.03G [00:04<00:11, 270MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  27%|██▋       | 1.07G/4.03G [00:04<00:12, 240MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  27%|██▋       | 1.10G/4.03G [00:04<00:11, 248MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  28%|██▊       | 1.14G/4.03G [00:04<00:10, 272MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  29%|██▉       | 1.17G/4.03G [00:04<00:10, 275MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  30%|██▉       | 1.21G/4.03G [00:04<00:10, 277MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  31%|███       | 1.24G/4.03G [00:04<00:11, 251MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  31%|███▏      | 1.27G/4.03G [00:05<00:13, 199MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  32%|███▏      | 1.30G/4.03G [00:05<00:14, 195MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  33%|███▎      | 1.33G/4.03G [00:05<00:13, 196MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  34%|███▍      | 1.36G/4.03G [00:05<00:12, 216MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  35%|███▍      | 1.39G/4.03G [00:05<00:11, 224MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  35%|███▌      | 1.43G/4.03G [00:05<00:11, 223MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  36%|███▌      | 1.46G/4.03G [00:05<00:11, 217MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  37%|███▋      | 1.49G/4.03G [00:06<00:11, 222MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  38%|███▊      | 1.52G/4.03G [00:06<00:16, 153MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  38%|███▊      | 1.54G/4.03G [00:06<00:17, 143MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  39%|███▊      | 1.56G/4.03G [00:06<00:17, 145MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  39%|███▉      | 1.58G/4.03G [00:06<00:20, 117MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  40%|████      | 1.61G/4.03G [00:07<00:16, 150MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  41%|████      | 1.66G/4.03G [00:07<00:12, 184MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  42%|████▏     | 1.70G/4.03G [00:07<00:10, 215MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  43%|████▎     | 1.73G/4.03G [00:07<00:10, 220MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  44%|████▎     | 1.76G/4.03G [00:07<00:11, 205MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  44%|████▍     | 1.79G/4.03G [00:07<00:10, 218MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  45%|████▌     | 1.84G/4.03G [00:07<00:08, 259MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  46%|████▋     | 1.87G/4.03G [00:08<00:08, 260MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  47%|████▋     | 1.90G/4.03G [00:08<00:08, 255MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  48%|████▊     | 1.94G/4.03G [00:08<00:07, 288MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  49%|████▉     | 1.98G/4.03G [00:08<00:06, 317MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  50%|█████     | 2.02G/4.03G [00:08<00:06, 287MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  51%|█████     | 2.07G/4.03G [00:08<00:06, 310MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  52%|█████▏    | 2.11G/4.03G [00:08<00:05, 333MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  53%|█████▎    | 2.15G/4.03G [00:08<00:05, 324MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  54%|█████▍    | 2.19G/4.03G [00:09<00:07, 253MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  55%|█████▌    | 2.22G/4.03G [00:09<00:07, 256MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  56%|█████▌    | 2.25G/4.03G [00:09<00:07, 251MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  57%|█████▋    | 2.29G/4.03G [00:09<00:07, 233MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  57%|█████▋    | 2.32G/4.03G [00:09<00:07, 229MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  58%|█████▊    | 2.35G/4.03G [00:09<00:07, 234MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  59%|█████▉    | 2.39G/4.03G [00:09<00:05, 277MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  60%|██████    | 2.42G/4.03G [00:10<00:07, 230MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  61%|██████    | 2.45G/4.03G [00:10<00:06, 248MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  62%|██████▏   | 2.50G/4.03G [00:10<00:06, 247MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  63%|██████▎   | 2.53G/4.03G [00:10<00:07, 209MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  63%|██████▎   | 2.56G/4.03G [00:10<00:07, 209MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  64%|██████▍   | 2.59G/4.03G [00:10<00:06, 215MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  65%|██████▍   | 2.62G/4.03G [00:11<00:06, 218MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  66%|██████▌   | 2.65G/4.03G [00:11<00:07, 186MB/s]\u001b[0m\n",
      "\u001b[34m#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  67%|██████▋   | 2.68G/4.03G [00:11<00:07, 181MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  67%|██████▋   | 2.71G/4.03G [00:11<00:11, 120MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  68%|██████▊   | 2.73G/4.03G [00:12<00:11, 110MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  68%|██████▊   | 2.75G/4.03G [00:12<00:11, 110MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  69%|██████▉   | 2.78G/4.03G [00:12<00:08, 142MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  70%|██████▉   | 2.81G/4.03G [00:12<00:07, 162MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  71%|███████   | 2.85G/4.03G [00:12<00:05, 201MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  72%|███████▏  | 2.89G/4.03G [00:12<00:04, 245MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  73%|███████▎  | 2.93G/4.03G [00:13<00:06, 179MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  73%|███████▎  | 2.96G/4.03G [00:13<00:08, 131MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  74%|███████▍  | 3.00G/4.03G [00:13<00:06, 167MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  75%|███████▌  | 3.04G/4.03G [00:13<00:04, 203MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  76%|███████▌  | 3.07G/4.03G [00:13<00:04, 217MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  77%|███████▋  | 3.10G/4.03G [00:13<00:04, 224MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  78%|███████▊  | 3.15G/4.03G [00:14<00:04, 211MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  79%|███████▉  | 3.18G/4.03G [00:14<00:03, 225MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  80%|███████▉  | 3.21G/4.03G [00:14<00:03, 218MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  80%|████████  | 3.24G/4.03G [00:14<00:03, 223MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  81%|████████▏ | 3.28G/4.03G [00:14<00:02, 254MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  82%|████████▏ | 3.31G/4.03G [00:14<00:02, 252MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  83%|████████▎ | 3.34G/4.03G [00:14<00:02, 253MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  84%|████████▎ | 3.38G/4.03G [00:15<00:02, 235MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  84%|████████▍ | 3.41G/4.03G [00:15<00:03, 207MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  85%|████████▌ | 3.44G/4.03G [00:15<00:02, 229MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  86%|████████▋ | 3.48G/4.03G [00:15<00:02, 270MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  87%|████████▋ | 3.51G/4.03G [00:15<00:01, 274MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  88%|████████▊ | 3.55G/4.03G [00:15<00:01, 296MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  89%|████████▉ | 3.60G/4.03G [00:15<00:01, 322MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  90%|█████████ | 3.64G/4.03G [00:16<00:02, 174MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  91%|█████████▏| 3.68G/4.03G [00:16<00:01, 193MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  92%|█████████▏| 3.72G/4.03G [00:16<00:01, 224MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  93%|█████████▎| 3.76G/4.03G [00:16<00:01, 259MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  94%|█████████▍| 3.81G/4.03G [00:16<00:00, 278MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  95%|█████████▌| 3.85G/4.03G [00:16<00:00, 293MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  96%|█████████▋| 3.89G/4.03G [00:17<00:00, 268MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  97%|█████████▋| 3.92G/4.03G [00:17<00:00, 191MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  98%|█████████▊| 3.96G/4.03G [00:17<00:00, 228MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin:  99%|█████████▉| 4.00G/4.03G [00:17<00:00, 229MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin: 100%|█████████▉| 4.03G/4.03G [00:17<00:00, 205MB/s]#033[A\u001b[0m\n",
      "\u001b[34mDownloading (…)l-00002-of-00002.bin: 100%|██████████| 4.03G/4.03G [00:17<00:00, 225MB/s]\u001b[0m\n",
      "\u001b[34mDownloading shards: 100%|██████████| 2/2 [00:57<00:00, 26.85s/it]\u001b[0m\n",
      "\u001b[34mDownloading shards: 100%|██████████| 2/2 [00:57<00:00, 28.74s/it]\u001b[0m\n",
      "\u001b[34mDownloading shards: 100%|██████████| 2/2 [00:57<00:00, 26.85s/it]\u001b[0m\n",
      "\u001b[34mDownloading shards: 100%|██████████| 2/2 [00:57<00:00, 28.74s/it]\u001b[0m\n",
      "\u001b[34mDownloading shards: 100%|██████████| 2/2 [00:57<00:00, 26.86s/it]\u001b[0m\n",
      "\u001b[34mDownloading shards: 100%|██████████| 2/2 [00:57<00:00, 28.75s/it]\u001b[0m\n",
      "\u001b[34mDownloading shards: 100%|██████████| 2/2 [00:57<00:00, 26.86s/it]\u001b[0m\n",
      "\u001b[34mDownloading shards: 100%|██████████| 2/2 [00:57<00:00, 28.74s/it]\u001b[0m\n",
      "\u001b[34m[INFO|modeling_utils.py:1173] 2023-07-17 23:36:00,507 >> Instantiating LlamaForCausalLM model under default dtype torch.float16.\u001b[0m\n",
      "\u001b[34m[INFO|modeling_utils.py:1173] 2023-07-17 23:36:00,507 >> Instantiating LlamaForCausalLM model under default dtype torch.float16.\u001b[0m\n",
      "\u001b[34m[INFO|configuration_utils.py:577] 2023-07-17 23:36:00,507 >> Generate config GenerationConfig {\n",
      "  \"_from_model_config\": true,\n",
      "  \"bos_token_id\": 1,\n",
      "  \"eos_token_id\": 2,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.30.2\"\u001b[0m\n",
      "\u001b[34m}\u001b[0m\n",
      "\u001b[34m[INFO|configuration_utils.py:577] 2023-07-17 23:36:00,507 >> Generate config GenerationConfig {\n",
      "  \"_from_model_config\": true,\n",
      "  \"bos_token_id\": 1,\n",
      "  \"eos_token_id\": 2,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.30.2\"\u001b[0m\n",
      "\u001b[34m}\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  50%|█████     | 1/2 [00:03<00:03,  3.69s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  50%|█████     | 1/2 [00:04<00:04,  4.31s/it]#015Loading checkpoint shards:  50%|█████     | 1/2 [00:04<00:04,  4.32s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards:  50%|█████     | 1/2 [00:04<00:04,  4.33s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 2/2 [00:05<00:00,  2.42s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 2/2 [00:05<00:00,  2.61s/it]\u001b[0m\n",
      "\u001b[34mDownloading (…)neration_config.json:   0%|          | 0.00/132 [00:00<?, ?B/s]\u001b[0m\n",
      "\u001b[34mDownloading (…)neration_config.json: 100%|██████████| 132/132 [00:00<00:00, 43.8kB/s]\u001b[0m\n",
      "\u001b[34m07/17/2023 23:36:06 - INFO - utils.common - Fine-tuning method: LoRA\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 2/2 [00:06<00:00,  2.83s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 2/2 [00:06<00:00,  3.05s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 2/2 [00:06<00:00,  2.82s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 2/2 [00:06<00:00,  3.05s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 2/2 [00:06<00:00,  2.82s/it]\u001b[0m\n",
      "\u001b[34mLoading checkpoint shards: 100%|██████████| 2/2 [00:06<00:00,  3.04s/it]\u001b[0m\n",
      "\u001b[34m[INFO|modeling_utils.py:3295] 2023-07-17 23:36:06,793 >> All model checkpoint weights were used when initializing LlamaForCausalLM.\u001b[0m\n",
      "\u001b[34m[INFO|modeling_utils.py:3295] 2023-07-17 23:36:06,793 >> All model checkpoint weights were used when initializing LlamaForCausalLM.\u001b[0m\n",
      "\u001b[34m[INFO|modeling_utils.py:3303] 2023-07-17 23:36:06,794 >> All the weights of LlamaForCausalLM were initialized from the model checkpoint at fireballoon/baichuan-vicuna-7b.\u001b[0m\n",
      "\u001b[34mIf your task is similar to the task the model of the checkpoint was trained on, you can already use LlamaForCausalLM for predictions without further training.\u001b[0m\n",
      "\u001b[34m[INFO|modeling_utils.py:3303] 2023-07-17 23:36:06,794 >> All the weights of LlamaForCausalLM were initialized from the model checkpoint at fireballoon/baichuan-vicuna-7b.\u001b[0m\n",
      "\u001b[34mIf your task is similar to the task the model of the checkpoint was trained on, you can already use LlamaForCausalLM for predictions without further training.\u001b[0m\n",
      "\u001b[34m07/17/2023 23:36:06 - INFO - utils.common - Fine-tuning method: LoRA\u001b[0m\n",
      "\u001b[34m[INFO|configuration_utils.py:539] 2023-07-17 23:36:06,890 >> loading configuration file generation_config.json from cache at /root/.cache/huggingface/hub/models--fireballoon--baichuan-vicuna-7b/snapshots/d3b83cf408270757cc8b6e6335d7feb45f868164/generation_config.json\u001b[0m\n",
      "\u001b[34m[INFO|configuration_utils.py:539] 2023-07-17 23:36:06,890 >> loading configuration file generation_config.json from cache at /root/.cache/huggingface/hub/models--fireballoon--baichuan-vicuna-7b/snapshots/d3b83cf408270757cc8b6e6335d7feb45f868164/generation_config.json\u001b[0m\n",
      "\u001b[34m[INFO|configuration_utils.py:577] 2023-07-17 23:36:06,890 >> Generate config GenerationConfig {\n",
      "  \"_from_model_config\": true,\n",
      "  \"bos_token_id\": 1,\n",
      "  \"eos_token_id\": 2,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.30.2\"\u001b[0m\n",
      "\u001b[34m}\u001b[0m\n",
      "\u001b[34m[INFO|configuration_utils.py:577] 2023-07-17 23:36:06,890 >> Generate config GenerationConfig {\n",
      "  \"_from_model_config\": true,\n",
      "  \"bos_token_id\": 1,\n",
      "  \"eos_token_id\": 2,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"transformers_version\": \"4.30.2\"\u001b[0m\n",
      "\u001b[34m}\u001b[0m\n",
      "\u001b[34m07/17/2023 23:36:06 - INFO - utils.common - Fine-tuning method: LoRA\u001b[0m\n",
      "\u001b[34m07/17/2023 23:36:06 - INFO - utils.common - Fine-tuning method: LoRA\u001b[0m\n",
      "\u001b[34mtrainable params: 4194304 || all params: 7004753920 || trainable%: 0.0599\u001b[0m\n",
      "\u001b[34mtrainable params: 4194304 || all params: 7004753920 || trainable%: 0.0599\u001b[0m\n",
      "\u001b[34mtrainable params: 4194304 || all params: 7004753920 || trainable%: 0.0599\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:   0%|          | 0/51461 [00:00<?, ? examples/s]\u001b[0m\n",
      "\u001b[34m07/17/2023 23:36:21 - INFO - datasets.arrow_dataset - Caching processed dataset at /root/.cache/huggingface/datasets/json/default-a8e55b3e8ea81aac/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-d1c04d345a0fd2ed.arrow\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:   2%|▏         | 1000/51461 [00:00<00:29, 1739.89 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:   4%|▍         | 2000/51461 [00:01<00:27, 1774.80 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:   6%|▌         | 3000/51461 [00:01<00:27, 1759.93 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:   8%|▊         | 4000/51461 [00:02<00:26, 1764.95 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  10%|▉         | 5000/51461 [00:02<00:26, 1770.82 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  12%|█▏        | 6000/51461 [00:03<00:25, 1780.49 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  14%|█▎        | 7000/51461 [00:03<00:25, 1775.96 examples/s]\u001b[0m\n",
      "\u001b[34mtrainable params: 4194304 || all params: 7004753920 || trainable%: 0.0599\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  16%|█▌        | 8000/51461 [00:04<00:24, 1784.68 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  17%|█▋        | 9000/51461 [00:05<00:23, 1788.24 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  19%|█▉        | 10000/51461 [00:05<00:23, 1772.36 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  21%|██▏       | 11000/51461 [00:06<00:22, 1794.35 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  23%|██▎       | 12000/51461 [00:06<00:22, 1788.64 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  25%|██▌       | 13000/51461 [00:07<00:21, 1801.00 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  27%|██▋       | 14000/51461 [00:07<00:20, 1808.57 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  29%|██▉       | 15000/51461 [00:08<00:20, 1805.27 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  31%|███       | 16000/51461 [00:08<00:19, 1806.43 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  33%|███▎      | 17000/51461 [00:09<00:19, 1805.51 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  35%|███▍      | 18000/51461 [00:10<00:18, 1802.96 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  37%|███▋      | 19000/51461 [00:10<00:18, 1800.81 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  39%|███▉      | 20000/51461 [00:11<00:17, 1787.51 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  41%|████      | 21000/51461 [00:11<00:16, 1802.67 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  43%|████▎     | 22000/51461 [00:12<00:16, 1805.48 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  45%|████▍     | 23000/51461 [00:12<00:15, 1802.36 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  47%|████▋     | 24000/51461 [00:13<00:15, 1794.47 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  49%|████▊     | 25000/51461 [00:13<00:14, 1805.38 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  51%|█████     | 26000/51461 [00:14<00:14, 1808.19 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  52%|█████▏    | 27000/51461 [00:15<00:13, 1799.50 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  54%|█████▍    | 28000/51461 [00:15<00:13, 1803.95 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  56%|█████▋    | 29000/51461 [00:16<00:12, 1788.82 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  58%|█████▊    | 30000/51461 [00:16<00:11, 1810.79 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  60%|██████    | 31000/51461 [00:17<00:11, 1816.33 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  62%|██████▏   | 32000/51461 [00:17<00:10, 1814.37 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  64%|██████▍   | 33000/51461 [00:18<00:10, 1815.18 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  66%|██████▌   | 34000/51461 [00:18<00:09, 1808.55 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  68%|██████▊   | 35000/51461 [00:19<00:09, 1816.59 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  70%|██████▉   | 36000/51461 [00:20<00:08, 1804.17 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  72%|███████▏  | 37000/51461 [00:20<00:08, 1791.49 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  74%|███████▍  | 38000/51461 [00:21<00:07, 1801.59 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  76%|███████▌  | 39000/51461 [00:21<00:06, 1807.81 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  78%|███████▊  | 40000/51461 [00:22<00:06, 1808.29 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  80%|███████▉  | 41000/51461 [00:22<00:05, 1799.94 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  82%|████████▏ | 42000/51461 [00:23<00:05, 1794.55 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  84%|████████▎ | 43000/51461 [00:23<00:04, 1793.74 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  86%|████████▌ | 44000/51461 [00:24<00:04, 1790.11 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  87%|████████▋ | 45000/51461 [00:25<00:03, 1783.19 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  89%|████████▉ | 46000/51461 [00:25<00:03, 1791.40 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  91%|█████████▏| 47000/51461 [00:26<00:02, 1800.43 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  93%|█████████▎| 48000/51461 [00:26<00:01, 1806.70 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  95%|█████████▌| 49000/51461 [00:27<00:01, 1802.55 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  97%|█████████▋| 50000/51461 [00:27<00:00, 1783.23 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  99%|█████████▉| 51000/51461 [00:28<00:00, 1786.00 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset: 100%|██████████| 51461/51461 [00:28<00:00, 1799.73 examples/s]\u001b[0m\n",
      "\u001b[34minput_ids:\u001b[0m\n",
      "\u001b[34m[703, 9805, 1493, 650, 15695, 3475, 680, 755, 1313, 9715, 945, 5840, 9091, 79, 776, 9091, 4278, 8922, 31125, 7320, 31125, 680, 1265, 952, 8943, 678, 656, 3475, 31155, 31114, 3155, 79, 5, 20250, 31143, 14758, 2838, 31173, 6881, 9143, 5972, 22973, 75, 5, 7905, 18056, 31143, 31106, 31106, 53, 79, 31106, 2377, 31525, 31323, 20191, 72, 31248, 31525, 31323, 33180, 33091, 32774, 31473, 31188, 31323, 21318, 73, 31106, 5, 54, 79, 31106, 2377, 31323, 32271, 31399, 31323, 33274, 14839, 4039, 32704, 31323, 72, 11815, 32050, 33021, 31188, 32050, 33091, 73, 31106, 5, 55, 79, 6390, 4492, 31177, 3806, 31525, 31323, 5956, 73, 31106, 5, 56, 79, 31106, 4697, 31323, 31378, 31188, 33341, 34973, 19200, 32827, 31323, 2083, 72, 31404, 4051, 14297, 10352, 73, 31106, 5, 57, 79, 31106, 25760, 1526, 27200, 72, 2377, 31664, 10753, 33180, 33091, 31473, 22366, 22973, 73, 31106, 5, 58, 79, 31106, 14839, 31963, 31323, 72, 7289, 31683, 31889, 31399, 2855, 31552, 30518, 6295, 73, 31106, 5, 59, 79, 31106, 32386, 32259, 31399, 33009, 31350, 31183, 31262, 32256, 31323, 21318, 73, 31106, 5, 60, 79, 31106, 5972, 33547, 31323, 32099, 33526, 6653, 73, 31106, 5, 61, 79, 31106, 22141, 25619, 15524, 2377, 32839, 31323, 31146, 5539, 23480, 31293, 76, 33091, 31832, 31323, 33166, 31188, 33180, 33091, 14291, 31145, 73, 31106, 5, 53, 52, 79, 31106, 31396, 5215, 9289, 10341, 7464, 32050, 33021, 31293, 31188, 23480, 31293, 73, 2]\u001b[0m\n",
      "\u001b[34minputs:\n",
      " A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions.\u001b[0m\n",
      "\u001b[34mHuman: 我们如何在日常生活中减少用水？\u001b[0m\n",
      "\u001b[34mAssistant:  1. 使用节水装置，如节水淋浴喷头和水龙头。 \u001b[0m\n",
      "\u001b[34m2. 使用水箱或水桶收集家庭废水，例如洗碗和洗浴。 \u001b[0m\n",
      "\u001b[34m3. 在社区中提高节水意识。 \u001b[0m\n",
      "\u001b[34m4. 检查水管和灌溉系统的漏水情况，并及时修复它们。 \u001b[0m\n",
      "\u001b[34m5. 洗澡时间缩短，使用低流量淋浴头节约用水。 \u001b[0m\n",
      "\u001b[34m6. 收集雨水，用于园艺或其他非饮用目的。 \u001b[0m\n",
      "\u001b[34m7. 刷牙或擦手时关掉水龙头。 \u001b[0m\n",
      "\u001b[34m8. 减少浇水草坪的时间。 \u001b[0m\n",
      "\u001b[34m9. 尽可能多地重复使用灰水(来自洗衣机、浴室水槽和淋浴的水)。 \u001b[0m\n",
      "\u001b[34m10. 只购买能源效率高的洗碗机和洗衣机。</s>\u001b[0m\n",
      "\u001b[34mlabel_ids:\u001b[0m\n",
      "\u001b[34m[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 31106, 53, 79, 31106, 2377, 31525, 31323, 20191, 72, 31248, 31525, 31323, 33180, 33091, 32774, 31473, 31188, 31323, 21318, 73, 31106, 5, 54, 79, 31106, 2377, 31323, 32271, 31399, 31323, 33274, 14839, 4039, 32704, 31323, 72, 11815, 32050, 33021, 31188, 32050, 33091, 73, 31106, 5, 55, 79, 6390, 4492, 31177, 3806, 31525, 31323, 5956, 73, 31106, 5, 56, 79, 31106, 4697, 31323, 31378, 31188, 33341, 34973, 19200, 32827, 31323, 2083, 72, 31404, 4051, 14297, 10352, 73, 31106, 5, 57, 79, 31106, 25760, 1526, 27200, 72, 2377, 31664, 10753, 33180, 33091, 31473, 22366, 22973, 73, 31106, 5, 58, 79, 31106, 14839, 31963, 31323, 72, 7289, 31683, 31889, 31399, 2855, 31552, 30518, 6295, 73, 31106, 5, 59, 79, 31106, 32386, 32259, 31399, 33009, 31350, 31183, 31262, 32256, 31323, 21318, 73, 31106, 5, 60, 79, 31106, 5972, 33547, 31323, 32099, 33526, 6653, 73, 31106, 5, 61, 79, 31106, 22141, 25619, 15524, 2377, 32839, 31323, 31146, 5539, 23480, 31293, 76, 33091, 31832, 31323, 33166, 31188, 33180, 33091, 14291, 31145, 73, 31106, 5, 53, 52, 79, 31106, 31396, 5215, 9289, 10341, 7464, 32050, 33021, 31293, 31188, 23480, 31293, 73, 2]\u001b[0m\n",
      "\u001b[34mlabels:\u001b[0m\n",
      "\u001b[34m<unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk> 1. 使用节水装置，如节水淋浴喷头和水龙头。 \u001b[0m\n",
      "\u001b[34m2. 使用水箱或水桶收集家庭废水，例如洗碗和洗浴。 \u001b[0m\n",
      "\u001b[34m3. 在社区中提高节水意识。 \u001b[0m\n",
      "\u001b[34m4. 检查水管和灌溉系统的漏水情况，并及时修复它们。 \u001b[0m\n",
      "\u001b[34m5. 洗澡时间缩短，使用低流量淋浴头节约用水。 \u001b[0m\n",
      "\u001b[34m6. 收集雨水，用于园艺或其他非饮用目的。 \u001b[0m\n",
      "\u001b[34m7. 刷牙或擦手时关掉水龙头。 \u001b[0m\n",
      "\u001b[34m8. 减少浇水草坪的时间。 \u001b[0m\n",
      "\u001b[34m9. 尽可能多地重复使用灰水(来自洗衣机、浴室水槽和淋浴的水)。 \u001b[0m\n",
      "\u001b[34m10. 只购买能源效率高的洗碗机和洗衣机。</s>\u001b[0m\n",
      "\u001b[34mNCCL version 2.14.3+cuda11.7\u001b[0m\n",
      "\u001b[34malgo-1:230:290 [0] ofi_init:1304 NCCL WARN NET/OFI Only EFA provider is supported\u001b[0m\n",
      "\u001b[34malgo-1:230:290 [0] ofi_init:1355 NCCL WARN NET/OFI aws-ofi-nccl initialization failed\u001b[0m\n",
      "\u001b[34malgo-1:231:291 [1] ofi_init:1304 NCCL WARN NET/OFI Only EFA provider is supported\u001b[0m\n",
      "\u001b[34malgo-1:231:291 [1] ofi_init:1355 NCCL WARN NET/OFI aws-ofi-nccl initialization failed\u001b[0m\n",
      "\u001b[34malgo-1:232:292 [2] ofi_init:1304 NCCL WARN NET/OFI Only EFA provider is supported\u001b[0m\n",
      "\u001b[34malgo-1:232:292 [2] ofi_init:1355 NCCL WARN NET/OFI aws-ofi-nccl initialization failed\u001b[0m\n",
      "\u001b[34malgo-1:233:293 [3] ofi_init:1304 NCCL WARN NET/OFI Only EFA provider is supported\u001b[0m\n",
      "\u001b[34malgo-1:233:293 [3] ofi_init:1355 NCCL WARN NET/OFI aws-ofi-nccl initialization failed\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:   0%|          | 0/51461 [00:00<?, ? examples/s]#015Running tokenizer on dataset:   0%|          | 0/51461 [00:00<?, ? examples/s]#015Running tokenizer on dataset:   0%|          | 0/51461 [00:00<?, ? examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:   2%|▏         | 1000/51461 [00:00<00:32, 1576.27 examples/s]#015Running tokenizer on dataset:   2%|▏         | 1000/51461 [00:00<00:32, 1576.22 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:   2%|▏         | 1000/51461 [00:00<00:32, 1564.18 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:   4%|▍         | 2000/51461 [00:01<00:28, 1710.83 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:   4%|▍         | 2000/51461 [00:01<00:29, 1702.46 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:   4%|▍         | 2000/51461 [00:01<00:29, 1691.52 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:   6%|▌         | 3000/51461 [00:01<00:27, 1738.72 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:   6%|▌         | 3000/51461 [00:01<00:28, 1727.25 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:   6%|▌         | 3000/51461 [00:01<00:28, 1716.57 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:   8%|▊         | 4000/51461 [00:02<00:26, 1762.22 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:   8%|▊         | 4000/51461 [00:02<00:27, 1748.57 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:   8%|▊         | 4000/51461 [00:02<00:27, 1738.07 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  10%|▉         | 5000/51461 [00:02<00:26, 1774.36 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  10%|▉         | 5000/51461 [00:02<00:26, 1761.95 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  10%|▉         | 5000/51461 [00:02<00:26, 1750.75 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  12%|█▏        | 6000/51461 [00:03<00:25, 1790.28 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  12%|█▏        | 6000/51461 [00:03<00:25, 1776.13 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  12%|█▏        | 6000/51461 [00:03<00:25, 1763.53 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  14%|█▎        | 7000/51461 [00:03<00:24, 1789.23 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  14%|█▎        | 7000/51461 [00:03<00:25, 1774.53 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  14%|█▎        | 7000/51461 [00:04<00:25, 1761.16 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  16%|█▌        | 8000/51461 [00:04<00:24, 1804.52 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  16%|█▌        | 8000/51461 [00:04<00:24, 1787.81 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  16%|█▌        | 8000/51461 [00:04<00:24, 1776.04 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  17%|█▋        | 9000/51461 [00:05<00:23, 1806.06 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  17%|█▋        | 9000/51461 [00:05<00:23, 1789.75 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  17%|█▋        | 9000/51461 [00:05<00:23, 1777.67 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  19%|█▉        | 10000/51461 [00:05<00:23, 1794.11 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  19%|█▉        | 10000/51461 [00:05<00:23, 1778.49 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  19%|█▉        | 10000/51461 [00:05<00:23, 1765.56 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  21%|██▏       | 11000/51461 [00:06<00:22, 1815.43 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  21%|██▏       | 11000/51461 [00:06<00:22, 1798.87 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  21%|██▏       | 11000/51461 [00:06<00:22, 1787.12 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  23%|██▎       | 12000/51461 [00:06<00:21, 1807.19 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  23%|██▎       | 12000/51461 [00:06<00:22, 1791.52 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  23%|██▎       | 12000/51461 [00:06<00:22, 1779.30 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  25%|██▌       | 13000/51461 [00:07<00:21, 1816.70 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  25%|██▌       | 13000/51461 [00:07<00:21, 1801.54 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  25%|██▌       | 13000/51461 [00:07<00:21, 1790.32 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  27%|██▋       | 14000/51461 [00:07<00:20, 1819.31 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  27%|██▋       | 14000/51461 [00:07<00:20, 1805.81 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  27%|██▋       | 14000/51461 [00:07<00:20, 1794.88 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  29%|██▉       | 15000/51461 [00:08<00:20, 1816.86 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  29%|██▉       | 15000/51461 [00:08<00:20, 1803.21 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  29%|██▉       | 15000/51461 [00:08<00:20, 1792.30 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  31%|███       | 16000/51461 [00:08<00:19, 1816.57 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  31%|███       | 16000/51461 [00:08<00:19, 1803.99 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  31%|███       | 16000/51461 [00:09<00:19, 1791.67 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  33%|███▎      | 17000/51461 [00:09<00:18, 1815.07 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  33%|███▎      | 17000/51461 [00:09<00:19, 1802.22 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  33%|███▎      | 17000/51461 [00:09<00:19, 1790.76 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  35%|███▍      | 18000/51461 [00:10<00:18, 1812.84 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  35%|███▍      | 18000/51461 [00:10<00:18, 1798.00 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  35%|███▍      | 18000/51461 [00:10<00:18, 1785.61 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  37%|███▋      | 19000/51461 [00:10<00:17, 1808.40 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  37%|███▋      | 19000/51461 [00:10<00:18, 1793.42 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  37%|███▋      | 19000/51461 [00:10<00:18, 1782.45 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  39%|███▉      | 20000/51461 [00:11<00:17, 1797.65 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  39%|███▉      | 20000/51461 [00:11<00:17, 1783.22 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  39%|███▉      | 20000/51461 [00:11<00:17, 1769.96 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  41%|████      | 21000/51461 [00:11<00:16, 1816.53 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  41%|████      | 21000/51461 [00:11<00:16, 1801.40 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  41%|████      | 21000/51461 [00:11<00:17, 1789.91 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  43%|████▎     | 22000/51461 [00:12<00:16, 1818.83 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  43%|████▎     | 22000/51461 [00:12<00:16, 1804.37 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  43%|████▎     | 22000/51461 [00:12<00:16, 1791.10 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  45%|████▍     | 23000/51461 [00:12<00:15, 1814.07 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  45%|████▍     | 23000/51461 [00:12<00:15, 1800.86 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  45%|████▍     | 23000/51461 [00:12<00:15, 1786.62 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  47%|████▋     | 24000/51461 [00:13<00:15, 1804.09 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  47%|████▋     | 24000/51461 [00:13<00:15, 1789.69 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  47%|████▋     | 24000/51461 [00:13<00:15, 1778.41 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  49%|████▊     | 25000/51461 [00:13<00:14, 1817.09 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  49%|████▊     | 25000/51461 [00:13<00:14, 1801.83 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  49%|████▊     | 25000/51461 [00:14<00:14, 1791.10 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  51%|█████     | 26000/51461 [00:14<00:13, 1820.66 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  51%|█████     | 26000/51461 [00:14<00:14, 1804.56 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  51%|█████     | 26000/51461 [00:14<00:14, 1793.64 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  52%|█████▏    | 27000/51461 [00:14<00:13, 1809.75 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  52%|█████▏    | 27000/51461 [00:15<00:13, 1795.71 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  52%|█████▏    | 27000/51461 [00:15<00:13, 1783.72 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  54%|█████▍    | 28000/51461 [00:15<00:12, 1811.40 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  54%|█████▍    | 28000/51461 [00:15<00:13, 1796.15 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  54%|█████▍    | 28000/51461 [00:15<00:13, 1785.69 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  56%|█████▋    | 29000/51461 [00:16<00:12, 1793.49 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  56%|█████▋    | 29000/51461 [00:16<00:12, 1777.76 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  56%|█████▋    | 29000/51461 [00:16<00:12, 1768.61 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  58%|█████▊    | 30000/51461 [00:16<00:11, 1815.30 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  58%|█████▊    | 30000/51461 [00:16<00:11, 1798.49 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  58%|█████▊    | 30000/51461 [00:16<00:11, 1789.90 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  60%|██████    | 31000/51461 [00:17<00:11, 1819.64 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  60%|██████    | 31000/51461 [00:17<00:11, 1804.18 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  60%|██████    | 31000/51461 [00:17<00:11, 1796.04 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  62%|██████▏   | 32000/51461 [00:17<00:10, 1816.29 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  62%|██████▏   | 32000/51461 [00:17<00:10, 1804.55 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  62%|██████▏   | 32000/51461 [00:18<00:10, 1795.02 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  64%|██████▍   | 33000/51461 [00:18<00:10, 1818.60 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  64%|██████▍   | 33000/51461 [00:18<00:10, 1806.21 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  64%|██████▍   | 33000/51461 [00:18<00:10, 1797.11 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  66%|██████▌   | 34000/51461 [00:18<00:09, 1814.28 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  66%|██████▌   | 34000/51461 [00:19<00:09, 1800.08 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  66%|██████▌   | 34000/51461 [00:19<00:09, 1791.25 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  68%|██████▊   | 35000/51461 [00:19<00:09, 1821.12 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  68%|██████▊   | 35000/51461 [00:19<00:09, 1807.20 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  68%|██████▊   | 35000/51461 [00:19<00:09, 1797.96 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  70%|██████▉   | 36000/51461 [00:19<00:08, 1808.64 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  70%|██████▉   | 36000/51461 [00:20<00:08, 1793.01 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  70%|██████▉   | 36000/51461 [00:20<00:08, 1784.67 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  72%|███████▏  | 37000/51461 [00:20<00:08, 1796.90 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  72%|███████▏  | 37000/51461 [00:20<00:08, 1782.51 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  72%|███████▏  | 37000/51461 [00:20<00:08, 1773.52 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  74%|███████▍  | 38000/51461 [00:21<00:07, 1807.05 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  74%|███████▍  | 38000/51461 [00:21<00:07, 1792.06 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  74%|███████▍  | 38000/51461 [00:21<00:07, 1784.04 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  76%|███████▌  | 39000/51461 [00:21<00:06, 1812.50 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  76%|███████▌  | 39000/51461 [00:21<00:06, 1797.79 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  76%|███████▌  | 39000/51461 [00:21<00:06, 1789.48 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  78%|███████▊  | 40000/51461 [00:22<00:06, 1812.13 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  78%|███████▊  | 40000/51461 [00:22<00:06, 1796.05 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  78%|███████▊  | 40000/51461 [00:22<00:06, 1789.25 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  80%|███████▉  | 41000/51461 [00:22<00:05, 1804.44 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  80%|███████▉  | 41000/51461 [00:22<00:05, 1791.55 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  80%|███████▉  | 41000/51461 [00:23<00:05, 1783.31 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  82%|████████▏ | 42000/51461 [00:23<00:05, 1798.56 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  82%|████████▏ | 42000/51461 [00:23<00:05, 1785.92 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  82%|████████▏ | 42000/51461 [00:23<00:05, 1776.47 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  84%|████████▎ | 43000/51461 [00:23<00:04, 1798.54 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  84%|████████▎ | 43000/51461 [00:24<00:04, 1786.89 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  84%|████████▎ | 43000/51461 [00:24<00:04, 1774.50 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  86%|████████▌ | 44000/51461 [00:24<00:04, 1792.74 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  86%|████████▌ | 44000/51461 [00:24<00:04, 1779.60 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  86%|████████▌ | 44000/51461 [00:24<00:04, 1768.68 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  87%|████████▋ | 45000/51461 [00:24<00:03, 1784.23 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  87%|████████▋ | 45000/51461 [00:25<00:03, 1771.62 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  87%|████████▋ | 45000/51461 [00:25<00:03, 1765.40 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  89%|████████▉ | 46000/51461 [00:25<00:03, 1796.94 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  89%|████████▉ | 46000/51461 [00:25<00:03, 1786.71 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  89%|████████▉ | 46000/51461 [00:25<00:03, 1775.68 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  91%|█████████▏| 47000/51461 [00:26<00:02, 1808.10 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  91%|█████████▏| 47000/51461 [00:26<00:02, 1799.14 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  91%|█████████▏| 47000/51461 [00:26<00:02, 1784.08 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  93%|█████████▎| 48000/51461 [00:26<00:01, 1812.97 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  93%|█████████▎| 48000/51461 [00:26<00:01, 1804.04 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  93%|█████████▎| 48000/51461 [00:26<00:01, 1788.92 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  95%|█████████▌| 49000/51461 [00:27<00:01, 1805.62 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  95%|█████████▌| 49000/51461 [00:27<00:01, 1796.43 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  95%|█████████▌| 49000/51461 [00:27<00:01, 1783.92 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  97%|█████████▋| 50000/51461 [00:27<00:00, 1786.58 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  97%|█████████▋| 50000/51461 [00:27<00:00, 1777.77 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  97%|█████████▋| 50000/51461 [00:28<00:00, 1764.10 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  99%|█████████▉| 51000/51461 [00:28<00:00, 1787.78 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  99%|█████████▉| 51000/51461 [00:28<00:00, 1777.37 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset: 100%|██████████| 51461/51461 [00:28<00:00, 1802.08 examples/s]\u001b[0m\n",
      "\u001b[34minput_ids:\u001b[0m\n",
      "\u001b[34m[703, 9805, 1493, 650, 15695, 3475, 680, 755, 1313, 9715, 945, 5840, 9091, 79, 776, 9091, 4278, 8922, 31125, 7320, 31125, 680, 1265, 952, 8943, 678, 656, 3475, 31155, 31114, 3155, 79, 5, 20250, 31143, 14758, 2838, 31173, 6881, 9143, 5972, 22973, 75, 5, 7905, 18056, 31143, 31106, 31106, 53, 79, 31106, 2377, 31525, 31323, 20191, 72, 31248, 31525, 31323, 33180, 33091, 32774, 31473, 31188, 31323, 21318, 73, 31106, 5, 54, 79, 31106, 2377, 31323, 32271, 31399, 31323, 33274, 14839, 4039, 32704, 31323, 72, 11815, 32050, 33021, 31188, 32050, 33091, 73, 31106, 5, 55, 79, 6390, 4492, 31177, 3806, 31525, 31323, 5956, 73, 31106, 5, 56, 79, 31106, 4697, 31323, 31378, 31188, 33341, 34973, 19200, 32827, 31323, 2083, 72, 31404, 4051, 14297, 10352, 73, 31106, 5, 57, 79, 31106, 25760, 1526, 27200, 72, 2377, 31664, 10753, 33180, 33091, 31473, 22366, 22973, 73, 31106, 5, 58, 79, 31106, 14839, 31963, 31323, 72, 7289, 31683, 31889, 31399, 2855, 31552, 30518, 6295, 73, 31106, 5, 59, 79, 31106, 32386, 32259, 31399, 33009, 31350, 31183, 31262, 32256, 31323, 21318, 73, 31106, 5, 60, 79, 31106, 5972, 33547, 31323, 32099, 33526, 6653, 73, 31106, 5, 61, 79, 31106, 22141, 25619, 15524, 2377, 32839, 31323, 31146, 5539, 23480, 31293, 76, 33091, 31832, 31323, 33166, 31188, 33180, 33091, 14291, 31145, 73, 31106, 5, 53, 52, 79, 31106, 31396, 5215, 9289, 10341, 7464, 32050, 33021, 31293, 31188, 23480, 31293, 73, 2]\u001b[0m\n",
      "\u001b[34minputs:\n",
      " A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions.\u001b[0m\n",
      "\u001b[34mHuman: 我们如何在日常生活中减少用水？\u001b[0m\n",
      "\u001b[34mAssistant:  1. 使用节水装置，如节水淋浴喷头和水龙头。 \u001b[0m\n",
      "\u001b[34m2. 使用水箱或水桶收集家庭废水，例如洗碗和洗浴。 \u001b[0m\n",
      "\u001b[34m3. 在社区中提高节水意识。 \u001b[0m\n",
      "\u001b[34m4. 检查水管和灌溉系统的漏水情况，并及时修复它们。 \u001b[0m\n",
      "\u001b[34m5. 洗澡时间缩短，使用低流量淋浴头节约用水。 \u001b[0m\n",
      "\u001b[34m6. 收集雨水，用于园艺或其他非饮用目的。 \u001b[0m\n",
      "\u001b[34m7. 刷牙或擦手时关掉水龙头。 \u001b[0m\n",
      "\u001b[34m8. 减少浇水草坪的时间。 \u001b[0m\n",
      "\u001b[34m9. 尽可能多地重复使用灰水(来自洗衣机、浴室水槽和淋浴的水)。 \u001b[0m\n",
      "\u001b[34m10. 只购买能源效率高的洗碗机和洗衣机。</s>\u001b[0m\n",
      "\u001b[34mlabel_ids:\u001b[0m\n",
      "\u001b[34m[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 31106, 53, 79, 31106, 2377, 31525, 31323, 20191, 72, 31248, 31525, 31323, 33180, 33091, 32774, 31473, 31188, 31323, 21318, 73, 31106, 5, 54, 79, 31106, 2377, 31323, 32271, 31399, 31323, 33274, 14839, 4039, 32704, 31323, 72, 11815, 32050, 33021, 31188, 32050, 33091, 73, 31106, 5, 55, 79, 6390, 4492, 31177, 3806, 31525, 31323, 5956, 73, 31106, 5, 56, 79, 31106, 4697, 31323, 31378, 31188, 33341, 34973, 19200, 32827, 31323, 2083, 72, 31404, 4051, 14297, 10352, 73, 31106, 5, 57, 79, 31106, 25760, 1526, 27200, 72, 2377, 31664, 10753, 33180, 33091, 31473, 22366, 22973, 73, 31106, 5, 58, 79, 31106, 14839, 31963, 31323, 72, 7289, 31683, 31889, 31399, 2855, 31552, 30518, 6295, 73, 31106, 5, 59, 79, 31106, 32386, 32259, 31399, 33009, 31350, 31183, 31262, 32256, 31323, 21318, 73, 31106, 5, 60, 79, 31106, 5972, 33547, 31323, 32099, 33526, 6653, 73, 31106, 5, 61, 79, 31106, 22141, 25619, 15524, 2377, 32839, 31323, 31146, 5539, 23480, 31293, 76, 33091, 31832, 31323, 33166, 31188, 33180, 33091, 14291, 31145, 73, 31106, 5, 53, 52, 79, 31106, 31396, 5215, 9289, 10341, 7464, 32050, 33021, 31293, 31188, 23480, 31293, 73, 2]\u001b[0m\n",
      "\u001b[34mlabels:\u001b[0m\n",
      "\u001b[34m<unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk> 1. 使用节水装置，如节水淋浴喷头和水龙头。 \u001b[0m\n",
      "\u001b[34m2. 使用水箱或水桶收集家庭废水，例如洗碗和洗浴。 \u001b[0m\n",
      "\u001b[34m3. 在社区中提高节水意识。 \u001b[0m\n",
      "\u001b[34m4. 检查水管和灌溉系统的漏水情况，并及时修复它们。 \u001b[0m\n",
      "\u001b[34m5. 洗澡时间缩短，使用低流量淋浴头节约用水。 \u001b[0m\n",
      "\u001b[34m6. 收集雨水，用于园艺或其他非饮用目的。 \u001b[0m\n",
      "\u001b[34m7. 刷牙或擦手时关掉水龙头。 \u001b[0m\n",
      "\u001b[34m8. 减少浇水草坪的时间。 \u001b[0m\n",
      "\u001b[34m9. 尽可能多地重复使用灰水(来自洗衣机、浴室水槽和淋浴的水)。 \u001b[0m\n",
      "\u001b[34m10. 只购买能源效率高的洗碗机和洗衣机。</s>\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset:  99%|█████████▉| 51000/51461 [00:28<00:00, 1764.30 examples/s]\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset: 100%|██████████| 51461/51461 [00:28<00:00, 1790.26 examples/s]\u001b[0m\n",
      "\u001b[34minput_ids:\u001b[0m\n",
      "\u001b[34m[703, 9805, 1493, 650, 15695, 3475, 680, 755, 1313, 9715, 945, 5840, 9091, 79, 776, 9091, 4278, 8922, 31125, 7320, 31125, 680, 1265, 952, 8943, 678, 656, 3475, 31155, 31114, 3155, 79, 5, 20250, 31143, 14758, 2838, 31173, 6881, 9143, 5972, 22973, 75, 5, 7905, 18056, 31143, 31106, 31106, 53, 79, 31106, 2377, 31525, 31323, 20191, 72, 31248, 31525, 31323, 33180, 33091, 32774, 31473, 31188, 31323, 21318, 73, 31106, 5, 54, 79, 31106, 2377, 31323, 32271, 31399, 31323, 33274, 14839, 4039, 32704, 31323, 72, 11815, 32050, 33021, 31188, 32050, 33091, 73, 31106, 5, 55, 79, 6390, 4492, 31177, 3806, 31525, 31323, 5956, 73, 31106, 5, 56, 79, 31106, 4697, 31323, 31378, 31188, 33341, 34973, 19200, 32827, 31323, 2083, 72, 31404, 4051, 14297, 10352, 73, 31106, 5, 57, 79, 31106, 25760, 1526, 27200, 72, 2377, 31664, 10753, 33180, 33091, 31473, 22366, 22973, 73, 31106, 5, 58, 79, 31106, 14839, 31963, 31323, 72, 7289, 31683, 31889, 31399, 2855, 31552, 30518, 6295, 73, 31106, 5, 59, 79, 31106, 32386, 32259, 31399, 33009, 31350, 31183, 31262, 32256, 31323, 21318, 73, 31106, 5, 60, 79, 31106, 5972, 33547, 31323, 32099, 33526, 6653, 73, 31106, 5, 61, 79, 31106, 22141, 25619, 15524, 2377, 32839, 31323, 31146, 5539, 23480, 31293, 76, 33091, 31832, 31323, 33166, 31188, 33180, 33091, 14291, 31145, 73, 31106, 5, 53, 52, 79, 31106, 31396, 5215, 9289, 10341, 7464, 32050, 33021, 31293, 31188, 23480, 31293, 73, 2]\u001b[0m\n",
      "\u001b[34minputs:\n",
      " A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions.\u001b[0m\n",
      "\u001b[34mHuman: 我们如何在日常生活中减少用水？\u001b[0m\n",
      "\u001b[34mAssistant:  1. 使用节水装置，如节水淋浴喷头和水龙头。 \u001b[0m\n",
      "\u001b[34m2. 使用水箱或水桶收集家庭废水，例如洗碗和洗浴。 \u001b[0m\n",
      "\u001b[34m3. 在社区中提高节水意识。 \u001b[0m\n",
      "\u001b[34m4. 检查水管和灌溉系统的漏水情况，并及时修复它们。 \u001b[0m\n",
      "\u001b[34m5. 洗澡时间缩短，使用低流量淋浴头节约用水。 \u001b[0m\n",
      "\u001b[34m6. 收集雨水，用于园艺或其他非饮用目的。 \u001b[0m\n",
      "\u001b[34m7. 刷牙或擦手时关掉水龙头。 \u001b[0m\n",
      "\u001b[34m8. 减少浇水草坪的时间。 \u001b[0m\n",
      "\u001b[34m9. 尽可能多地重复使用灰水(来自洗衣机、浴室水槽和淋浴的水)。 \u001b[0m\n",
      "\u001b[34m10. 只购买能源效率高的洗碗机和洗衣机。</s>\u001b[0m\n",
      "\u001b[34mlabel_ids:\u001b[0m\n",
      "\u001b[34m[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 31106, 53, 79, 31106, 2377, 31525, 31323, 20191, 72, 31248, 31525, 31323, 33180, 33091, 32774, 31473, 31188, 31323, 21318, 73, 31106, 5, 54, 79, 31106, 2377, 31323, 32271, 31399, 31323, 33274, 14839, 4039, 32704, 31323, 72, 11815, 32050, 33021, 31188, 32050, 33091, 73, 31106, 5, 55, 79, 6390, 4492, 31177, 3806, 31525, 31323, 5956, 73, 31106, 5, 56, 79, 31106, 4697, 31323, 31378, 31188, 33341, 34973, 19200, 32827, 31323, 2083, 72, 31404, 4051, 14297, 10352, 73, 31106, 5, 57, 79, 31106, 25760, 1526, 27200, 72, 2377, 31664, 10753, 33180, 33091, 31473, 22366, 22973, 73, 31106, 5, 58, 79, 31106, 14839, 31963, 31323, 72, 7289, 31683, 31889, 31399, 2855, 31552, 30518, 6295, 73, 31106, 5, 59, 79, 31106, 32386, 32259, 31399, 33009, 31350, 31183, 31262, 32256, 31323, 21318, 73, 31106, 5, 60, 79, 31106, 5972, 33547, 31323, 32099, 33526, 6653, 73, 31106, 5, 61, 79, 31106, 22141, 25619, 15524, 2377, 32839, 31323, 31146, 5539, 23480, 31293, 76, 33091, 31832, 31323, 33166, 31188, 33180, 33091, 14291, 31145, 73, 31106, 5, 53, 52, 79, 31106, 31396, 5215, 9289, 10341, 7464, 32050, 33021, 31293, 31188, 23480, 31293, 73, 2]\u001b[0m\n",
      "\u001b[34mlabels:\u001b[0m\n",
      "\u001b[34m<unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk> 1. 使用节水装置，如节水淋浴喷头和水龙头。 \u001b[0m\n",
      "\u001b[34m2. 使用水箱或水桶收集家庭废水，例如洗碗和洗浴。 \u001b[0m\n",
      "\u001b[34m3. 在社区中提高节水意识。 \u001b[0m\n",
      "\u001b[34m4. 检查水管和灌溉系统的漏水情况，并及时修复它们。 \u001b[0m\n",
      "\u001b[34m5. 洗澡时间缩短，使用低流量淋浴头节约用水。 \u001b[0m\n",
      "\u001b[34m6. 收集雨水，用于园艺或其他非饮用目的。 \u001b[0m\n",
      "\u001b[34m7. 刷牙或擦手时关掉水龙头。 \u001b[0m\n",
      "\u001b[34m8. 减少浇水草坪的时间。 \u001b[0m\n",
      "\u001b[34m9. 尽可能多地重复使用灰水(来自洗衣机、浴室水槽和淋浴的水)。 \u001b[0m\n",
      "\u001b[34m10. 只购买能源效率高的洗碗机和洗衣机。</s>\u001b[0m\n",
      "\u001b[34mRunning tokenizer on dataset: 100%|██████████| 51461/51461 [00:28<00:00, 1778.92 examples/s]\u001b[0m\n",
      "\u001b[34minput_ids:\u001b[0m\n",
      "\u001b[34m[703, 9805, 1493, 650, 15695, 3475, 680, 755, 1313, 9715, 945, 5840, 9091, 79, 776, 9091, 4278, 8922, 31125, 7320, 31125, 680, 1265, 952, 8943, 678, 656, 3475, 31155, 31114, 3155, 79, 5, 20250, 31143, 14758, 2838, 31173, 6881, 9143, 5972, 22973, 75, 5, 7905, 18056, 31143, 31106, 31106, 53, 79, 31106, 2377, 31525, 31323, 20191, 72, 31248, 31525, 31323, 33180, 33091, 32774, 31473, 31188, 31323, 21318, 73, 31106, 5, 54, 79, 31106, 2377, 31323, 32271, 31399, 31323, 33274, 14839, 4039, 32704, 31323, 72, 11815, 32050, 33021, 31188, 32050, 33091, 73, 31106, 5, 55, 79, 6390, 4492, 31177, 3806, 31525, 31323, 5956, 73, 31106, 5, 56, 79, 31106, 4697, 31323, 31378, 31188, 33341, 34973, 19200, 32827, 31323, 2083, 72, 31404, 4051, 14297, 10352, 73, 31106, 5, 57, 79, 31106, 25760, 1526, 27200, 72, 2377, 31664, 10753, 33180, 33091, 31473, 22366, 22973, 73, 31106, 5, 58, 79, 31106, 14839, 31963, 31323, 72, 7289, 31683, 31889, 31399, 2855, 31552, 30518, 6295, 73, 31106, 5, 59, 79, 31106, 32386, 32259, 31399, 33009, 31350, 31183, 31262, 32256, 31323, 21318, 73, 31106, 5, 60, 79, 31106, 5972, 33547, 31323, 32099, 33526, 6653, 73, 31106, 5, 61, 79, 31106, 22141, 25619, 15524, 2377, 32839, 31323, 31146, 5539, 23480, 31293, 76, 33091, 31832, 31323, 33166, 31188, 33180, 33091, 14291, 31145, 73, 31106, 5, 53, 52, 79, 31106, 31396, 5215, 9289, 10341, 7464, 32050, 33021, 31293, 31188, 23480, 31293, 73, 2]\u001b[0m\n",
      "\u001b[34minputs:\n",
      " A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions.\u001b[0m\n",
      "\u001b[34mHuman: 我们如何在日常生活中减少用水？\u001b[0m\n",
      "\u001b[34mAssistant:  1. 使用节水装置，如节水淋浴喷头和水龙头。 \u001b[0m\n",
      "\u001b[34m2. 使用水箱或水桶收集家庭废水，例如洗碗和洗浴。 \u001b[0m\n",
      "\u001b[34m3. 在社区中提高节水意识。 \u001b[0m\n",
      "\u001b[34m4. 检查水管和灌溉系统的漏水情况，并及时修复它们。 \u001b[0m\n",
      "\u001b[34m5. 洗澡时间缩短，使用低流量淋浴头节约用水。 \u001b[0m\n",
      "\u001b[34m6. 收集雨水，用于园艺或其他非饮用目的。 \u001b[0m\n",
      "\u001b[34m7. 刷牙或擦手时关掉水龙头。 \u001b[0m\n",
      "\u001b[34m8. 减少浇水草坪的时间。 \u001b[0m\n",
      "\u001b[34m9. 尽可能多地重复使用灰水(来自洗衣机、浴室水槽和淋浴的水)。 \u001b[0m\n",
      "\u001b[34m10. 只购买能源效率高的洗碗机和洗衣机。</s>\u001b[0m\n",
      "\u001b[34mlabel_ids:\u001b[0m\n",
      "\u001b[34m[-100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, 31106, 53, 79, 31106, 2377, 31525, 31323, 20191, 72, 31248, 31525, 31323, 33180, 33091, 32774, 31473, 31188, 31323, 21318, 73, 31106, 5, 54, 79, 31106, 2377, 31323, 32271, 31399, 31323, 33274, 14839, 4039, 32704, 31323, 72, 11815, 32050, 33021, 31188, 32050, 33091, 73, 31106, 5, 55, 79, 6390, 4492, 31177, 3806, 31525, 31323, 5956, 73, 31106, 5, 56, 79, 31106, 4697, 31323, 31378, 31188, 33341, 34973, 19200, 32827, 31323, 2083, 72, 31404, 4051, 14297, 10352, 73, 31106, 5, 57, 79, 31106, 25760, 1526, 27200, 72, 2377, 31664, 10753, 33180, 33091, 31473, 22366, 22973, 73, 31106, 5, 58, 79, 31106, 14839, 31963, 31323, 72, 7289, 31683, 31889, 31399, 2855, 31552, 30518, 6295, 73, 31106, 5, 59, 79, 31106, 32386, 32259, 31399, 33009, 31350, 31183, 31262, 32256, 31323, 21318, 73, 31106, 5, 60, 79, 31106, 5972, 33547, 31323, 32099, 33526, 6653, 73, 31106, 5, 61, 79, 31106, 22141, 25619, 15524, 2377, 32839, 31323, 31146, 5539, 23480, 31293, 76, 33091, 31832, 31323, 33166, 31188, 33180, 33091, 14291, 31145, 73, 31106, 5, 53, 52, 79, 31106, 31396, 5215, 9289, 10341, 7464, 32050, 33021, 31293, 31188, 23480, 31293, 73, 2]\u001b[0m\n",
      "\u001b[34mlabels:\u001b[0m\n",
      "\u001b[34m<unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk><unk> 1. 使用节水装置，如节水淋浴喷头和水龙头。 \u001b[0m\n",
      "\u001b[34m2. 使用水箱或水桶收集家庭废水，例如洗碗和洗浴。 \u001b[0m\n",
      "\u001b[34m3. 在社区中提高节水意识。 \u001b[0m\n",
      "\u001b[34m4. 检查水管和灌溉系统的漏水情况，并及时修复它们。 \u001b[0m\n",
      "\u001b[34m5. 洗澡时间缩短，使用低流量淋浴头节约用水。 \u001b[0m\n",
      "\u001b[34m6. 收集雨水，用于园艺或其他非饮用目的。 \u001b[0m\n",
      "\u001b[34m7. 刷牙或擦手时关掉水龙头。 \u001b[0m\n",
      "\u001b[34m8. 减少浇水草坪的时间。 \u001b[0m\n",
      "\u001b[34m9. 尽可能多地重复使用灰水(来自洗衣机、浴室水槽和淋浴的水)。 \u001b[0m\n",
      "\u001b[34m10. 只购买能源效率高的洗碗机和洗衣机。</s>\u001b[0m\n",
      "\u001b[34m[INFO|trainer.py:1786] 2023-07-17 23:37:25,044 >> ***** Running training *****\u001b[0m\n",
      "\u001b[34m[INFO|trainer.py:1787] 2023-07-17 23:37:25,044 >>   Num examples = 51,425\u001b[0m\n",
      "\u001b[34m[INFO|trainer.py:1788] 2023-07-17 23:37:25,044 >>   Num Epochs = 1\u001b[0m\n",
      "\u001b[34m[INFO|trainer.py:1789] 2023-07-17 23:37:25,044 >>   Instantaneous batch size per device = 4\u001b[0m\n",
      "\u001b[34m[INFO|trainer.py:1786] 2023-07-17 23:37:25,044 >> ***** Running training *****\u001b[0m\n",
      "\u001b[34m[INFO|trainer.py:1787] 2023-07-17 23:37:25,044 >>   Num examples = 51,425\u001b[0m\n",
      "\u001b[34m[INFO|trainer.py:1788] 2023-07-17 23:37:25,044 >>   Num Epochs = 1\u001b[0m\n",
      "\u001b[34m[INFO|trainer.py:1789] 2023-07-17 23:37:25,044 >>   Instantaneous batch size per device = 4\u001b[0m\n",
      "\u001b[34m[INFO|trainer.py:1790] 2023-07-17 23:37:25,044 >>   Total train batch size (w. parallel, distributed & accumulation) = 64\u001b[0m\n",
      "\u001b[34m[INFO|trainer.py:1791] 2023-07-17 23:37:25,045 >>   Gradient Accumulation steps = 4\u001b[0m\n",
      "\u001b[34m[INFO|trainer.py:1792] 2023-07-17 23:37:25,045 >>   Total optimization steps = 803\u001b[0m\n",
      "\u001b[34m[INFO|trainer.py:1790] 2023-07-17 23:37:25,044 >>   Total train batch size (w. parallel, distributed & accumulation) = 64\u001b[0m\n",
      "\u001b[34m[INFO|trainer.py:1791] 2023-07-17 23:37:25,045 >>   Gradient Accumulation steps = 4\u001b[0m\n",
      "\u001b[34m[INFO|trainer.py:1792] 2023-07-17 23:37:25,045 >>   Total optimization steps = 803\u001b[0m\n",
      "\u001b[34m[INFO|trainer.py:1793] 2023-07-17 23:37:25,047 >>   Number of trainable parameters = 4,194,304\u001b[0m\n",
      "\u001b[34m[INFO|trainer.py:1793] 2023-07-17 23:37:25,047 >>   Number of trainable parameters = 4,194,304\u001b[0m\n",
      "\u001b[34m0%|          | 0/803 [00:00<?, ?it/s]\u001b[0m\n",
      "\u001b[34m[2023-07-17 23:37:26.442: W smdistributed/modelparallel/torch/nn/predefined_hooks.py:78] Found unsupported HuggingFace version 4.30.2 for automated tensor parallelism. HuggingFace modules will not be automatically distributed. You can use smp.tp_register_with_module API to register desired modules for tensor parallelism, or directly instantiate an smp.nn.DistributedModule. Supported HuggingFace transformers versions for automated tensor parallelism: ['4.17.0', '4.20.1', '4.21.0']\u001b[0m\n",
      "\u001b[34m[2023-07-17 23:37:26.443: W smdistributed/modelparallel/torch/nn/predefined_hooks.py:78] Found unsupported HuggingFace version 4.30.2 for automated tensor parallelism. HuggingFace modules will not be automatically distributed. You can use smp.tp_register_with_module API to register desired modules for tensor parallelism, or directly instantiate an smp.nn.DistributedModule. Supported HuggingFace transformers versions for automated tensor parallelism: ['4.17.0', '4.20.1', '4.21.0']\u001b[0m\n",
      "\u001b[34m07/17/2023 23:37:26 - INFO - root - Using NamedTuple = typing._NamedTuple instead.\u001b[0m\n",
      "\u001b[34m07/17/2023 23:37:26 - INFO - root - Using NamedTuple = typing._NamedTuple instead.\u001b[0m\n",
      "\u001b[34m[2023-07-17 23:37:26.468 algo-1:230 INFO utils.py:28] RULE_JOB_STOP_SIGNAL_FILENAME: None\u001b[0m\n",
      "\u001b[34m[2023-07-17 23:37:26.470 algo-1:232 INFO utils.py:28] RULE_JOB_STOP_SIGNAL_FILENAME: None\u001b[0m\n",
      "\u001b[34m[2023-07-17 23:37:26.493 algo-1:230 INFO profiler_config_parser.py:111] Unable to find config at /opt/ml/input/config/profilerconfig.json. Profiler is disabled.\u001b[0m\n",
      "\u001b[34m[2023-07-17 23:37:26.497 algo-1:232 INFO profiler_config_parser.py:111] Unable to find config at /opt/ml/input/config/profilerconfig.json. Profiler is disabled.\u001b[0m\n",
      "\u001b[34m[2023-07-17 23:37:26.654: W smdistributed/modelparallel/torch/nn/predefined_hooks.py:78] Found unsupported HuggingFace version 4.30.2 for automated tensor parallelism. HuggingFace modules will not be automatically distributed. You can use smp.tp_register_with_module API to register desired modules for tensor parallelism, or directly instantiate an smp.nn.DistributedModule. Supported HuggingFace transformers versions for automated tensor parallelism: ['4.17.0', '4.20.1', '4.21.0']\u001b[0m\n",
      "\u001b[34m07/17/2023 23:37:26 - INFO - root - Using NamedTuple = typing._NamedTuple instead.\u001b[0m\n",
      "\u001b[34m[2023-07-17 23:37:26.663: W smdistributed/modelparallel/torch/nn/predefined_hooks.py:78] Found unsupported HuggingFace version 4.30.2 for automated tensor parallelism. HuggingFace modules will not be automatically distributed. You can use smp.tp_register_with_module API to register desired modules for tensor parallelism, or directly instantiate an smp.nn.DistributedModule. Supported HuggingFace transformers versions for automated tensor parallelism: ['4.17.0', '4.20.1', '4.21.0']\u001b[0m\n",
      "\u001b[34m07/17/2023 23:37:26 - INFO - root - Using NamedTuple = typing._NamedTuple instead.\u001b[0m\n",
      "\u001b[34m[2023-07-17 23:37:26.680 algo-1:233 INFO utils.py:28] RULE_JOB_STOP_SIGNAL_FILENAME: None\u001b[0m\n",
      "\u001b[34m[2023-07-17 23:37:26.689 algo-1:231 INFO utils.py:28] RULE_JOB_STOP_SIGNAL_FILENAME: None\u001b[0m\n",
      "\u001b[34m[2023-07-17 23:37:26.704 algo-1:233 INFO profiler_config_parser.py:111] Unable to find config at /opt/ml/input/config/profilerconfig.json. Profiler is disabled.\u001b[0m\n",
      "\u001b[34m[2023-07-17 23:37:26.714 algo-1:231 INFO profiler_config_parser.py:111] Unable to find config at /opt/ml/input/config/profilerconfig.json. Profiler is disabled.\u001b[0m\n",
      "\u001b[34m0%|          | 1/803 [00:04<57:18,  4.29s/it]\u001b[0m\n",
      "\u001b[34m07/17/2023 23:37:30 - INFO - torch.nn.parallel.distributed - Reducer buckets have been rebuilt in this iteration.\u001b[0m\n",
      "\u001b[34m07/17/2023 23:37:30 - INFO - torch.nn.parallel.distributed - Reducer buckets have been rebuilt in this iteration.\u001b[0m\n",
      "\u001b[34m07/17/2023 23:37:30 - INFO - torch.nn.parallel.distributed - Reducer buckets have been rebuilt in this iteration.\u001b[0m\n",
      "\u001b[34m07/17/2023 23:37:30 - INFO - torch.nn.parallel.distributed - Reducer buckets have been rebuilt in this iteration.\u001b[0m\n",
      "\u001b[34m0%|          | 2/803 [00:08<52:45,  3.95s/it]\u001b[0m\n",
      "\u001b[34m0%|          | 3/803 [00:10<45:45,  3.43s/it]\u001b[0m\n",
      "\u001b[34m0%|          | 4/803 [00:14<45:51,  3.44s/it]\u001b[0m\n",
      "\u001b[34m1%|          | 5/803 [00:17<43:16,  3.25s/it]\u001b[0m\n",
      "\u001b[34m1%|          | 6/803 [00:21<46:20,  3.49s/it]\u001b[0m\n",
      "\u001b[34m1%|          | 7/803 [00:24<45:45,  3.45s/it]\u001b[0m\n",
      "\u001b[34m1%|          | 8/803 [00:28<48:23,  3.65s/it]\u001b[0m\n",
      "\u001b[34m1%|          | 9/803 [00:32<49:45,  3.76s/it]\u001b[0m\n",
      "\u001b[34m1%|          | 10/803 [00:36<48:59,  3.71s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 2.7687, 'learning_rate': 4.99808696343081e-05, 'epoch': 0.01}\u001b[0m\n",
      "\u001b[34m1%|          | 10/803 [00:36<48:59,  3.71s/it]\u001b[0m\n",
      "\u001b[34m1%|▏         | 11/803 [00:39<48:12,  3.65s/it]\u001b[0m\n",
      "\u001b[34m1%|▏         | 12/803 [00:43<50:30,  3.83s/it]\u001b[0m\n",
      "\u001b[34m2%|▏         | 13/803 [00:47<48:23,  3.68s/it]\u001b[0m\n",
      "\u001b[34m2%|▏         | 14/803 [00:50<46:51,  3.56s/it]\u001b[0m\n",
      "\u001b[34m2%|▏         | 15/803 [00:53<45:37,  3.47s/it]\u001b[0m\n",
      "\u001b[34m2%|▏         | 16/803 [00:57<48:15,  3.68s/it]\u001b[0m\n",
      "\u001b[34m2%|▏         | 17/803 [01:01<49:17,  3.76s/it]\u001b[0m\n",
      "\u001b[34m2%|▏         | 18/803 [01:05<48:39,  3.72s/it]\u001b[0m\n",
      "\u001b[34m2%|▏         | 19/803 [01:09<47:47,  3.66s/it]\u001b[0m\n",
      "\u001b[34m2%|▏         | 20/803 [01:12<46:36,  3.57s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 2.4653, 'learning_rate': 4.9923507814903724e-05, 'epoch': 0.02}\u001b[0m\n",
      "\u001b[34m2%|▏         | 20/803 [01:12<46:36,  3.57s/it]\u001b[0m\n",
      "\u001b[34m3%|▎         | 21/803 [01:15<45:27,  3.49s/it]\u001b[0m\n",
      "\u001b[34m3%|▎         | 22/803 [01:19<45:27,  3.49s/it]\u001b[0m\n",
      "\u001b[34m3%|▎         | 23/803 [01:24<52:22,  4.03s/it]\u001b[0m\n",
      "\u001b[34m3%|▎         | 24/803 [01:28<53:19,  4.11s/it]\u001b[0m\n",
      "\u001b[34m3%|▎         | 25/803 [01:31<49:16,  3.80s/it]\u001b[0m\n",
      "\u001b[34m3%|▎         | 26/803 [01:35<49:55,  3.86s/it]\u001b[0m\n",
      "\u001b[34m3%|▎         | 27/803 [01:39<49:58,  3.86s/it]\u001b[0m\n",
      "\u001b[34m3%|▎         | 28/803 [01:43<50:35,  3.92s/it]\u001b[0m\n",
      "\u001b[34m4%|▎         | 29/803 [01:47<49:39,  3.85s/it]\u001b[0m\n",
      "\u001b[34m4%|▎         | 30/803 [01:51<48:15,  3.75s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 2.2821, 'learning_rate': 4.982800232999343e-05, 'epoch': 0.04}\u001b[0m\n",
      "\u001b[34m4%|▎         | 30/803 [01:51<48:15,  3.75s/it]\u001b[0m\n",
      "\u001b[34m4%|▍         | 31/803 [01:55<50:05,  3.89s/it]\u001b[0m\n",
      "\u001b[34m4%|▍         | 32/803 [01:58<48:18,  3.76s/it]\u001b[0m\n",
      "\u001b[34m4%|▍         | 33/803 [02:03<53:43,  4.19s/it]\u001b[0m\n",
      "\u001b[34m4%|▍         | 34/803 [02:07<51:47,  4.04s/it]\u001b[0m\n",
      "\u001b[34m4%|▍         | 35/803 [02:11<50:59,  3.98s/it]\u001b[0m\n",
      "\u001b[34m4%|▍         | 36/803 [02:14<48:12,  3.77s/it]\u001b[0m\n",
      "\u001b[34m5%|▍         | 37/803 [02:18<46:23,  3.63s/it]\u001b[0m\n",
      "\u001b[34m5%|▍         | 38/803 [02:21<45:14,  3.55s/it]\u001b[0m\n",
      "\u001b[34m5%|▍         | 39/803 [02:25<46:45,  3.67s/it]\u001b[0m\n",
      "\u001b[34m5%|▍         | 40/803 [02:28<45:23,  3.57s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 2.1371, 'learning_rate': 4.9694499343965376e-05, 'epoch': 0.05}\u001b[0m\n",
      "\u001b[34m5%|▍         | 40/803 [02:28<45:23,  3.57s/it]\u001b[0m\n",
      "\u001b[34m5%|▌         | 41/803 [02:33<51:46,  4.08s/it]\u001b[0m\n",
      "\u001b[34m5%|▌         | 42/803 [02:37<48:27,  3.82s/it]\u001b[0m\n",
      "\u001b[34m5%|▌         | 43/803 [02:41<50:09,  3.96s/it]\u001b[0m\n",
      "\u001b[34m5%|▌         | 44/803 [02:45<51:11,  4.05s/it]\u001b[0m\n",
      "\u001b[34m6%|▌         | 45/803 [02:49<50:07,  3.97s/it]\u001b[0m\n",
      "\u001b[34m6%|▌         | 46/803 [02:53<49:16,  3.91s/it]\u001b[0m\n",
      "\u001b[34m6%|▌         | 47/803 [02:56<47:59,  3.81s/it]\u001b[0m\n",
      "\u001b[34m6%|▌         | 48/803 [03:00<48:55,  3.89s/it]\u001b[0m\n",
      "\u001b[34m6%|▌         | 49/803 [03:04<49:34,  3.94s/it]\u001b[0m\n",
      "\u001b[34m6%|▌         | 50/803 [03:08<46:22,  3.70s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 2.0424, 'learning_rate': 4.952320317369503e-05, 'epoch': 0.06}\u001b[0m\n",
      "\u001b[34m6%|▌         | 50/803 [03:08<46:22,  3.70s/it]\u001b[0m\n",
      "\u001b[34m6%|▋         | 51/803 [03:12<50:32,  4.03s/it]\u001b[0m\n",
      "\u001b[34m6%|▋         | 52/803 [03:16<49:43,  3.97s/it]\u001b[0m\n",
      "\u001b[34m7%|▋         | 53/803 [03:20<50:30,  4.04s/it]\u001b[0m\n",
      "\u001b[34m7%|▋         | 54/803 [03:24<48:46,  3.91s/it]\u001b[0m\n",
      "\u001b[34m7%|▋         | 55/803 [03:27<45:14,  3.63s/it]\u001b[0m\n",
      "\u001b[34m7%|▋         | 56/803 [03:32<48:58,  3.93s/it]\u001b[0m\n",
      "\u001b[34m7%|▋         | 57/803 [03:35<47:47,  3.84s/it]\u001b[0m\n",
      "\u001b[34m7%|▋         | 58/803 [03:40<49:33,  3.99s/it]\u001b[0m\n",
      "\u001b[34m7%|▋         | 59/803 [03:43<48:02,  3.87s/it]\u001b[0m\n",
      "\u001b[34m7%|▋         | 60/803 [03:47<46:56,  3.79s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.9388, 'learning_rate': 4.9314375975852735e-05, 'epoch': 0.07}\u001b[0m\n",
      "\u001b[34m7%|▋         | 60/803 [03:47<46:56,  3.79s/it]\u001b[0m\n",
      "\u001b[34m8%|▊         | 61/803 [03:52<50:44,  4.10s/it]\u001b[0m\n",
      "\u001b[34m8%|▊         | 62/803 [03:55<48:13,  3.91s/it]\u001b[0m\n",
      "\u001b[34m8%|▊         | 63/803 [03:58<45:08,  3.66s/it]\u001b[0m\n",
      "\u001b[34m8%|▊         | 64/803 [04:02<46:06,  3.74s/it]\u001b[0m\n",
      "\u001b[34m8%|▊         | 65/803 [04:07<49:13,  4.00s/it]\u001b[0m\n",
      "\u001b[34m8%|▊         | 66/803 [04:10<44:54,  3.66s/it]\u001b[0m\n",
      "\u001b[34m8%|▊         | 67/803 [04:13<42:38,  3.48s/it]\u001b[0m\n",
      "\u001b[34m8%|▊         | 68/803 [04:16<41:25,  3.38s/it]\u001b[0m\n",
      "\u001b[34m9%|▊         | 69/803 [04:19<42:14,  3.45s/it]\u001b[0m\n",
      "\u001b[34m9%|▊         | 70/803 [04:23<43:09,  3.53s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.8891, 'learning_rate': 4.9068337345691365e-05, 'epoch': 0.09}\u001b[0m\n",
      "\u001b[34m9%|▊         | 70/803 [04:23<43:09,  3.53s/it]\u001b[0m\n",
      "\u001b[34m9%|▉         | 71/803 [04:27<43:58,  3.60s/it]\u001b[0m\n",
      "\u001b[34m9%|▉         | 72/803 [04:32<48:13,  3.96s/it]\u001b[0m\n",
      "\u001b[34m9%|▉         | 73/803 [04:35<45:57,  3.78s/it]\u001b[0m\n",
      "\u001b[34m9%|▉         | 74/803 [04:38<44:31,  3.66s/it]\u001b[0m\n",
      "\u001b[34m9%|▉         | 75/803 [04:42<43:01,  3.55s/it]\u001b[0m\n",
      "\u001b[34m9%|▉         | 76/803 [04:45<42:18,  3.49s/it]\u001b[0m\n",
      "\u001b[34m10%|▉         | 77/803 [04:50<48:25,  4.00s/it]\u001b[0m\n",
      "\u001b[34m10%|▉         | 78/803 [04:54<47:00,  3.89s/it]\u001b[0m\n",
      "\u001b[34m10%|▉         | 79/803 [04:57<44:59,  3.73s/it]\u001b[0m\n",
      "\u001b[34m10%|▉         | 80/803 [05:01<44:29,  3.69s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.906, 'learning_rate': 4.878546382792847e-05, 'epoch': 0.1}\u001b[0m\n",
      "\u001b[34m10%|▉         | 80/803 [05:01<44:29,  3.69s/it]\u001b[0m\n",
      "\u001b[34m10%|█         | 81/803 [05:04<42:01,  3.49s/it]\u001b[0m\n",
      "\u001b[34m10%|█         | 82/803 [05:07<41:54,  3.49s/it]\u001b[0m\n",
      "\u001b[34m10%|█         | 83/803 [05:12<46:53,  3.91s/it]\u001b[0m\n",
      "\u001b[34m10%|█         | 84/803 [05:16<47:07,  3.93s/it]\u001b[0m\n",
      "\u001b[34m11%|█         | 85/803 [05:20<45:41,  3.82s/it]\u001b[0m\n",
      "\u001b[34m11%|█         | 86/803 [05:22<41:26,  3.47s/it]\u001b[0m\n",
      "\u001b[34m11%|█         | 87/803 [05:26<41:45,  3.50s/it]\u001b[0m\n",
      "\u001b[34m11%|█         | 88/803 [05:29<41:37,  3.49s/it]\u001b[0m\n",
      "\u001b[34m11%|█         | 89/803 [05:33<42:08,  3.54s/it]\u001b[0m\n",
      "\u001b[34m11%|█         | 90/803 [05:38<46:42,  3.93s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.8435, 'learning_rate': 4.846618834047121e-05, 'epoch': 0.11}\u001b[0m\n",
      "\u001b[34m11%|█         | 90/803 [05:38<46:42,  3.93s/it]\u001b[0m\n",
      "\u001b[34m11%|█▏        | 91/803 [05:42<46:25,  3.91s/it]\u001b[0m\n",
      "\u001b[34m11%|█▏        | 92/803 [05:46<45:49,  3.87s/it]\u001b[0m\n",
      "\u001b[34m12%|█▏        | 93/803 [05:49<45:02,  3.81s/it]\u001b[0m\n",
      "\u001b[34m12%|█▏        | 94/803 [05:53<44:39,  3.78s/it]\u001b[0m\n",
      "\u001b[34m12%|█▏        | 95/803 [05:57<45:46,  3.88s/it]\u001b[0m\n",
      "\u001b[34m12%|█▏        | 96/803 [06:01<46:07,  3.91s/it]\u001b[0m\n",
      "\u001b[34m12%|█▏        | 97/803 [06:05<45:39,  3.88s/it]\u001b[0m\n",
      "\u001b[34m12%|█▏        | 98/803 [06:10<48:35,  4.14s/it]\u001b[0m\n",
      "\u001b[34m12%|█▏        | 99/803 [06:13<47:10,  4.02s/it]\u001b[0m\n",
      "\u001b[34m12%|█▏        | 100/803 [06:17<44:49,  3.83s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.9321, 'learning_rate': 4.811099951186609e-05, 'epoch': 0.12}\u001b[0m\n",
      "\u001b[34m12%|█▏        | 100/803 [06:17<44:49,  3.83s/it]\u001b[0m\n",
      "\u001b[34m13%|█▎        | 101/803 [06:20<41:32,  3.55s/it]\u001b[0m\n",
      "\u001b[34m13%|█▎        | 102/803 [06:24<43:12,  3.70s/it]\u001b[0m\n",
      "\u001b[34m13%|█▎        | 103/803 [06:29<47:15,  4.05s/it]\u001b[0m\n",
      "\u001b[34m13%|█▎        | 104/803 [06:33<47:26,  4.07s/it]\u001b[0m\n",
      "\u001b[34m13%|█▎        | 105/803 [06:38<52:20,  4.50s/it]\u001b[0m\n",
      "\u001b[34m13%|█▎        | 106/803 [06:42<48:28,  4.17s/it]\u001b[0m\n",
      "\u001b[34m13%|█▎        | 107/803 [06:46<48:15,  4.16s/it]\u001b[0m\n",
      "\u001b[34m13%|█▎        | 108/803 [06:49<46:10,  3.99s/it]\u001b[0m\n",
      "\u001b[34m14%|█▎        | 109/803 [06:53<44:01,  3.81s/it]\u001b[0m\n",
      "\u001b[34m14%|█▎        | 110/803 [06:56<43:35,  3.77s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.8532, 'learning_rate': 4.7720440933487575e-05, 'epoch': 0.14}\u001b[0m\n",
      "\u001b[34m14%|█▎        | 110/803 [06:56<43:35,  3.77s/it]\u001b[0m\n",
      "\u001b[34m14%|█▍        | 111/803 [07:00<42:20,  3.67s/it]\u001b[0m\n",
      "\u001b[34m14%|█▍        | 112/803 [07:05<46:24,  4.03s/it]\u001b[0m\n",
      "\u001b[34m14%|█▍        | 113/803 [07:09<47:17,  4.11s/it]\u001b[0m\n",
      "\u001b[34m14%|█▍        | 114/803 [07:14<49:28,  4.31s/it]\u001b[0m\n",
      "\u001b[34m14%|█▍        | 115/803 [07:17<47:21,  4.13s/it]\u001b[0m\n",
      "\u001b[34m14%|█▍        | 116/803 [07:22<47:02,  4.11s/it]\u001b[0m\n",
      "\u001b[34m15%|█▍        | 117/803 [07:25<45:00,  3.94s/it]\u001b[0m\n",
      "\u001b[34m15%|█▍        | 118/803 [07:29<44:32,  3.90s/it]\u001b[0m\n",
      "\u001b[34m15%|█▍        | 119/803 [07:32<42:35,  3.74s/it]\u001b[0m\n",
      "\u001b[34m15%|█▍        | 120/803 [07:37<45:55,  4.03s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.8934, 'learning_rate': 4.729511032760996e-05, 'epoch': 0.15}\u001b[0m\n",
      "\u001b[34m15%|█▍        | 120/803 [07:37<45:55,  4.03s/it]\u001b[0m\n",
      "\u001b[34m15%|█▌        | 121/803 [07:42<50:13,  4.42s/it]\u001b[0m\n",
      "\u001b[34m15%|█▌        | 122/803 [07:47<51:06,  4.50s/it]\u001b[0m\n",
      "\u001b[34m15%|█▌        | 123/803 [07:52<51:48,  4.57s/it]\u001b[0m\n",
      "\u001b[34m15%|█▌        | 124/803 [07:55<48:58,  4.33s/it]\u001b[0m\n",
      "\u001b[34m16%|█▌        | 125/803 [07:58<44:01,  3.90s/it]\u001b[0m\n",
      "\u001b[34m16%|█▌        | 126/803 [08:02<41:58,  3.72s/it]\u001b[0m\n",
      "\u001b[34m16%|█▌        | 127/803 [08:05<41:03,  3.64s/it]\u001b[0m\n",
      "\u001b[34m16%|█▌        | 128/803 [08:08<39:03,  3.47s/it]\u001b[0m\n",
      "\u001b[34m16%|█▌        | 129/803 [08:12<38:43,  3.45s/it]\u001b[0m\n",
      "\u001b[34m16%|█▌        | 130/803 [08:15<39:22,  3.51s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.839, 'learning_rate': 4.683565863263568e-05, 'epoch': 0.16}\u001b[0m\n",
      "\u001b[34m16%|█▌        | 130/803 [08:15<39:22,  3.51s/it]\u001b[0m\n",
      "\u001b[34m16%|█▋        | 131/803 [08:18<37:47,  3.37s/it]\u001b[0m\n",
      "\u001b[34m16%|█▋        | 132/803 [08:23<41:23,  3.70s/it]\u001b[0m\n",
      "\u001b[34m17%|█▋        | 133/803 [08:26<39:25,  3.53s/it]\u001b[0m\n",
      "\u001b[34m17%|█▋        | 134/803 [08:30<40:15,  3.61s/it]\u001b[0m\n",
      "\u001b[34m17%|█▋        | 135/803 [08:33<38:34,  3.47s/it]\u001b[0m\n",
      "\u001b[34m17%|█▋        | 136/803 [08:36<37:28,  3.37s/it]\u001b[0m\n",
      "\u001b[34m17%|█▋        | 137/803 [08:40<38:50,  3.50s/it]\u001b[0m\n",
      "\u001b[34m17%|█▋        | 138/803 [08:44<41:25,  3.74s/it]\u001b[0m\n",
      "\u001b[34m17%|█▋        | 139/803 [08:48<40:26,  3.66s/it]\u001b[0m\n",
      "\u001b[34m17%|█▋        | 140/803 [08:53<46:11,  4.18s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.8915, 'learning_rate': 4.634278900688013e-05, 'epoch': 0.17}\u001b[0m\n",
      "\u001b[34m17%|█▋        | 140/803 [08:53<46:11,  4.18s/it]\u001b[0m\n",
      "\u001b[34m18%|█▊        | 141/803 [08:59<50:45,  4.60s/it]\u001b[0m\n",
      "\u001b[34m18%|█▊        | 142/803 [09:02<47:43,  4.33s/it]\u001b[0m\n",
      "\u001b[34m18%|█▊        | 143/803 [09:06<45:17,  4.12s/it]\u001b[0m\n",
      "\u001b[34m18%|█▊        | 144/803 [09:11<47:47,  4.35s/it]\u001b[0m\n",
      "\u001b[34m18%|█▊        | 145/803 [09:14<44:04,  4.02s/it]\u001b[0m\n",
      "\u001b[34m18%|█▊        | 146/803 [09:17<42:11,  3.85s/it]\u001b[0m\n",
      "\u001b[34m18%|█▊        | 147/803 [09:21<40:40,  3.72s/it]\u001b[0m\n",
      "\u001b[34m18%|█▊        | 148/803 [09:25<43:32,  3.99s/it]\u001b[0m\n",
      "\u001b[34m19%|█▊        | 149/803 [09:29<42:59,  3.94s/it]\u001b[0m\n",
      "\u001b[34m19%|█▊        | 150/803 [09:33<41:34,  3.82s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.8379, 'learning_rate': 4.581725575243765e-05, 'epoch': 0.19}\u001b[0m\n",
      "\u001b[34m19%|█▊        | 150/803 [09:33<41:34,  3.82s/it]\u001b[0m\n",
      "\u001b[34m19%|█▉        | 151/803 [09:37<42:43,  3.93s/it]\u001b[0m\n",
      "\u001b[34m19%|█▉        | 152/803 [09:41<41:34,  3.83s/it]\u001b[0m\n",
      "\u001b[34m19%|█▉        | 153/803 [09:44<39:09,  3.61s/it]\u001b[0m\n",
      "\u001b[34m19%|█▉        | 154/803 [09:47<37:21,  3.45s/it]\u001b[0m\n",
      "\u001b[34m19%|█▉        | 155/803 [09:52<41:21,  3.83s/it]\u001b[0m\n",
      "\u001b[34m19%|█▉        | 156/803 [09:56<42:36,  3.95s/it]\u001b[0m\n",
      "\u001b[34m20%|█▉        | 157/803 [09:59<40:42,  3.78s/it]\u001b[0m\n",
      "\u001b[34m20%|█▉        | 158/803 [10:02<39:03,  3.63s/it]\u001b[0m\n",
      "\u001b[34m20%|█▉        | 159/803 [10:05<37:06,  3.46s/it]\u001b[0m\n",
      "\u001b[34m20%|█▉        | 160/803 [10:09<38:07,  3.56s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.8369, 'learning_rate': 4.525986316077551e-05, 'epoch': 0.2}\u001b[0m\n",
      "\u001b[34m20%|█▉        | 160/803 [10:09<38:07,  3.56s/it]\u001b[0m\n",
      "\u001b[34m20%|██        | 161/803 [10:13<39:44,  3.71s/it]\u001b[0m\n",
      "\u001b[34m20%|██        | 162/803 [10:17<39:15,  3.67s/it]\u001b[0m\n",
      "\u001b[34m20%|██        | 163/803 [10:20<37:11,  3.49s/it]\u001b[0m\n",
      "\u001b[34m20%|██        | 164/803 [10:24<38:39,  3.63s/it]\u001b[0m\n",
      "\u001b[34m21%|██        | 165/803 [10:27<38:06,  3.58s/it]\u001b[0m\n",
      "\u001b[34m21%|██        | 166/803 [10:30<35:57,  3.39s/it]\u001b[0m\n",
      "\u001b[34m21%|██        | 167/803 [10:33<34:21,  3.24s/it]\u001b[0m\n",
      "\u001b[34m21%|██        | 168/803 [10:37<35:20,  3.34s/it]\u001b[0m\n",
      "\u001b[34m21%|██        | 169/803 [10:40<36:20,  3.44s/it]\u001b[0m\n",
      "\u001b[34m21%|██        | 170/803 [10:43<34:18,  3.25s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.8494, 'learning_rate': 4.467146428182267e-05, 'epoch': 0.21}\u001b[0m\n",
      "\u001b[34m21%|██        | 170/803 [10:43<34:18,  3.25s/it]\u001b[0m\n",
      "\u001b[34m21%|██▏       | 171/803 [10:47<36:03,  3.42s/it]\u001b[0m\n",
      "\u001b[34m21%|██▏       | 172/803 [10:51<38:38,  3.67s/it]\u001b[0m\n",
      "\u001b[34m22%|██▏       | 173/803 [10:55<37:38,  3.58s/it]\u001b[0m\n",
      "\u001b[34m22%|██▏       | 174/803 [10:58<37:36,  3.59s/it]\u001b[0m\n",
      "\u001b[34m22%|██▏       | 175/803 [11:03<39:30,  3.77s/it]\u001b[0m\n",
      "\u001b[34m22%|██▏       | 176/803 [11:06<38:09,  3.65s/it]\u001b[0m\n",
      "\u001b[34m22%|██▏       | 177/803 [11:10<40:18,  3.86s/it]\u001b[0m\n",
      "\u001b[34m22%|██▏       | 178/803 [11:13<36:44,  3.53s/it]\u001b[0m\n",
      "\u001b[34m22%|██▏       | 179/803 [11:16<36:11,  3.48s/it]\u001b[0m\n",
      "\u001b[34m22%|██▏       | 180/803 [11:20<37:29,  3.61s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.9084, 'learning_rate': 4.4052959618437336e-05, 'epoch': 0.22}\u001b[0m\n",
      "\u001b[34m22%|██▏       | 180/803 [11:20<37:29,  3.61s/it]\u001b[0m\n",
      "\u001b[34m23%|██▎       | 181/803 [11:23<35:06,  3.39s/it]\u001b[0m\n",
      "\u001b[34m23%|██▎       | 182/803 [11:28<39:14,  3.79s/it]\u001b[0m\n",
      "\u001b[34m23%|██▎       | 183/803 [11:32<39:10,  3.79s/it]\u001b[0m\n",
      "\u001b[34m23%|██▎       | 184/803 [11:35<38:21,  3.72s/it]\u001b[0m\n",
      "\u001b[34m23%|██▎       | 185/803 [11:39<37:25,  3.63s/it]\u001b[0m\n",
      "\u001b[34m23%|██▎       | 186/803 [11:43<40:25,  3.93s/it]\u001b[0m\n",
      "\u001b[34m23%|██▎       | 187/803 [11:47<38:38,  3.76s/it]\u001b[0m\n",
      "\u001b[34m23%|██▎       | 188/803 [11:51<38:59,  3.80s/it]\u001b[0m\n",
      "\u001b[34m24%|██▎       | 189/803 [11:54<37:20,  3.65s/it]\u001b[0m\n",
      "\u001b[34m24%|██▎       | 190/803 [11:57<36:08,  3.54s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.8468, 'learning_rate': 4.3405295748250903e-05, 'epoch': 0.24}\u001b[0m\n",
      "\u001b[34m24%|██▎       | 190/803 [11:57<36:08,  3.54s/it]\u001b[0m\n",
      "\u001b[34m24%|██▍       | 191/803 [12:00<35:06,  3.44s/it]\u001b[0m\n",
      "\u001b[34m24%|██▍       | 192/803 [12:04<35:33,  3.49s/it]\u001b[0m\n",
      "\u001b[34m24%|██▍       | 193/803 [12:07<34:55,  3.44s/it]\u001b[0m\n",
      "\u001b[34m24%|██▍       | 194/803 [12:11<36:35,  3.60s/it]\u001b[0m\n",
      "\u001b[34m24%|██▍       | 195/803 [12:16<41:10,  4.06s/it]\u001b[0m\n",
      "\u001b[34m24%|██▍       | 196/803 [12:20<38:55,  3.85s/it]\u001b[0m\n",
      "\u001b[34m25%|██▍       | 197/803 [12:24<38:34,  3.82s/it]\u001b[0m\n",
      "\u001b[34m25%|██▍       | 198/803 [12:28<41:02,  4.07s/it]\u001b[0m\n",
      "\u001b[34m25%|██▍       | 199/803 [12:32<40:27,  4.02s/it]\u001b[0m\n",
      "\u001b[34m25%|██▍       | 200/803 [12:36<40:28,  4.03s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.835, 'learning_rate': 4.272946387499794e-05, 'epoch': 0.25}\u001b[0m\n",
      "\u001b[34m25%|██▍       | 200/803 [12:36<40:28,  4.03s/it]\u001b[0m\n",
      "\u001b[34m25%|██▌       | 201/803 [12:40<39:08,  3.90s/it]\u001b[0m\n",
      "\u001b[34m25%|██▌       | 202/803 [12:43<37:21,  3.73s/it]\u001b[0m\n",
      "\u001b[34m25%|██▌       | 203/803 [12:47<37:55,  3.79s/it]\u001b[0m\n",
      "\u001b[34m25%|██▌       | 204/803 [12:51<37:14,  3.73s/it]\u001b[0m\n",
      "\u001b[34m26%|██▌       | 205/803 [12:55<39:29,  3.96s/it]\u001b[0m\n",
      "\u001b[34m26%|██▌       | 206/803 [12:58<37:19,  3.75s/it]\u001b[0m\n",
      "\u001b[34m26%|██▌       | 207/803 [13:03<38:42,  3.90s/it]\u001b[0m\n",
      "\u001b[34m26%|██▌       | 208/803 [13:07<40:01,  4.04s/it]\u001b[0m\n",
      "\u001b[34m41%|████      | 327/803 [20:27<27:12,  3.43s/it]\u001b[0m\n",
      "\u001b[34m41%|████      | 328/803 [20:30<26:29,  3.35s/it]\u001b[0m\n",
      "\u001b[34m41%|████      | 329/803 [20:35<30:10,  3.82s/it]\u001b[0m\n",
      "\u001b[34m41%|████      | 330/803 [20:38<28:52,  3.66s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.8143, 'learning_rate': 3.1902424327436734e-05, 'epoch': 0.41}\u001b[0m\n",
      "\u001b[34m41%|████      | 330/803 [20:38<28:52,  3.66s/it]\u001b[0m\n",
      "\u001b[34m41%|████      | 331/803 [20:42<28:56,  3.68s/it]\u001b[0m\n",
      "\u001b[34m41%|████▏     | 332/803 [20:45<26:52,  3.42s/it]\u001b[0m\n",
      "\u001b[34m41%|████▏     | 333/803 [20:48<25:59,  3.32s/it]\u001b[0m\n",
      "\u001b[34m42%|████▏     | 334/803 [20:53<28:42,  3.67s/it]\u001b[0m\n",
      "\u001b[34m42%|████▏     | 335/803 [20:56<28:16,  3.62s/it]\u001b[0m\n",
      "\u001b[34m42%|████▏     | 336/803 [20:59<27:33,  3.54s/it]\u001b[0m\n",
      "\u001b[34m42%|████▏     | 337/803 [21:03<28:19,  3.65s/it]\u001b[0m\n",
      "\u001b[34m42%|████▏     | 338/803 [21:07<29:06,  3.76s/it]\u001b[0m\n",
      "\u001b[34m42%|████▏     | 339/803 [21:10<27:08,  3.51s/it]\u001b[0m\n",
      "\u001b[34m42%|████▏     | 340/803 [21:14<27:24,  3.55s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.8626, 'learning_rate': 3.0957320559282015e-05, 'epoch': 0.42}\u001b[0m\n",
      "\u001b[34m42%|████▏     | 340/803 [21:14<27:24,  3.55s/it]\u001b[0m\n",
      "\u001b[34m42%|████▏     | 341/803 [21:17<26:43,  3.47s/it]\u001b[0m\n",
      "\u001b[34m43%|████▎     | 342/803 [21:21<26:57,  3.51s/it]\u001b[0m\n",
      "\u001b[34m43%|████▎     | 343/803 [21:24<26:34,  3.47s/it]\u001b[0m\n",
      "\u001b[34m43%|████▎     | 344/803 [21:28<26:39,  3.48s/it]\u001b[0m\n",
      "\u001b[34m43%|████▎     | 345/803 [21:31<25:58,  3.40s/it]\u001b[0m\n",
      "\u001b[34m43%|████▎     | 346/803 [21:34<24:59,  3.28s/it]\u001b[0m\n",
      "\u001b[34m43%|████▎     | 347/803 [21:37<25:41,  3.38s/it]\u001b[0m\n",
      "\u001b[34m43%|████▎     | 348/803 [21:41<26:41,  3.52s/it]\u001b[0m\n",
      "\u001b[34m43%|████▎     | 349/803 [21:44<25:35,  3.38s/it]\u001b[0m\n",
      "\u001b[34m44%|████▎     | 350/803 [21:49<27:23,  3.63s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.8595, 'learning_rate': 3.0003099533459878e-05, 'epoch': 0.44}\u001b[0m\n",
      "\u001b[34m44%|████▎     | 350/803 [21:49<27:23,  3.63s/it]\u001b[0m\n",
      "\u001b[34m44%|████▎     | 351/803 [21:52<27:36,  3.67s/it]\u001b[0m\n",
      "\u001b[34m44%|████▍     | 352/803 [21:56<27:07,  3.61s/it]\u001b[0m\n",
      "\u001b[34m44%|████▍     | 353/803 [22:00<27:44,  3.70s/it]\u001b[0m\n",
      "\u001b[34m44%|████▍     | 354/803 [22:03<27:52,  3.73s/it]\u001b[0m\n",
      "\u001b[34m44%|████▍     | 355/803 [22:07<28:07,  3.77s/it]\u001b[0m\n",
      "\u001b[34m44%|████▍     | 356/803 [22:10<26:34,  3.57s/it]\u001b[0m\n",
      "\u001b[34m44%|████▍     | 357/803 [22:14<27:24,  3.69s/it]\u001b[0m\n",
      "\u001b[34m45%|████▍     | 358/803 [22:18<26:19,  3.55s/it]\u001b[0m\n",
      "\u001b[34m45%|████▍     | 359/803 [22:21<26:45,  3.62s/it]\u001b[0m\n",
      "\u001b[34m45%|████▍     | 360/803 [22:25<26:28,  3.59s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.805, 'learning_rate': 2.9041221617744296e-05, 'epoch': 0.45}\u001b[0m\n",
      "\u001b[34m45%|████▍     | 360/803 [22:25<26:28,  3.59s/it]\u001b[0m\n",
      "\u001b[34m45%|████▍     | 361/803 [22:29<27:15,  3.70s/it]\u001b[0m\n",
      "\u001b[34m45%|████▌     | 362/803 [22:32<26:49,  3.65s/it]\u001b[0m\n",
      "\u001b[34m45%|████▌     | 363/803 [22:36<26:44,  3.65s/it]\u001b[0m\n",
      "\u001b[34m45%|████▌     | 364/803 [22:40<28:16,  3.86s/it]\u001b[0m\n",
      "\u001b[34m45%|████▌     | 365/803 [22:45<29:46,  4.08s/it]\u001b[0m\n",
      "\u001b[34m46%|████▌     | 366/803 [22:49<28:37,  3.93s/it]\u001b[0m\n",
      "\u001b[34m46%|████▌     | 367/803 [22:52<28:27,  3.92s/it]\u001b[0m\n",
      "\u001b[34m46%|████▌     | 368/803 [22:57<29:07,  4.02s/it]\u001b[0m\n",
      "\u001b[34m46%|████▌     | 369/803 [23:00<27:39,  3.82s/it]\u001b[0m\n",
      "\u001b[34m46%|████▌     | 370/803 [23:03<26:32,  3.68s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.813, 'learning_rate': 2.8073158898237552e-05, 'epoch': 0.46}\u001b[0m\n",
      "\u001b[34m46%|████▌     | 370/803 [23:03<26:32,  3.68s/it]\u001b[0m\n",
      "\u001b[34m46%|████▌     | 371/803 [23:08<27:52,  3.87s/it]\u001b[0m\n",
      "\u001b[34m46%|████▋     | 372/803 [23:11<27:05,  3.77s/it]\u001b[0m\n",
      "\u001b[34m46%|████▋     | 373/803 [23:15<26:34,  3.71s/it]\u001b[0m\n",
      "\u001b[34m47%|████▋     | 374/803 [23:18<24:31,  3.43s/it]\u001b[0m\n",
      "\u001b[34m47%|████▋     | 375/803 [23:21<24:48,  3.48s/it]\u001b[0m\n",
      "\u001b[34m47%|████▋     | 376/803 [23:25<25:37,  3.60s/it]\u001b[0m\n",
      "\u001b[34m47%|████▋     | 377/803 [23:29<25:45,  3.63s/it]\u001b[0m\n",
      "\u001b[34m47%|████▋     | 378/803 [23:33<26:08,  3.69s/it]\u001b[0m\n",
      "\u001b[34m47%|████▋     | 379/803 [23:36<26:00,  3.68s/it]\u001b[0m\n",
      "\u001b[34m47%|████▋     | 380/803 [23:40<26:07,  3.71s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.8139, 'learning_rate': 2.710039292644661e-05, 'epoch': 0.47}\u001b[0m\n",
      "\u001b[34m47%|████▋     | 380/803 [23:40<26:07,  3.71s/it]\u001b[0m\n",
      "\u001b[34m47%|████▋     | 381/803 [23:43<25:11,  3.58s/it]\u001b[0m\n",
      "\u001b[34m48%|████▊     | 382/803 [23:47<24:38,  3.51s/it]\u001b[0m\n",
      "\u001b[34m48%|████▊     | 383/803 [23:50<23:57,  3.42s/it]\u001b[0m\n",
      "\u001b[34m48%|████▊     | 384/803 [23:54<24:25,  3.50s/it]\u001b[0m\n",
      "\u001b[34m48%|████▊     | 385/803 [23:58<25:23,  3.65s/it]\u001b[0m\n",
      "\u001b[34m48%|████▊     | 386/803 [24:01<25:46,  3.71s/it]\u001b[0m\n",
      "\u001b[34m48%|████▊     | 387/803 [24:05<25:01,  3.61s/it]\u001b[0m\n",
      "\u001b[34m48%|████▊     | 388/803 [24:09<25:31,  3.69s/it]\u001b[0m\n",
      "\u001b[34m48%|████▊     | 389/803 [24:13<26:27,  3.83s/it]\u001b[0m\n",
      "\u001b[34m49%|████▊     | 390/803 [24:16<25:34,  3.71s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.805, 'learning_rate': 2.6124412451873293e-05, 'epoch': 0.49}\u001b[0m\n",
      "\u001b[34m49%|████▊     | 390/803 [24:16<25:34,  3.71s/it]\u001b[0m\n",
      "\u001b[34m49%|████▊     | 391/803 [24:19<23:51,  3.47s/it]\u001b[0m\n",
      "\u001b[34m49%|████▉     | 392/803 [24:23<24:35,  3.59s/it]\u001b[0m\n",
      "\u001b[34m49%|████▉     | 393/803 [24:27<24:49,  3.63s/it]\u001b[0m\n",
      "\u001b[34m49%|████▉     | 394/803 [24:30<24:28,  3.59s/it]\u001b[0m\n",
      "\u001b[34m49%|████▉     | 395/803 [24:34<24:36,  3.62s/it]\u001b[0m\n",
      "\u001b[34m49%|████▉     | 396/803 [24:38<25:38,  3.78s/it]\u001b[0m\n",
      "\u001b[34m49%|████▉     | 397/803 [24:41<24:21,  3.60s/it]\u001b[0m\n",
      "\u001b[34m50%|████▉     | 398/803 [24:45<24:42,  3.66s/it]\u001b[0m\n",
      "\u001b[34m50%|████▉     | 399/803 [24:48<23:28,  3.49s/it]\u001b[0m\n",
      "\u001b[34m50%|████▉     | 400/803 [24:51<22:34,  3.36s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.8044, 'learning_rate': 2.514671114358855e-05, 'epoch': 0.5}\u001b[0m\n",
      "\u001b[34m50%|████▉     | 400/803 [24:51<22:34,  3.36s/it]\u001b[0m\n",
      "\u001b[34m50%|████▉     | 401/803 [24:55<23:14,  3.47s/it]\u001b[0m\n",
      "\u001b[34m50%|█████     | 402/803 [24:59<25:03,  3.75s/it]\u001b[0m\n",
      "\u001b[34m50%|█████     | 403/803 [25:03<25:28,  3.82s/it]\u001b[0m\n",
      "\u001b[34m50%|█████     | 404/803 [25:07<24:22,  3.67s/it]\u001b[0m\n",
      "\u001b[34m50%|█████     | 405/803 [25:11<24:54,  3.76s/it]\u001b[0m\n",
      "\u001b[34m51%|█████     | 406/803 [25:15<25:45,  3.89s/it]\u001b[0m\n",
      "\u001b[34m51%|█████     | 407/803 [25:18<25:02,  3.79s/it]\u001b[0m\n",
      "\u001b[34m51%|█████     | 408/803 [25:22<25:09,  3.82s/it]\u001b[0m\n",
      "\u001b[34m51%|█████     | 409/803 [25:25<23:34,  3.59s/it]\u001b[0m\n",
      "\u001b[34m51%|█████     | 410/803 [25:29<24:08,  3.69s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.7668, 'learning_rate': 2.4168785304277562e-05, 'epoch': 0.51}\u001b[0m\n",
      "\u001b[34m51%|█████     | 410/803 [25:29<24:08,  3.69s/it]\u001b[0m\n",
      "\u001b[34m51%|█████     | 411/803 [25:33<23:39,  3.62s/it]\u001b[0m\n",
      "\u001b[34m51%|█████▏    | 412/803 [25:36<22:09,  3.40s/it]\u001b[0m\n",
      "\u001b[34m51%|█████▏    | 413/803 [25:39<21:52,  3.37s/it]\u001b[0m\n",
      "\u001b[34m52%|█████▏    | 414/803 [25:42<20:37,  3.18s/it]\u001b[0m\n",
      "\u001b[34m52%|█████▏    | 415/803 [25:45<20:21,  3.15s/it]\u001b[0m\n",
      "\u001b[34m52%|█████▏    | 416/803 [25:48<20:27,  3.17s/it]\u001b[0m\n",
      "\u001b[34m52%|█████▏    | 417/803 [25:53<23:16,  3.62s/it]\u001b[0m\n",
      "\u001b[34m52%|█████▏    | 418/803 [25:56<23:05,  3.60s/it]\u001b[0m\n",
      "\u001b[34m52%|█████▏    | 419/803 [26:00<22:59,  3.59s/it]\u001b[0m\n",
      "\u001b[34m52%|█████▏    | 420/803 [26:03<22:51,  3.58s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.8519, 'learning_rate': 2.31921315802544e-05, 'epoch': 0.52}\u001b[0m\n",
      "\u001b[34m52%|█████▏    | 420/803 [26:03<22:51,  3.58s/it]\u001b[0m\n",
      "\u001b[34m52%|█████▏    | 421/803 [26:07<22:18,  3.50s/it]\u001b[0m\n",
      "\u001b[34m53%|█████▎    | 422/803 [26:10<22:29,  3.54s/it]\u001b[0m\n",
      "\u001b[34m53%|█████▎    | 423/803 [26:14<22:18,  3.52s/it]\u001b[0m\n",
      "\u001b[34m53%|█████▎    | 424/803 [26:18<24:22,  3.86s/it]\u001b[0m\n",
      "\u001b[34m53%|█████▎    | 425/803 [26:23<24:50,  3.94s/it]\u001b[0m\n",
      "\u001b[34m53%|█████▎    | 426/803 [26:26<23:17,  3.71s/it]\u001b[0m\n",
      "\u001b[34m53%|█████▎    | 427/803 [26:30<23:55,  3.82s/it]\u001b[0m\n",
      "\u001b[34m53%|█████▎    | 428/803 [26:33<23:09,  3.71s/it]\u001b[0m\n",
      "\u001b[34m53%|█████▎    | 429/803 [26:37<22:34,  3.62s/it]\u001b[0m\n",
      "\u001b[34m54%|█████▎    | 430/803 [26:41<23:24,  3.77s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.8192, 'learning_rate': 2.2218244670950638e-05, 'epoch': 0.53}\u001b[0m\n",
      "\u001b[34m54%|█████▎    | 430/803 [26:41<23:24,  3.77s/it]\u001b[0m\n",
      "\u001b[34m54%|█████▎    | 431/803 [26:45<24:11,  3.90s/it]\u001b[0m\n",
      "\u001b[34m54%|█████▍    | 432/803 [26:49<24:21,  3.94s/it]\u001b[0m\n",
      "\u001b[34m54%|█████▍    | 433/803 [26:53<23:32,  3.82s/it]\u001b[0m\n",
      "\u001b[34m54%|█████▍    | 434/803 [26:56<22:25,  3.65s/it]\u001b[0m\n",
      "\u001b[34m54%|█████▍    | 435/803 [26:59<22:06,  3.60s/it]\u001b[0m\n",
      "\u001b[34m54%|█████▍    | 436/803 [27:04<23:24,  3.83s/it]\u001b[0m\n",
      "\u001b[34m54%|█████▍    | 437/803 [27:07<22:55,  3.76s/it]\u001b[0m\n",
      "\u001b[34m55%|█████▍    | 438/803 [27:11<22:02,  3.62s/it]\u001b[0m\n",
      "\u001b[34m55%|█████▍    | 439/803 [27:14<21:15,  3.50s/it]\u001b[0m\n",
      "\u001b[34m55%|█████▍    | 440/803 [27:17<20:52,  3.45s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.8043, 'learning_rate': 2.1248615041383685e-05, 'epoch': 0.55}\u001b[0m\n",
      "\u001b[34m55%|█████▍    | 440/803 [27:17<20:52,  3.45s/it]\u001b[0m\n",
      "\u001b[34m55%|█████▍    | 441/803 [27:22<23:05,  3.83s/it]\u001b[0m\n",
      "\u001b[34m55%|█████▌    | 442/803 [27:25<21:37,  3.60s/it]\u001b[0m\n",
      "\u001b[34m55%|█████▌    | 443/803 [27:30<23:58,  4.00s/it]\u001b[0m\n",
      "\u001b[34m55%|█████▌    | 444/803 [27:34<24:23,  4.08s/it]\u001b[0m\n",
      "\u001b[34m55%|█████▌    | 445/803 [27:38<23:56,  4.01s/it]\u001b[0m\n",
      "\u001b[34m56%|█████▌    | 446/803 [27:41<22:06,  3.72s/it]\u001b[0m\n",
      "\u001b[34m56%|█████▌    | 451/803 [27:59<21:39,  3.69s/it]\u001b[0m\n",
      "\u001b[34m56%|█████▋    | 452/803 [28:02<20:11,  3.45s/it]\u001b[0m\n",
      "\u001b[34m56%|█████▋    | 453/803 [28:05<19:55,  3.42s/it]\u001b[0m\n",
      "\u001b[34m57%|█████▋    | 454/803 [28:09<20:52,  3.59s/it]\u001b[0m\n",
      "\u001b[34m57%|█████▋    | 455/803 [28:13<20:54,  3.61s/it]\u001b[0m\n",
      "\u001b[34m57%|█████▋    | 456/803 [28:17<21:21,  3.69s/it]\u001b[0m\n",
      "\u001b[34m57%|█████▋    | 457/803 [28:20<20:50,  3.61s/it]\u001b[0m\n",
      "\u001b[34m57%|█████▋    | 458/803 [28:24<21:12,  3.69s/it]\u001b[0m\n",
      "\u001b[34m57%|█████▋    | 459/803 [28:28<21:39,  3.78s/it]\u001b[0m\n",
      "\u001b[34m57%|█████▋    | 460/803 [28:31<20:11,  3.53s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.8147, 'learning_rate': 1.932805463312272e-05, 'epoch': 0.57}\u001b[0m\n",
      "\u001b[34m57%|█████▋    | 460/803 [28:31<20:11,  3.53s/it]\u001b[0m\n",
      "\u001b[34m57%|█████▋    | 461/803 [28:35<20:37,  3.62s/it]\u001b[0m\n",
      "\u001b[34m58%|█████▊    | 462/803 [28:39<21:20,  3.76s/it]\u001b[0m\n",
      "\u001b[34m58%|█████▊    | 463/803 [28:42<20:10,  3.56s/it]\u001b[0m\n",
      "\u001b[34m58%|█████▊    | 464/803 [28:47<22:10,  3.92s/it]\u001b[0m\n",
      "\u001b[34m58%|█████▊    | 465/803 [28:51<22:44,  4.04s/it]\u001b[0m\n",
      "\u001b[34m58%|█████▊    | 466/803 [28:55<22:39,  4.03s/it]\u001b[0m\n",
      "\u001b[34m58%|█████▊    | 467/803 [28:58<21:25,  3.83s/it]\u001b[0m\n",
      "\u001b[34m58%|█████▊    | 468/803 [29:01<19:48,  3.55s/it]\u001b[0m\n",
      "\u001b[34m58%|█████▊    | 469/803 [29:05<20:17,  3.64s/it]\u001b[0m\n",
      "\u001b[34m59%|█████▊    | 470/803 [29:09<20:12,  3.64s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.8284, 'learning_rate': 1.838006313626418e-05, 'epoch': 0.58}\u001b[0m\n",
      "\u001b[34m59%|█████▊    | 470/803 [29:09<20:12,  3.64s/it]\u001b[0m\n",
      "\u001b[34m59%|█████▊    | 471/803 [29:12<20:01,  3.62s/it]\u001b[0m\n",
      "\u001b[34m59%|█████▉    | 472/803 [29:17<21:45,  3.94s/it]\u001b[0m\n",
      "\u001b[34m59%|█████▉    | 473/803 [29:20<20:09,  3.67s/it]\u001b[0m\n",
      "\u001b[34m59%|█████▉    | 474/803 [29:24<20:15,  3.69s/it]\u001b[0m\n",
      "\u001b[34m59%|█████▉    | 475/803 [29:27<19:25,  3.55s/it]\u001b[0m\n",
      "\u001b[34m59%|█████▉    | 476/803 [29:32<21:02,  3.86s/it]\u001b[0m\n",
      "\u001b[34m59%|█████▉    | 477/803 [29:35<20:14,  3.72s/it]\u001b[0m\n",
      "\u001b[34m60%|█████▉    | 478/803 [29:38<19:27,  3.59s/it]\u001b[0m\n",
      "\u001b[34m60%|█████▉    | 479/803 [29:43<20:39,  3.82s/it]\u001b[0m\n",
      "\u001b[34m60%|█████▉    | 480/803 [29:46<20:11,  3.75s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.812, 'learning_rate': 1.744220298445049e-05, 'epoch': 0.6}\u001b[0m\n",
      "\u001b[34m60%|█████▉    | 480/803 [29:46<20:11,  3.75s/it]\u001b[0m\n",
      "\u001b[34m60%|█████▉    | 481/803 [29:50<20:06,  3.75s/it]\u001b[0m\n",
      "\u001b[34m60%|██████    | 482/803 [29:53<19:27,  3.64s/it]\u001b[0m\n",
      "\u001b[34m60%|██████    | 483/803 [29:57<18:52,  3.54s/it]\u001b[0m\n",
      "\u001b[34m60%|██████    | 484/803 [30:00<17:56,  3.37s/it]\u001b[0m\n",
      "\u001b[34m60%|██████    | 485/803 [30:04<18:44,  3.54s/it]\u001b[0m\n",
      "\u001b[34m61%|██████    | 486/803 [30:07<18:12,  3.45s/it]\u001b[0m\n",
      "\u001b[34m61%|██████    | 487/803 [30:11<18:39,  3.54s/it]\u001b[0m\n",
      "\u001b[34m61%|██████    | 488/803 [30:14<18:21,  3.50s/it]\u001b[0m\n",
      "\u001b[34m61%|██████    | 489/803 [30:18<19:41,  3.76s/it]\u001b[0m\n",
      "\u001b[34m61%|██████    | 490/803 [30:22<19:19,  3.70s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.731, 'learning_rate': 1.6515909506295398e-05, 'epoch': 0.61}\u001b[0m\n",
      "\u001b[34m61%|██████    | 490/803 [30:22<19:19,  3.70s/it]\u001b[0m\n",
      "\u001b[34m61%|██████    | 491/803 [30:26<20:05,  3.86s/it]\u001b[0m\n",
      "\u001b[34m61%|██████▏   | 492/803 [30:30<19:23,  3.74s/it]\u001b[0m\n",
      "\u001b[34m61%|██████▏   | 493/803 [30:33<18:39,  3.61s/it]\u001b[0m\n",
      "\u001b[34m62%|██████▏   | 494/803 [30:36<17:37,  3.42s/it]\u001b[0m\n",
      "\u001b[34m62%|██████▏   | 495/803 [30:40<18:28,  3.60s/it]\u001b[0m\n",
      "\u001b[34m62%|██████▏   | 496/803 [30:44<19:24,  3.79s/it]\u001b[0m\n",
      "\u001b[34m62%|██████▏   | 497/803 [30:48<19:38,  3.85s/it]\u001b[0m\n",
      "\u001b[34m62%|██████▏   | 498/803 [30:51<18:22,  3.61s/it]\u001b[0m\n",
      "\u001b[34m62%|██████▏   | 499/803 [30:55<17:43,  3.50s/it]\u001b[0m\n",
      "\u001b[34m62%|██████▏   | 500/803 [30:58<17:17,  3.42s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.8419, 'learning_rate': 1.5602600328436922e-05, 'epoch': 0.62}\u001b[0m\n",
      "\u001b[34m62%|██████▏   | 500/803 [30:58<17:17,  3.42s/it]\u001b[0m\n",
      "\u001b[34m62%|██████▏   | 501/803 [31:01<16:35,  3.30s/it]\u001b[0m\n",
      "\u001b[34m63%|██████▎   | 502/803 [31:04<16:59,  3.39s/it]\u001b[0m\n",
      "\u001b[34m63%|██████▎   | 503/803 [31:08<17:41,  3.54s/it]\u001b[0m\n",
      "\u001b[34m63%|██████▎   | 504/803 [31:12<17:40,  3.55s/it]\u001b[0m\n",
      "\u001b[34m63%|██████▎   | 505/803 [31:17<19:38,  3.95s/it]\u001b[0m\n",
      "\u001b[34m63%|██████▎   | 506/803 [31:20<18:05,  3.65s/it]\u001b[0m\n",
      "\u001b[34m63%|██████▎   | 507/803 [31:23<17:23,  3.53s/it]\u001b[0m\n",
      "\u001b[34m63%|██████▎   | 508/803 [31:27<18:29,  3.76s/it]\u001b[0m\n",
      "\u001b[34m63%|██████▎   | 509/803 [31:31<18:12,  3.71s/it]\u001b[0m\n",
      "\u001b[34m64%|██████▎   | 510/803 [31:35<19:17,  3.95s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.8782, 'learning_rate': 1.470367320596005e-05, 'epoch': 0.63}\u001b[0m\n",
      "\u001b[34m64%|██████▎   | 510/803 [31:35<19:17,  3.95s/it]\u001b[0m\n",
      "\u001b[34m64%|██████▎   | 511/803 [31:39<19:26,  3.99s/it]\u001b[0m\n",
      "\u001b[34m64%|██████▍   | 512/803 [31:44<20:32,  4.24s/it]\u001b[0m\n",
      "\u001b[34m64%|██████▍   | 513/803 [31:48<19:24,  4.02s/it]\u001b[0m\n",
      "\u001b[34m64%|██████▍   | 514/803 [31:52<19:09,  3.98s/it]\u001b[0m\n",
      "\u001b[34m64%|██████▍   | 515/803 [31:55<18:04,  3.77s/it]\u001b[0m\n",
      "\u001b[34m64%|██████▍   | 516/803 [31:58<17:12,  3.60s/it]\u001b[0m\n",
      "\u001b[34m64%|██████▍   | 517/803 [32:02<17:37,  3.70s/it]\u001b[0m\n",
      "\u001b[34m65%|██████▍   | 518/803 [32:06<17:25,  3.67s/it]\u001b[0m\n",
      "\u001b[34m65%|██████▍   | 519/803 [32:09<17:33,  3.71s/it]\u001b[0m\n",
      "\u001b[34m65%|██████▍   | 520/803 [32:13<17:19,  3.67s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.7871, 'learning_rate': 1.3820503883231445e-05, 'epoch': 0.65}\u001b[0m\n",
      "\u001b[34m65%|██████▍   | 520/803 [32:13<17:19,  3.67s/it]\u001b[0m\n",
      "\u001b[34m65%|██████▍   | 521/803 [32:17<16:57,  3.61s/it]\u001b[0m\n",
      "\u001b[34m65%|██████▌   | 522/803 [32:20<16:42,  3.57s/it]\u001b[0m\n",
      "\u001b[34m65%|██████▌   | 523/803 [32:24<17:18,  3.71s/it]\u001b[0m\n",
      "\u001b[34m65%|██████▌   | 524/803 [32:28<17:53,  3.85s/it]\u001b[0m\n",
      "\u001b[34m65%|██████▌   | 525/803 [32:31<16:48,  3.63s/it]\u001b[0m\n",
      "\u001b[34m66%|██████▌   | 526/803 [32:35<16:29,  3.57s/it]\u001b[0m\n",
      "\u001b[34m66%|██████▌   | 527/803 [32:38<16:24,  3.57s/it]\u001b[0m\n",
      "\u001b[34m66%|██████▌   | 528/803 [32:43<17:17,  3.77s/it]\u001b[0m\n",
      "\u001b[34m66%|██████▌   | 529/803 [32:46<16:34,  3.63s/it]\u001b[0m\n",
      "\u001b[34m66%|██████▌   | 530/803 [32:51<18:01,  3.96s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.7894, 'learning_rate': 1.2954443988420022e-05, 'epoch': 0.66}\u001b[0m\n",
      "\u001b[34m66%|██████▌   | 530/803 [32:51<18:01,  3.96s/it]\u001b[0m\n",
      "\u001b[34m66%|██████▌   | 531/803 [32:55<17:53,  3.95s/it]\u001b[0m\n",
      "\u001b[34m66%|██████▋   | 532/803 [33:00<20:24,  4.52s/it]\u001b[0m\n",
      "\u001b[34m66%|██████▋   | 533/803 [33:05<20:55,  4.65s/it]\u001b[0m\n",
      "\u001b[34m67%|██████▋   | 534/803 [33:11<22:05,  4.93s/it]\u001b[0m\n",
      "\u001b[34m67%|██████▋   | 535/803 [33:14<20:01,  4.48s/it]\u001b[0m\n",
      "\u001b[34m67%|██████▋   | 536/803 [33:18<19:00,  4.27s/it]\u001b[0m\n",
      "\u001b[34m67%|██████▋   | 537/803 [33:22<17:49,  4.02s/it]\u001b[0m\n",
      "\u001b[34m67%|██████▋   | 538/803 [33:25<16:39,  3.77s/it]\u001b[0m\n",
      "\u001b[34m67%|██████▋   | 539/803 [33:29<16:51,  3.83s/it]\u001b[0m\n",
      "\u001b[34m67%|██████▋   | 540/803 [33:33<17:21,  3.96s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.7768, 'learning_rate': 1.2106818964925708e-05, 'epoch': 0.67}\u001b[0m\n",
      "\u001b[34m67%|██████▋   | 540/803 [33:33<17:21,  3.96s/it]\u001b[0m\n",
      "\u001b[34m67%|██████▋   | 541/803 [33:37<17:05,  3.91s/it]\u001b[0m\n",
      "\u001b[34m67%|██████▋   | 542/803 [33:40<16:11,  3.72s/it]\u001b[0m\n",
      "\u001b[34m68%|██████▊   | 543/803 [33:43<15:24,  3.56s/it]\u001b[0m\n",
      "\u001b[34m68%|██████▊   | 544/803 [33:47<15:57,  3.70s/it]\u001b[0m\n",
      "\u001b[34m68%|██████▊   | 545/803 [33:50<14:46,  3.44s/it]\u001b[0m\n",
      "\u001b[34m68%|██████▊   | 546/803 [33:54<15:30,  3.62s/it]\u001b[0m\n",
      "\u001b[34m68%|██████▊   | 547/803 [33:58<16:19,  3.83s/it]\u001b[0m\n",
      "\u001b[34m68%|██████▊   | 548/803 [34:02<16:02,  3.77s/it]\u001b[0m\n",
      "\u001b[34m68%|██████▊   | 549/803 [34:06<16:18,  3.85s/it]\u001b[0m\n",
      "\u001b[34m68%|██████▊   | 550/803 [34:11<17:22,  4.12s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.8392, 'learning_rate': 1.1278926042882026e-05, 'epoch': 0.68}\u001b[0m\n",
      "\u001b[34m68%|██████▊   | 550/803 [34:11<17:22,  4.12s/it]\u001b[0m\n",
      "\u001b[34m69%|██████▊   | 551/803 [34:15<16:52,  4.02s/it]\u001b[0m\n",
      "\u001b[34m69%|██████▊   | 552/803 [34:18<16:33,  3.96s/it]\u001b[0m\n",
      "\u001b[34m69%|██████▉   | 553/803 [34:22<15:38,  3.75s/it]\u001b[0m\n",
      "\u001b[34m69%|██████▉   | 554/803 [34:26<15:51,  3.82s/it]\u001b[0m\n",
      "\u001b[34m69%|██████▉   | 555/803 [34:29<15:17,  3.70s/it]\u001b[0m\n",
      "\u001b[34m69%|██████▉   | 556/803 [34:32<14:03,  3.42s/it]\u001b[0m\n",
      "\u001b[34m69%|██████▉   | 557/803 [34:35<13:43,  3.35s/it]\u001b[0m\n",
      "\u001b[34m69%|██████▉   | 558/803 [34:39<14:02,  3.44s/it]\u001b[0m\n",
      "\u001b[34m70%|██████▉   | 559/803 [34:42<13:34,  3.34s/it]\u001b[0m\n",
      "\u001b[34m70%|██████▉   | 560/803 [34:45<13:13,  3.27s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.805, 'learning_rate': 1.047203225383715e-05, 'epoch': 0.7}\u001b[0m\n",
      "\u001b[34m70%|██████▉   | 560/803 [34:45<13:13,  3.27s/it]\u001b[0m\n",
      "\u001b[34m70%|██████▉   | 561/803 [34:48<12:52,  3.19s/it]\u001b[0m\n",
      "\u001b[34m70%|██████▉   | 562/803 [34:51<12:57,  3.23s/it]\u001b[0m\n",
      "\u001b[34m70%|███████   | 563/803 [34:55<13:26,  3.36s/it]\u001b[0m\n",
      "\u001b[34m70%|███████   | 564/803 [34:59<14:00,  3.52s/it]\u001b[0m\n",
      "\u001b[34m70%|███████   | 565/803 [35:02<13:43,  3.46s/it]\u001b[0m\n",
      "\u001b[34m70%|███████   | 566/803 [35:06<13:42,  3.47s/it]\u001b[0m\n",
      "\u001b[34m71%|███████   | 567/803 [35:10<14:38,  3.72s/it]\u001b[0m\n",
      "\u001b[34m71%|███████   | 568/803 [35:13<14:06,  3.60s/it]\u001b[0m\n",
      "\u001b[34m71%|███████   | 569/803 [35:17<14:32,  3.73s/it]\u001b[0m\n",
      "\u001b[34m71%|███████   | 570/803 [35:21<14:11,  3.66s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.8357, 'learning_rate': 9.687372491651824e-06, 'epoch': 0.71}\u001b[0m\n",
      "\u001b[34m71%|███████   | 570/803 [35:21<14:11,  3.66s/it]\u001b[0m\n",
      "\u001b[34m71%|███████   | 571/803 [35:24<13:37,  3.52s/it]\u001b[0m\n",
      "\u001b[34m71%|███████   | 572/803 [35:28<14:04,  3.66s/it]\u001b[0m\n",
      "\u001b[34m71%|███████▏  | 573/803 [35:32<14:32,  3.80s/it]\u001b[0m\n",
      "\u001b[34m71%|███████▏  | 574/803 [35:36<15:04,  3.95s/it]\u001b[0m\n",
      "\u001b[34m72%|███████▏  | 575/803 [35:40<15:03,  3.96s/it]\u001b[0m\n",
      "\u001b[34m72%|███████▏  | 576/803 [35:45<15:19,  4.05s/it]\u001b[0m\n",
      "\u001b[34m72%|███████▏  | 577/803 [35:48<14:21,  3.81s/it]\u001b[0m\n",
      "\u001b[34m72%|███████▏  | 578/803 [35:51<13:51,  3.70s/it]\u001b[0m\n",
      "\u001b[34m72%|███████▏  | 579/803 [35:55<13:32,  3.63s/it]\u001b[0m\n",
      "\u001b[34m72%|███████▏  | 580/803 [35:59<13:41,  3.69s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.7484, 'learning_rate': 8.926147622581574e-06, 'epoch': 0.72}\u001b[0m\n",
      "\u001b[34m72%|███████▏  | 580/803 [35:59<13:41,  3.69s/it]\u001b[0m\n",
      "\u001b[34m72%|███████▏  | 581/803 [36:02<13:05,  3.54s/it]\u001b[0m\n",
      "\u001b[34m72%|███████▏  | 582/803 [36:07<14:30,  3.94s/it]\u001b[0m\n",
      "\u001b[34m73%|███████▎  | 583/803 [36:12<15:32,  4.24s/it]\u001b[0m\n",
      "\u001b[34m73%|███████▎  | 584/803 [36:15<14:52,  4.08s/it]\u001b[0m\n",
      "\u001b[34m73%|███████▎  | 585/803 [36:19<14:37,  4.02s/it]\u001b[0m\n",
      "\u001b[34m73%|███████▎  | 586/803 [36:22<13:36,  3.76s/it]\u001b[0m\n",
      "\u001b[34m73%|███████▎  | 587/803 [36:26<12:55,  3.59s/it]\u001b[0m\n",
      "\u001b[34m73%|███████▎  | 588/803 [36:29<12:27,  3.47s/it]\u001b[0m\n",
      "\u001b[34m73%|███████▎  | 589/803 [36:33<13:04,  3.66s/it]\u001b[0m\n",
      "\u001b[34m73%|███████▎  | 590/803 [36:39<15:59,  4.51s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.8445, 'learning_rate': 8.189522647435932e-06, 'epoch': 0.73}\u001b[0m\n",
      "\u001b[34m73%|███████▎  | 590/803 [36:39<15:59,  4.51s/it]\u001b[0m\n",
      "\u001b[34m74%|███████▎  | 591/803 [36:44<15:37,  4.42s/it]\u001b[0m\n",
      "\u001b[34m74%|███████▎  | 592/803 [36:47<14:44,  4.19s/it]\u001b[0m\n",
      "\u001b[34m74%|███████▍  | 593/803 [36:51<13:54,  3.97s/it]\u001b[0m\n",
      "\u001b[34m74%|███████▍  | 594/803 [36:54<13:00,  3.73s/it]\u001b[0m\n",
      "\u001b[34m74%|███████▍  | 595/803 [36:57<12:19,  3.56s/it]\u001b[0m\n",
      "\u001b[34m74%|███████▍  | 596/803 [37:01<13:01,  3.77s/it]\u001b[0m\n",
      "\u001b[34m74%|███████▍  | 597/803 [37:06<14:14,  4.15s/it]\u001b[0m\n",
      "\u001b[34m74%|███████▍  | 598/803 [37:10<13:21,  3.91s/it]\u001b[0m\n",
      "\u001b[34m75%|███████▍  | 599/803 [37:14<13:30,  3.98s/it]\u001b[0m\n",
      "\u001b[34m75%|███████▍  | 600/803 [37:19<14:31,  4.29s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.7783, 'learning_rate': 7.478624918627084e-06, 'epoch': 0.75}\u001b[0m\n",
      "\u001b[34m75%|███████▍  | 600/803 [37:19<14:31,  4.29s/it]\u001b[0m\n",
      "\u001b[34m75%|███████▍  | 601/803 [37:23<13:56,  4.14s/it]\u001b[0m\n",
      "\u001b[34m75%|███████▍  | 602/803 [37:26<12:56,  3.86s/it]\u001b[0m\n",
      "\u001b[34m75%|███████▌  | 603/803 [37:29<12:29,  3.75s/it]\u001b[0m\n",
      "\u001b[34m75%|███████▌  | 604/803 [37:33<11:59,  3.62s/it]\u001b[0m\n",
      "\u001b[34m75%|███████▌  | 605/803 [37:36<11:45,  3.56s/it]\u001b[0m\n",
      "\u001b[34m75%|███████▌  | 606/803 [37:40<12:23,  3.77s/it]\u001b[0m\n",
      "\u001b[34m76%|███████▌  | 607/803 [37:44<12:06,  3.71s/it]\u001b[0m\n",
      "\u001b[34m76%|███████▌  | 608/803 [37:48<12:53,  3.97s/it]\u001b[0m\n",
      "\u001b[34m76%|███████▌  | 609/803 [37:52<12:50,  3.97s/it]\u001b[0m\n",
      "\u001b[34m76%|███████▌  | 610/803 [37:55<11:51,  3.69s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.824, 'learning_rate': 6.794542414836763e-06, 'epoch': 0.76}\u001b[0m\n",
      "\u001b[34m76%|███████▌  | 610/803 [37:55<11:51,  3.69s/it]\u001b[0m\n",
      "\u001b[34m76%|███████▌  | 611/803 [38:00<12:16,  3.84s/it]\u001b[0m\n",
      "\u001b[34m76%|███████▌  | 612/803 [38:03<11:45,  3.69s/it]\u001b[0m\n",
      "\u001b[34m76%|███████▋  | 613/803 [38:08<12:49,  4.05s/it]\u001b[0m\n",
      "\u001b[34m76%|███████▋  | 614/803 [38:12<12:22,  3.93s/it]\u001b[0m\n",
      "\u001b[34m77%|███████▋  | 615/803 [38:15<12:00,  3.83s/it]\u001b[0m\n",
      "\u001b[34m77%|███████▋  | 616/803 [38:20<12:33,  4.03s/it]\u001b[0m\n",
      "\u001b[34m77%|███████▋  | 617/803 [38:23<12:05,  3.90s/it]\u001b[0m\n",
      "\u001b[34m77%|███████▋  | 618/803 [38:27<11:48,  3.83s/it]\u001b[0m\n",
      "\u001b[34m77%|███████▋  | 619/803 [38:31<12:05,  3.94s/it]\u001b[0m\n",
      "\u001b[34m77%|███████▋  | 620/803 [38:35<11:52,  3.89s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.757, 'learning_rate': 6.138322075941852e-06, 'epoch': 0.77}\u001b[0m\n",
      "\u001b[34m77%|███████▋  | 620/803 [38:35<11:52,  3.89s/it]\u001b[0m\n",
      "\u001b[34m77%|███████▋  | 621/803 [38:39<11:59,  3.95s/it]\u001b[0m\n",
      "\u001b[34m77%|███████▋  | 622/803 [38:43<12:11,  4.04s/it]\u001b[0m\n",
      "\u001b[34m78%|███████▊  | 623/803 [38:46<11:14,  3.75s/it]\u001b[0m\n",
      "\u001b[34m78%|███████▊  | 624/803 [38:50<11:23,  3.82s/it]\u001b[0m\n",
      "\u001b[34m78%|███████▊  | 625/803 [38:54<11:11,  3.77s/it]\u001b[0m\n",
      "\u001b[34m78%|███████▊  | 626/803 [38:57<10:39,  3.61s/it]\u001b[0m\n",
      "\u001b[34m78%|███████▊  | 627/803 [39:01<10:43,  3.66s/it]\u001b[0m\n",
      "\u001b[34m78%|███████▊  | 628/803 [39:05<10:59,  3.77s/it]\u001b[0m\n",
      "\u001b[34m78%|███████▊  | 629/803 [39:09<10:56,  3.77s/it]\u001b[0m\n",
      "\u001b[34m78%|███████▊  | 630/803 [39:12<10:43,  3.72s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.8156, 'learning_rate': 5.51096820074693e-06, 'epoch': 0.78}\u001b[0m\n",
      "\u001b[34m78%|███████▊  | 630/803 [39:12<10:43,  3.72s/it]\u001b[0m\n",
      "\u001b[34m79%|███████▊  | 631/803 [39:17<11:13,  3.91s/it]\u001b[0m\n",
      "\u001b[34m79%|███████▊  | 632/803 [39:20<10:39,  3.74s/it]\u001b[0m\n",
      "\u001b[34m79%|███████▉  | 633/803 [39:24<10:51,  3.83s/it]\u001b[0m\n",
      "\u001b[34m79%|███████▉  | 634/803 [39:29<11:48,  4.19s/it]\u001b[0m\n",
      "\u001b[34m79%|███████▉  | 635/803 [39:33<11:22,  4.06s/it]\u001b[0m\n",
      "\u001b[34m79%|███████▉  | 636/803 [39:36<10:52,  3.91s/it]\u001b[0m\n",
      "\u001b[34m79%|███████▉  | 637/803 [39:39<09:51,  3.56s/it]\u001b[0m\n",
      "\u001b[34m79%|███████▉  | 638/803 [39:43<09:39,  3.51s/it]\u001b[0m\n",
      "\u001b[34m80%|███████▉  | 639/803 [39:46<09:26,  3.46s/it]\u001b[0m\n",
      "\u001b[34m80%|███████▉  | 640/803 [39:49<09:27,  3.48s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.8307, 'learning_rate': 4.913440909976083e-06, 'epoch': 0.8}\u001b[0m\n",
      "\u001b[34m80%|███████▉  | 640/803 [39:49<09:27,  3.48s/it]\u001b[0m\n",
      "\u001b[34m80%|███████▉  | 641/803 [39:53<09:13,  3.42s/it]\u001b[0m\n",
      "\u001b[34m80%|███████▉  | 642/803 [39:57<09:37,  3.59s/it]\u001b[0m\n",
      "\u001b[34m80%|████████  | 643/803 [40:01<10:11,  3.82s/it]\u001b[0m\n",
      "\u001b[34m80%|████████  | 644/803 [40:04<09:32,  3.60s/it]\u001b[0m\n",
      "\u001b[34m80%|████████  | 645/803 [40:09<10:12,  3.88s/it]\u001b[0m\n",
      "\u001b[34m80%|████████  | 646/803 [40:13<10:20,  3.96s/it]\u001b[0m\n",
      "\u001b[34m81%|████████  | 647/803 [40:17<10:41,  4.11s/it]\u001b[0m\n",
      "\u001b[34m81%|████████  | 648/803 [40:21<10:38,  4.12s/it]\u001b[0m\n",
      "\u001b[34m81%|████████  | 649/803 [40:26<11:15,  4.38s/it]\u001b[0m\n",
      "\u001b[34m81%|████████  | 650/803 [40:31<11:15,  4.42s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.7538, 'learning_rate': 4.3466546768759555e-06, 'epoch': 0.81}\u001b[0m\n",
      "\u001b[34m81%|████████  | 650/803 [40:31<11:15,  4.42s/it]\u001b[0m\n",
      "\u001b[34m81%|████████  | 651/803 [40:34<10:13,  4.04s/it]\u001b[0m\n",
      "\u001b[34m81%|████████  | 652/803 [40:38<10:05,  4.01s/it]\u001b[0m\n",
      "\u001b[34m81%|████████▏ | 653/803 [40:42<10:22,  4.15s/it]\u001b[0m\n",
      "\u001b[34m81%|████████▏ | 654/803 [40:45<09:18,  3.75s/it]\u001b[0m\n",
      "\u001b[34m82%|████████▏ | 655/803 [40:50<10:16,  4.17s/it]\u001b[0m\n",
      "\u001b[34m82%|████████▏ | 656/803 [40:53<09:12,  3.76s/it]\u001b[0m\n",
      "\u001b[34m82%|████████▏ | 657/803 [40:57<09:24,  3.87s/it]\u001b[0m\n",
      "\u001b[34m82%|████████▏ | 658/803 [41:01<08:53,  3.68s/it]\u001b[0m\n",
      "\u001b[34m82%|████████▏ | 659/803 [41:04<08:45,  3.65s/it]\u001b[0m\n",
      "\u001b[34m82%|████████▏ | 660/803 [41:08<08:34,  3.60s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.805, 'learning_rate': 3.8114769276792278e-06, 'epoch': 0.82}\u001b[0m\n",
      "\u001b[34m82%|████████▏ | 660/803 [41:08<08:34,  3.60s/it]\u001b[0m\n",
      "\u001b[34m82%|████████▏ | 661/803 [41:11<08:34,  3.62s/it]\u001b[0m\n",
      "\u001b[34m82%|████████▏ | 662/803 [41:14<08:04,  3.44s/it]\u001b[0m\n",
      "\u001b[34m83%|████████▎ | 663/803 [41:18<08:19,  3.57s/it]\u001b[0m\n",
      "\u001b[34m83%|████████▎ | 664/803 [41:22<08:10,  3.53s/it]\u001b[0m\n",
      "\u001b[34m83%|████████▎ | 665/803 [41:26<08:21,  3.63s/it]\u001b[0m\n",
      "\u001b[34m83%|████████▎ | 666/803 [41:29<07:55,  3.47s/it]\u001b[0m\n",
      "\u001b[34m83%|████████▎ | 667/803 [41:33<08:24,  3.71s/it]\u001b[0m\n",
      "\u001b[34m83%|████████▎ | 668/803 [41:36<08:13,  3.66s/it]\u001b[0m\n",
      "\u001b[34m83%|████████▎ | 669/803 [41:41<08:42,  3.90s/it]\u001b[0m\n",
      "\u001b[34m83%|████████▎ | 670/803 [41:44<08:14,  3.72s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.7826, 'learning_rate': 3.3087267140700677e-06, 'epoch': 0.83}\u001b[0m\n",
      "\u001b[34m83%|████████▎ | 670/803 [41:44<08:14,  3.72s/it]\u001b[0m\n",
      "\u001b[34m84%|████████▎ | 671/803 [41:48<07:57,  3.61s/it]\u001b[0m\n",
      "\u001b[34m84%|████████▎ | 672/803 [41:51<07:30,  3.44s/it]\u001b[0m\n",
      "\u001b[34m84%|████████▍ | 673/803 [41:54<07:40,  3.54s/it]\u001b[0m\n",
      "\u001b[34m84%|████████▍ | 674/803 [41:58<07:46,  3.61s/it]\u001b[0m\n",
      "\u001b[34m84%|████████▍ | 675/803 [42:01<07:29,  3.51s/it]\u001b[0m\n",
      "\u001b[34m84%|████████▍ | 676/803 [42:05<07:37,  3.61s/it]\u001b[0m\n",
      "\u001b[34m84%|████████▍ | 677/803 [42:09<07:24,  3.53s/it]\u001b[0m\n",
      "\u001b[34m84%|████████▍ | 678/803 [42:13<07:41,  3.69s/it]\u001b[0m\n",
      "\u001b[34m85%|████████▍ | 679/803 [42:16<07:16,  3.52s/it]\u001b[0m\n",
      "\u001b[34m85%|████████▍ | 680/803 [42:20<07:28,  3.65s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.8462, 'learning_rate': 2.8391734596835334e-06, 'epoch': 0.85}\u001b[0m\n",
      "\u001b[34m85%|████████▍ | 680/803 [42:20<07:28,  3.65s/it]\u001b[0m\n",
      "\u001b[34m85%|████████▍ | 681/803 [42:23<07:24,  3.64s/it]\u001b[0m\n",
      "\u001b[34m85%|████████▍ | 682/803 [42:26<07:02,  3.49s/it]\u001b[0m\n",
      "\u001b[34m85%|████████▌ | 683/803 [42:30<06:43,  3.36s/it]\u001b[0m\n",
      "\u001b[34m85%|████████▌ | 684/803 [42:34<07:04,  3.57s/it]\u001b[0m\n",
      "\u001b[34m85%|████████▌ | 685/803 [42:37<07:08,  3.63s/it]\u001b[0m\n",
      "\u001b[34m85%|████████▌ | 686/803 [42:41<07:22,  3.78s/it]\u001b[0m\n",
      "\u001b[34m86%|████████▌ | 687/803 [42:45<07:04,  3.66s/it]\u001b[0m\n",
      "\u001b[34m86%|████████▌ | 688/803 [42:48<06:47,  3.54s/it]\u001b[0m\n",
      "\u001b[34m86%|████████▌ | 689/803 [42:52<07:04,  3.72s/it]\u001b[0m\n",
      "\u001b[34m86%|████████▌ | 690/803 [42:55<06:39,  3.54s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.7703, 'learning_rate': 2.403535782557084e-06, 'epoch': 0.86}\u001b[0m\n",
      "\u001b[34m86%|████████▌ | 690/803 [42:55<06:39,  3.54s/it]\u001b[0m\n",
      "\u001b[34m86%|████████▌ | 691/803 [42:59<06:33,  3.51s/it]\u001b[0m\n",
      "\u001b[34m86%|████████▌ | 692/803 [43:02<06:12,  3.36s/it]\u001b[0m\n",
      "\u001b[34m86%|████████▋ | 693/803 [43:06<06:20,  3.46s/it]\u001b[0m\n",
      "\u001b[34m86%|████████▋ | 694/803 [43:09<06:23,  3.52s/it]\u001b[0m\n",
      "\u001b[34m87%|████████▋ | 695/803 [43:13<06:18,  3.50s/it]\u001b[0m\n",
      "\u001b[34m87%|████████▋ | 696/803 [43:16<06:10,  3.46s/it]\u001b[0m\n",
      "\u001b[34m87%|████████▋ | 697/803 [43:20<06:12,  3.52s/it]\u001b[0m\n",
      "\u001b[34m87%|████████▋ | 698/803 [43:24<06:42,  3.83s/it]\u001b[0m\n",
      "\u001b[34m87%|████████▋ | 699/803 [43:28<06:40,  3.85s/it]\u001b[0m\n",
      "\u001b[34m87%|████████▋ | 700/803 [43:31<06:16,  3.65s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.8767, 'learning_rate': 2.0024803953365173e-06, 'epoch': 0.87}\u001b[0m\n",
      "\u001b[34m87%|████████▋ | 700/803 [43:31<06:16,  3.65s/it]\u001b[0m\n",
      "\u001b[34m87%|████████▋ | 701/803 [43:36<06:55,  4.08s/it]\u001b[0m\n",
      "\u001b[34m87%|████████▋ | 702/803 [43:40<06:37,  3.94s/it]\u001b[0m\n",
      "\u001b[34m88%|████████▊ | 703/803 [43:45<06:59,  4.19s/it]\u001b[0m\n",
      "\u001b[34m88%|████████▊ | 704/803 [43:48<06:27,  3.91s/it]\u001b[0m\n",
      "\u001b[34m88%|████████▊ | 705/803 [43:52<06:17,  3.85s/it]\u001b[0m\n",
      "\u001b[34m88%|████████▊ | 706/803 [43:56<06:11,  3.83s/it]\u001b[0m\n",
      "\u001b[34m88%|████████▊ | 707/803 [43:59<06:07,  3.82s/it]\u001b[0m\n",
      "\u001b[34m88%|████████▊ | 708/803 [44:04<06:20,  4.01s/it]\u001b[0m\n",
      "\u001b[34m88%|████████▊ | 709/803 [44:07<06:00,  3.84s/it]\u001b[0m\n",
      "\u001b[34m88%|████████▊ | 710/803 [44:13<06:40,  4.31s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.8553, 'learning_rate': 1.636621084919454e-06, 'epoch': 0.88}\u001b[0m\n",
      "\u001b[34m88%|████████▊ | 710/803 [44:13<06:40,  4.31s/it]\u001b[0m\n",
      "\u001b[34m89%|████████▊ | 711/803 [44:16<06:00,  3.92s/it]\u001b[0m\n",
      "\u001b[34m89%|████████▊ | 712/803 [44:20<06:15,  4.13s/it]\u001b[0m\n",
      "\u001b[34m89%|████████▉ | 713/803 [44:24<05:58,  3.98s/it]\u001b[0m\n",
      "\u001b[34m89%|████████▉ | 714/803 [44:28<05:44,  3.87s/it]\u001b[0m\n",
      "\u001b[34m89%|████████▉ | 715/803 [44:32<05:43,  3.91s/it]\u001b[0m\n",
      "\u001b[34m89%|████████▉ | 716/803 [44:35<05:41,  3.92s/it]\u001b[0m\n",
      "\u001b[34m89%|████████▉ | 717/803 [44:39<05:18,  3.70s/it]\u001b[0m\n",
      "\u001b[34m89%|████████▉ | 718/803 [44:42<05:04,  3.59s/it]\u001b[0m\n",
      "\u001b[34m90%|████████▉ | 719/803 [44:45<04:57,  3.54s/it]\u001b[0m\n",
      "\u001b[34m90%|████████▉ | 720/803 [44:49<04:53,  3.53s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.7836, 'learning_rate': 1.306517773097904e-06, 'epoch': 0.9}\u001b[0m\n",
      "\u001b[34m90%|████████▉ | 720/803 [44:49<04:53,  3.53s/it]\u001b[0m\n",
      "\u001b[34m90%|████████▉ | 721/803 [44:53<05:12,  3.81s/it]\u001b[0m\n",
      "\u001b[34m90%|████████▉ | 722/803 [44:57<05:06,  3.79s/it]\u001b[0m\n",
      "\u001b[34m90%|█████████ | 723/803 [45:00<04:41,  3.52s/it]\u001b[0m\n",
      "\u001b[34m90%|█████████ | 724/803 [45:04<04:38,  3.52s/it]\u001b[0m\n",
      "\u001b[34m90%|█████████ | 725/803 [45:07<04:27,  3.43s/it]\u001b[0m\n",
      "\u001b[34m90%|█████████ | 726/803 [45:12<04:57,  3.87s/it]\u001b[0m\n",
      "\u001b[34m91%|█████████ | 727/803 [45:15<04:33,  3.61s/it]\u001b[0m\n",
      "\u001b[34m91%|█████████ | 728/803 [45:18<04:21,  3.49s/it]\u001b[0m\n",
      "\u001b[34m91%|█████████ | 729/803 [45:21<04:11,  3.39s/it]\u001b[0m\n",
      "\u001b[34m91%|█████████ | 730/803 [45:24<04:07,  3.39s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.8503, 'learning_rate': 1.0126756596375686e-06, 'epoch': 0.91}\u001b[0m\n",
      "\u001b[34m91%|█████████ | 730/803 [45:24<04:07,  3.39s/it]\u001b[0m\n",
      "\u001b[34m91%|█████████ | 731/803 [45:28<03:57,  3.30s/it]\u001b[0m\n",
      "\u001b[34m91%|█████████ | 732/803 [45:31<03:57,  3.34s/it]\u001b[0m\n",
      "\u001b[34m91%|█████████▏| 733/803 [45:35<04:01,  3.45s/it]\u001b[0m\n",
      "\u001b[34m91%|█████████▏| 734/803 [45:39<04:07,  3.59s/it]\u001b[0m\n",
      "\u001b[34m92%|█████████▏| 735/803 [45:42<04:00,  3.53s/it]\u001b[0m\n",
      "\u001b[34m92%|█████████▏| 736/803 [45:46<04:16,  3.83s/it]\u001b[0m\n",
      "\u001b[34m92%|█████████▏| 737/803 [45:50<04:15,  3.87s/it]\u001b[0m\n",
      "\u001b[34m92%|█████████▏| 738/803 [45:54<04:13,  3.90s/it]\u001b[0m\n",
      "\u001b[34m92%|█████████▏| 739/803 [45:58<03:57,  3.71s/it]\u001b[0m\n",
      "\u001b[34m92%|█████████▏| 740/803 [46:01<03:51,  3.67s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.8268, 'learning_rate': 7.555444491053304e-07, 'epoch': 0.92}\u001b[0m\n",
      "\u001b[34m92%|█████████▏| 740/803 [46:01<03:51,  3.67s/it]\u001b[0m\n",
      "\u001b[34m92%|█████████▏| 741/803 [46:05<03:39,  3.55s/it]\u001b[0m\n",
      "\u001b[34m92%|█████████▏| 742/803 [46:07<03:23,  3.33s/it]\u001b[0m\n",
      "\u001b[34m93%|█████████▎| 743/803 [46:11<03:29,  3.49s/it]\u001b[0m\n",
      "\u001b[34m93%|█████████▎| 744/803 [46:14<03:20,  3.39s/it]\u001b[0m\n",
      "\u001b[34m93%|█████████▎| 745/803 [46:18<03:13,  3.34s/it]\u001b[0m\n",
      "\u001b[34m93%|█████████▎| 746/803 [46:21<03:13,  3.40s/it]\u001b[0m\n",
      "\u001b[34m93%|█████████▎| 747/803 [46:25<03:13,  3.46s/it]\u001b[0m\n",
      "\u001b[34m93%|█████████▎| 748/803 [46:28<03:09,  3.44s/it]\u001b[0m\n",
      "\u001b[34m93%|█████████▎| 749/803 [46:31<03:02,  3.38s/it]\u001b[0m\n",
      "\u001b[34m93%|█████████▎| 750/803 [46:35<03:00,  3.41s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.7819, 'learning_rate': 5.355176626282682e-07, 'epoch': 0.93}\u001b[0m\n",
      "\u001b[34m93%|█████████▎| 750/803 [46:35<03:00,  3.41s/it]\u001b[0m\n",
      "\u001b[34m94%|█████████▎| 751/803 [46:38<02:59,  3.45s/it]\u001b[0m\n",
      "\u001b[34m94%|█████████▎| 752/803 [46:44<03:26,  4.05s/it]\u001b[0m\n",
      "\u001b[34m94%|█████████▍| 753/803 [46:47<03:03,  3.68s/it]\u001b[0m\n",
      "\u001b[34m94%|█████████▍| 754/803 [46:50<02:53,  3.55s/it]\u001b[0m\n",
      "\u001b[34m94%|█████████▍| 755/803 [46:54<02:56,  3.68s/it]\u001b[0m\n",
      "\u001b[34m94%|█████████▍| 756/803 [46:57<02:49,  3.60s/it]\u001b[0m\n",
      "\u001b[34m94%|█████████▍| 757/803 [47:01<02:42,  3.54s/it]\u001b[0m\n",
      "\u001b[34m94%|█████████▍| 758/803 [47:05<02:45,  3.69s/it]\u001b[0m\n",
      "\u001b[34m95%|█████████▍| 759/803 [47:08<02:42,  3.69s/it]\u001b[0m\n",
      "\u001b[34m95%|█████████▍| 760/803 [47:12<02:37,  3.67s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.834, 'learning_rate': 3.5293203563735444e-07, 'epoch': 0.95}\u001b[0m\n",
      "\u001b[34m95%|█████████▍| 760/803 [47:12<02:37,  3.67s/it]\u001b[0m\n",
      "\u001b[34m95%|█████████▍| 761/803 [47:16<02:37,  3.75s/it]\u001b[0m\n",
      "\u001b[34m95%|█████████▍| 762/803 [47:19<02:22,  3.48s/it]\u001b[0m\n",
      "\u001b[34m95%|█████████▌| 763/803 [47:23<02:32,  3.81s/it]\u001b[0m\n",
      "\u001b[34m95%|█████████▌| 764/803 [47:27<02:28,  3.80s/it]\u001b[0m\n",
      "\u001b[34m95%|█████████▌| 765/803 [47:30<02:15,  3.56s/it]\u001b[0m\n",
      "\u001b[34m95%|█████████▌| 766/803 [47:34<02:17,  3.72s/it]\u001b[0m\n",
      "\u001b[34m96%|█████████▌| 767/803 [47:38<02:09,  3.61s/it]\u001b[0m\n",
      "\u001b[34m96%|█████████▌| 768/803 [47:42<02:10,  3.74s/it]\u001b[0m\n",
      "\u001b[34m96%|█████████▌| 769/803 [47:46<02:14,  3.96s/it]\u001b[0m\n",
      "\u001b[34m96%|█████████▌| 770/803 [47:51<02:14,  4.08s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.8709, 'learning_rate': 2.0806700251775057e-07, 'epoch': 0.96}\u001b[0m\n",
      "\u001b[34m96%|█████████▌| 770/803 [47:51<02:14,  4.08s/it]\u001b[0m\n",
      "\u001b[34m96%|█████████▌| 771/803 [47:54<02:02,  3.83s/it]\u001b[0m\n",
      "\u001b[34m96%|█████████▌| 772/803 [47:58<01:59,  3.86s/it]\u001b[0m\n",
      "\u001b[34m96%|█████████▋| 773/803 [48:01<01:54,  3.83s/it]\u001b[0m\n",
      "\u001b[34m96%|█████████▋| 774/803 [48:05<01:52,  3.89s/it]\u001b[0m\n",
      "\u001b[34m97%|█████████▋| 775/803 [48:09<01:42,  3.67s/it]\u001b[0m\n",
      "\u001b[34m97%|█████████▋| 776/803 [48:13<01:41,  3.77s/it]\u001b[0m\n",
      "\u001b[34m97%|█████████▋| 777/803 [48:16<01:35,  3.68s/it]\u001b[0m\n",
      "\u001b[34m97%|█████████▋| 778/803 [48:20<01:34,  3.79s/it]\u001b[0m\n",
      "\u001b[34m97%|█████████▋| 779/803 [48:25<01:36,  4.01s/it]\u001b[0m\n",
      "\u001b[34m97%|█████████▋| 780/803 [48:28<01:26,  3.74s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.7528, 'learning_rate': 1.0114426895421746e-07, 'epoch': 0.97}\u001b[0m\n",
      "\u001b[34m97%|█████████▋| 780/803 [48:28<01:26,  3.74s/it]\u001b[0m\n",
      "\u001b[34m97%|█████████▋| 781/803 [48:32<01:23,  3.82s/it]\u001b[0m\n",
      "\u001b[34m97%|█████████▋| 782/803 [48:35<01:18,  3.74s/it]\u001b[0m\n",
      "\u001b[34m98%|█████████▊| 783/803 [48:39<01:13,  3.68s/it]\u001b[0m\n",
      "\u001b[34m98%|█████████▊| 784/803 [48:43<01:11,  3.75s/it]\u001b[0m\n",
      "\u001b[34m98%|█████████▊| 785/803 [48:48<01:13,  4.07s/it]\u001b[0m\n",
      "\u001b[34m98%|█████████▊| 786/803 [48:51<01:05,  3.87s/it]\u001b[0m\n",
      "\u001b[34m98%|█████████▊| 787/803 [48:55<01:03,  3.96s/it]\u001b[0m\n",
      "\u001b[34m98%|█████████▊| 788/803 [48:58<00:54,  3.65s/it]\u001b[0m\n",
      "\u001b[34m98%|█████████▊| 789/803 [49:01<00:48,  3.48s/it]\u001b[0m\n",
      "\u001b[34m98%|█████████▊| 790/803 [49:07<00:53,  4.12s/it]\u001b[0m\n",
      "\u001b[34m{'loss': 1.7676, 'learning_rate': 3.232747262626179e-08, 'epoch': 0.98}\u001b[0m\n",
      "\u001b[34m98%|█████████▊| 790/803 [49:07<00:53,  4.12s/it]\u001b[0m\n",
      "\u001b[34m99%|█████████▊| 791/803 [49:10<00:45,  3.80s/it]\u001b[0m\n",
      "\u001b[34m99%|█████████▊| 792/803 [49:14<00:43,  3.92s/it]\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "from sagemaker.estimator import Estimator\n",
    "\n",
    "environment = {\n",
    "              'MODEL_S3_PATH': model_s3_path # The bucket to store pretrained model and fine-tune model\n",
    "}\n",
    "\n",
    "base_job_name = 'baichuan-finetuning'\n",
    "\n",
    "instance_type = 'ml.g5.24xlarge'\n",
    "\n",
    "estimator = Estimator(role=role,\n",
    "                      entry_point='train_distribute.sh',\n",
    "                      source_dir='./LLaMA-Efficient-Tuning/',\n",
    "                      base_job_name=base_job_name,\n",
    "                      instance_count=1,\n",
    "                      instance_type=instance_type,\n",
    "                      image_uri=image_uri,\n",
    "                      environment=environment,\n",
    "                      disable_profiler=True,\n",
    "                      debugger_hook_config=False,\n",
    "                      max_run=24*60*60*2)\n",
    "\n",
    "estimator.fit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e498053a-03e8-4847-9c34-6fc1c45edc09",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_amazonei_pytorch_latest_p37",
   "language": "python",
   "name": "conda_amazonei_pytorch_latest_p37"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}