sagemaker-trainging-bloomz-7B.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "5749602f-57a7-463a-b2a6-243739702005",
   "metadata": {
    "tags": []
   },
   "source": [
    "# 在Amazon SageMaker上训练bloomz7b(BELLE)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1d5ec6b3-9eb6-4baf-90f5-12431606909e",
   "metadata": {
    "tags": []
   },
   "source": [
    "BELLE用200万条中文对话在Bloomz7b上FT了一版模型，之前测试法律问答分数比同样200万中文对话FT LLama130亿高一些。\n",
    "所以第一次训练，采用参数较少的200万条FT的Bloom7b作为我们的基座。\n",
    "后续如果在增加训练样本后模型表现无任何提升，则证明70亿参数基座无法满足我们的需求，到时候再考虑选择130亿LLama。\n",
    "（目前Bloomz蒸馏的最大参数仅有70亿参数版本，再往上就是千亿参数了）"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5ae53805-559a-49d1-a5a6-4365723f505e",
   "metadata": {},
   "source": [
    "## 1、环境配置、准备数据、下载模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "77e30111-f013-4125-8fa8-63c6a7b1e6e8",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Looking in indexes: https://pypi.org/simple, https://pip.repos.neuron.amazonaws.com\n",
      "Requirement already satisfied: sagemaker in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (2.158.0)\n",
      "Requirement already satisfied: platformdirs in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from sagemaker) (2.6.2)\n",
      "Requirement already satisfied: attrs<24,>=23.1.0 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from sagemaker) (23.1.0)\n",
      "Requirement already satisfied: jsonschema in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from sagemaker) (3.2.0)\n",
      "Requirement already satisfied: boto3<2.0,>=1.26.131 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from sagemaker) (1.26.138)\n",
      "Requirement already satisfied: protobuf3-to-dict<1.0,>=0.1.5 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from sagemaker) (0.1.5)\n",
      "Requirement already satisfied: smdebug-rulesconfig==1.0.1 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from sagemaker) (1.0.1)\n",
      "Requirement already satisfied: importlib-metadata<5.0,>=1.4.0 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from sagemaker) (4.13.0)\n",
      "Requirement already satisfied: schema in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from sagemaker) (0.7.5)\n",
      "Requirement already satisfied: numpy<2.0,>=1.9.0 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from sagemaker) (1.23.5)\n",
      "Requirement already satisfied: pandas in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from sagemaker) (1.4.4)\n",
      "Requirement already satisfied: pathos in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from sagemaker) (0.3.0)\n",
      "Requirement already satisfied: packaging>=20.0 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from sagemaker) (21.3)\n",
      "Requirement already satisfied: tblib==1.7.0 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from sagemaker) (1.7.0)\n",
      "Requirement already satisfied: protobuf<4.0,>=3.1 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from sagemaker) (3.20.2)\n",
      "Requirement already satisfied: cloudpickle==2.2.1 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from sagemaker) (2.2.1)\n",
      "Requirement already satisfied: google-pasta in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from sagemaker) (0.2.0)\n",
      "Requirement already satisfied: PyYAML==6.0 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from sagemaker) (6.0)\n",
      "Requirement already satisfied: s3transfer<0.7.0,>=0.6.0 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from boto3<2.0,>=1.26.131->sagemaker) (0.6.0)\n",
      "Requirement already satisfied: botocore<1.30.0,>=1.29.138 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from boto3<2.0,>=1.26.131->sagemaker) (1.29.138)\n",
      "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from boto3<2.0,>=1.26.131->sagemaker) (1.0.1)\n",
      "Requirement already satisfied: zipp>=0.5 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from importlib-metadata<5.0,>=1.4.0->sagemaker) (3.11.0)\n",
      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from packaging>=20.0->sagemaker) (3.0.9)\n",
      "Requirement already satisfied: six in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from protobuf3-to-dict<1.0,>=0.1.5->sagemaker) (1.16.0)\n",
      "Requirement already satisfied: pyrsistent>=0.14.0 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from jsonschema->sagemaker) (0.19.3)\n",
      "Requirement already satisfied: setuptools in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from jsonschema->sagemaker) (65.6.3)\n",
      "Requirement already satisfied: python-dateutil>=2.8.1 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from pandas->sagemaker) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from pandas->sagemaker) (2022.7)\n",
      "Requirement already satisfied: dill>=0.3.6 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from pathos->sagemaker) (0.3.6)\n",
      "Requirement already satisfied: pox>=0.3.2 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from pathos->sagemaker) (0.3.2)\n",
      "Requirement already satisfied: ppft>=1.7.6.6 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from pathos->sagemaker) (1.7.6.6)\n",
      "Requirement already satisfied: multiprocess>=0.70.14 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from pathos->sagemaker) (0.70.14)\n",
      "Requirement already satisfied: contextlib2>=0.5.5 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from schema->sagemaker) (21.6.0)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.25.4 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from botocore<1.30.0,>=1.29.138->boto3<2.0,>=1.26.131->sagemaker) (1.26.8)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:botocore.credentials:Found credentials from IAM Role: BaseNotebookInstanceEc2InstanceRole\n",
      "INFO:botocore.credentials:Found credentials from IAM Role: BaseNotebookInstanceEc2InstanceRole\n"
     ]
    }
   ],
   "source": [
    "##注意使用这个笔记本时，尽量翻墙，否则可能出现504导致你写了一半的代码无法保存。\n",
    "## Update sagemaker python sdk version\n",
    "!pip install -U sagemaker\n",
    "\n",
    "import sagemaker\n",
    "import boto3\n",
    "from sagemaker import get_execution_role\n",
    "\n",
    "sess = sagemaker.Session()\n",
    "role = get_execution_role()\n",
    "sagemaker_default_bucket = sess.default_bucket()\n",
    "\n",
    "account = sess.boto_session.client(\"sts\").get_caller_identity()[\"Account\"]\n",
    "region = sess.boto_session.region_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "4afd8fcb-f191-4f86-80a0-1af1cd79d874",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cloning into 'BELLE'...\n",
      "remote: Enumerating objects: 1758, done.\u001b[K\n",
      "remote: Counting objects: 100% (754/754), done.\u001b[K\n",
      "remote: Compressing objects: 100% (435/435), done.\u001b[K\n",
      "remote: Total 1758 (delta 482), reused 501 (delta 312), pack-reused 1004\u001b[K\n",
      "Receiving objects: 100% (1758/1758), 11.61 MiB | 24.93 MiB/s, done.\n",
      "Resolving deltas: 100% (926/926), done.\n",
      "Cloning into 'alpaca-on-amazon-sagemaker'...\n",
      "remote: Enumerating objects: 46, done.\u001b[K\n",
      "remote: Counting objects: 100% (46/46), done.\u001b[K\n",
      "remote: Compressing objects: 100% (39/39), done.\u001b[K\n",
      "remote: Total 46 (delta 20), reused 17 (delta 6), pack-reused 0\u001b[K\n",
      "Receiving objects: 100% (46/46), 4.03 MiB | 16.42 MiB/s, done.\n",
      "Resolving deltas: 100% (20/20), done.\n"
     ]
    }
   ],
   "source": [
    "## 下载BELLE项目的训练代码\n",
    "!git clone https://github.com/LianjiaTech/BELLE.git\n",
    "## 下载亚马逊官方教程代码\n",
    "!git clone https://github.com/snowolf/alpaca-on-amazon-sagemaker.git"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b5ff9662-7775-49cd-89d9-775ef52a84f5",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"id\": \"ilaw-1\", \"conversations\": [{\"from\": \"human\", \"value\": \"民法商法-证券法2019-12-28:    \\\"第一百一十五条 证券交易所依照法律、行政法规和国务院证券监督管理机构的规定，制定上市规则、交易规则、会员管理规则和其他有关业务规则，并报国务院证券监督管理机构批准。\\\",\\n民法商法-证券法2019-12-28:    \\\"在证券交易所从事证券交易，应当遵守证券交易所依法制定的业务规则。违反业务规则的，由证券交易所给予纪律处分或者采取其他自律管理措施。\\\",\\n民法商法-证券法2019-12-28:    \\\"第一百一十六条 证券交易所的负责人和其他从业人员执行与证券交易有关的职务时，与其本人或者其亲属有利害关系的，应当回避。\\\",\\n民法商法-证券法2019-12-28:    \\\"第一百一十七条 按照依法制定的交易规则进行的交易，不得改变其交易结果，但本法第一百一十一条第二款规定的除外。对交易中违规交易者应负的民事责任不得免除；在违规交易中所获利益，依照有关规定处理。\\\",\\n请根据以上法律，选择最合适的法律生成问题的合理答复，问题是：\\n小张在证券交易所进行证券交易时，未遵守证券交易所的业务规则，而被证券交易所给予了纪律处分，他想知道依据什么法规证券交易所可以制定业务规则。\\n\\n证券交易所可以根据哪些法规制定业务规则？\"}, {\"from\": \"assistant\", \"value\": \"证券交易所可以根据法律、行政法规和国务院证券监督管理机构的规定制定上市规则、交易规则、会员管理规则和其他有关业务规则，并报国务院证券监督管理机构批准，这是根据《中华人民共和国证券法》第一百一十五条规定的。\"}]}\n",
      "{\"id\": \"ilaw-2\", \"conversations\": [{\"from\": \"human\", \"value\": \"软件著作权产品登记流程有哪些\"}, {\"from\": \"assistant\", \"value\": \"计算机软件版权登记完成整个时间大约需要4个月左右。整个流程为递交申请、版权中心受理、审查登记材料、获取登记证书 所需文件：计算机软件登记申请表、计算机源程序、软件说明书、著作权归属协议、委托书。\"}]}\n",
      "{\"id\": \"ilaw-3\", \"conversations\": [{\"from\": \"human\", \"value\": \"给出一个任务，要求GPT模型完成一个类似短促广告或标语的创意句子。\\n将推广某种美容护肤产品的一句话。\"}, {\"from\": \"assistant\", \"value\": \"“拥抱自然之美，由内而外，展现你最美的肌肤。”\"}]}\n",
      "{\"id\": \"ilaw-4\", \"conversations\": [{\"from\": \"human\", \"value\": \"基于给定的电影评论，判断评论的情感（正面、负面或中性）。\\n评论: \\\"这部电影真是太精彩了，我一直都被吸引着看完了整个故事，大力推荐！\\\"\"}, {\"from\": \"assistant\", \"value\": \"正面。\"}]}\n",
      "{\"id\": \"ilaw-5\", \"conversations\": [{\"from\": \"human\", \"value\": \"为以下机器学习任务设计算法。\\n任务：根据客户的历史购买记录创建个性化产品推荐。\\n\"}, {\"from\": \"assistant\", \"value\": \"推荐系统算法：基于协同过滤、矩阵分解、内容过滤、深度学习等技术构建模型，在推荐系统中为每个用户提供个性化的商品推荐。\"}]}\n",
      "{\"id\": \"ilaw-6\", \"conversations\": [{\"from\": \"human\", \"value\": \"讲述一件历史事件并提到该事件的地点、时间和主要参与者。\\n\"}, {\"from\": \"assistant\", \"value\": \"1776年7月4日，美国的费城市（位于宾夕法尼亚州）发生了历史性的事件：美国独立宣言的通过。这项宣言的主要参与者是美国的开国元勋，如托马斯·杰斐逊、约翰·亚当斯和本杰明·富兰克林等。该事件宣告了美国对英国的独立，并成为美国建国的重要里程碑之一。\"}]}\n",
      "{\"id\": \"ilaw-7\", \"conversations\": [{\"from\": \"human\", \"value\": \"根据提供的关键词生成一个段落（5-7句）的文本。\\n关键词: 猫，行为\\n\"}, {\"from\": \"assistant\", \"value\": \"猫是一种行为高度特异的动物，它们经常表现出趋近、狩猎和攻击的行为。猫是掠食者，而他们的行为往往是为了寻找猎物。此外，猫也是很有好奇心的动物，它们喜欢在新环境中探索和观察。猫的清洁习惯也很独特，它们花费大量时间舔毛和清理身体。猫还会表现出拍打、拉扯等玩耍行为，这些行为对于猫来说也是一种锻炼和放松的方式。总的来说，猫的行为给人带来了很多乐趣和惊喜，也使得它们成为了人类最喜欢的宠物之一。\"}]}\n",
      "{\"id\": \"ilaw-8\", \"conversations\": [{\"from\": \"human\", \"value\": \"对话内容：\\nJohn：欢迎来到巴黎，Sara！这里是卢浮宫，欧洲最大的博物馆之一。\\nSara：哇，真是太壮观了！这里有多少件艺术品啊？\\nJohn：大约有38万件，其中包括蒙娜丽莎这样的知名作品。您想要看哪些展览？\\nSara：我想看看文艺复兴时期的绘画作品。\\nJohn：那我们来到这个区域吧，这里有达芬奇、拉斐尔和米开朗琪罗的杰作。\\n问题：\\nSara最喜欢的是哪个时期的绘画作品？\\n根据对话内容回答这个问题。\"}, {\"from\": \"assistant\", \"value\": \"\\nSara最喜欢文艺复兴时期的绘画作品。\"}]}\n",
      "{\"id\": \"ilaw-9\", \"conversations\": [{\"from\": \"human\", \"value\": \"给出一份家庭聚餐菜单，该菜单应适合各年龄段的家庭成员。\\n家庭聚餐，有5位成人和3个孩子，8-10岁。\"}, {\"from\": \"assistant\", \"value\": \"以下是适合各年龄段的家庭聚餐菜单：\\n1. 香煎牛排或烤鸡腿 - 提供蛋白质给成人和孩子\\n2. 烤蔬菜 - 健康的蔬菜可以搭配肉类提供平衡饮食\\n3. 奶油土豆泥 - 可以搭配牛排或鸡腿\\n4. 咖喱焗南瓜 - 适合素食者和孩子\\n5. 意式番茄酱意大利面 - 适合爱吃面食的成人和孩子\\n6. 柠檬蛋糕 - 甜品是家庭聚餐必不可少的\\n此菜单满足了各年龄段的家庭成员饮食需求和口味喜好。\"}]}\n",
      "{\"id\": \"ilaw-10\", \"conversations\": [{\"from\": \"human\", \"value\": \"生成一个翻译任务。\\nMy favorite color is blue.\"}, {\"from\": \"assistant\", \"value\": \"Translate \\\"My favorite color is blue\\\" into Chinese.\"}]}\n",
      "{\"id\": \"ilaw-11\", \"conversations\": [{\"from\": \"human\", \"value\": \"为给定的名称和日期创建一个生日祝福。\\n名字：张三；日期：7月1日\\n\"}, {\"from\": \"assistant\", \"value\": \"祝张三生日快乐，7月1日。\"}]}\n",
      "{\"id\": \"ilaw-12\", \"conversations\": [{\"from\": \"human\", \"value\": \"要求GPT模型列出一些适合夏天的户外活动。 \\n\"}, {\"from\": \"assistant\", \"value\": \"一些适合夏天的户外活动包括游泳、野餐、露营、骑自行车、登山、沙滩排球、皮划艇、皮划艇、草地高尔夫球和钓鱼。\"}]}\n",
      "{\"id\": \"ilaw-13\", \"conversations\": [{\"from\": \"human\", \"value\": \"给定一个段落或文章，请编辑使其更易于阅读。\\n段落或文章（例如，一篇新闻报道或学术论文）\"}, {\"from\": \"assistant\", \"value\": \"请提供一篇具体的文章或段落以供编辑。\"}]}\n",
      "{\"id\": \"ilaw-14\", \"conversations\": [{\"from\": \"human\", \"value\": \"构成价格欺诈的行为有哪些\"}, {\"from\": \"assistant\", \"value\": \"目前规定的属于价格欺诈的行为主要有十三种之多。就包括虚假标价、两套价格、欺骗性标价、模糊标价和虚夸标价、虚假折价、对处理品不标价、不如实披露赠品情况、隐蔽价格附加条件、虚构原价、不履行价格承诺、虚构比较价格、价质不符以及谎称“政府定价”。\"}]}\n",
      "{\"id\": \"ilaw-15\", \"conversations\": [{\"from\": \"human\", \"value\": \"帮我翻译以下文本: \\\"The quick brown fox jumps over the lazy dog.\\\"\\n\"}, {\"from\": \"assistant\", \"value\": \"\\\"The quick brown fox jumps over the lazy dog.\\\" 的翻译是：敏捷的棕色狐狸跳过懒惰的狗。\"}]}\n",
      "{\"id\": \"ilaw-16\", \"conversations\": [{\"from\": \"human\", \"value\": \"给定一个数学问题，输出该问题的答案。\\n22 + 17\\n\"}, {\"from\": \"assistant\", \"value\": \"39\"}]}\n",
      "{\"id\": \"ilaw-17\", \"conversations\": [{\"from\": \"human\", \"value\": \"交通事故伤残鉴定费由谁来支付,具体规定有哪些\"}, {\"from\": \"assistant\", \"value\": \"交通事故伤残鉴定费由由公安机关交通管理部门承担，但当事人自行委托伤残评定、财产损失评估的由当事人自行承担。根据规定，检验、鉴定费用由公安机关交通管理部门承担，但法律法规另有规定或者当事人自行委托伤残评定、财产损失评估的除外。法律依据：《道路交通事故处理程序规定》第五十条检验、鉴定费用由公安机关交通管理部门承担，但法律法规另有规定或者当事人自行委托伤残评定、财产损失评估的除外。第九十二条因确定损害赔偿的数额，需要进行伤残评定、财产损失评估的，由各方当事人协商确定有资质的机构进行，但财产损失数额巨大涉嫌刑事犯罪的，由公安机关交通管理部门委托。当事人委托伤残评定、财产损失评估的费用，由当事人承担。\"}]}\n",
      "{\"id\": \"ilaw-18\", \"conversations\": [{\"from\": \"human\", \"value\": \"根据提供的一段文本，生成一篇类似的文章。\\n原文本：随着科技的不断发展，我们的生活方式和工作方式也在不断改变。在数字时代，数字技术已经成为我们生活和工作的重要组成部分。从教育、医疗、金融到娱乐，数字化已经席卷了几乎所有的领域。然而，数字世界也带来了一些新挑战，如网络欺凌和数据隐私问题等。\"}, {\"from\": \"assistant\", \"value\": \"随着时代在不断地推进，技术的发展也在不断的深化，使我们的生活和工作方式都得到了很大的改变。现如今在数字时代，数字技术已经成为我们日常生活和工作中的重要组成部分。无论是在教育、医疗、金融还是娱乐领域，数字化的影响已经无处不在。但是，数字世界所带来的也有一些新的挑战，例如网络欺凌与数据隐私的问题等等。\"}]}\n",
      "{\"id\": \"ilaw-19\", \"conversations\": [{\"from\": \"human\", \"value\": \"公安部-公安机关办理行政案件程序规定2019-04-27:    \\\"第二百二十一条 对同时被决定行政拘留和社区戒毒或者强制隔离戒毒的人员，应当先执行行政拘留，由拘留所给予必要的戒毒治疗，强制隔离戒毒期限连续计算。\\\",\\n公安部-公安机关办理行政案件程序规定2019-04-27:    \\\"拘留所不具备戒毒治疗条件的，行政拘留决定机关可以直接将被行政拘留人送公安机关管理的强制隔离戒毒所代为执行行政拘留，强制隔离戒毒期限连续计算。\\\",\\n公安部-公安机关办理行政案件程序规定2019-04-27:    \\\"第二百二十二条 被处罚人不服行政拘留处罚决定，申请行政复议或者提起行政诉讼的，可以向作出行政拘留决定的公安机关提出暂缓执行行政拘留的申请；口头提出申请的，公安机关人民警察应当予以记录，并由申请人签名或者捺指印。\\\",\\n公安部-公安机关办理行政案件程序规定2019-04-27:    \\\"被处罚人在行政拘留执行期间，提出暂缓执行行政拘留申请的，拘留所应当立即将申请转交作出行政拘留决定的公安机关。\\\",\\n请根据以上法律，选择最合适的法律生成问题的合理答复，问题是：\\n如果一个人同时被决定执行行政拘留和社区戒毒，应该先执行哪个？行政拘留执行期间该如何进行戒毒治疗？\"}, {\"from\": \"assistant\", \"value\": \"根据公安部-公安机关办理行政案件程序规定，对同时被决定行政拘留和社区戒毒的人员，应当先执行行政拘留，并由拘留所给予必要的戒毒治疗，强制隔离戒毒期限连续计算。如果拘留所不具备戒毒治疗条件，行政拘留决定机关可以直接将被行政拘留人送公安机关管理的强制隔离戒毒所代为执行行政拘留，强制隔离戒毒期限连续计算。如果被处罚人在行政拘留执行期间提出暂缓执行行政拘留申请，拘留所应当立即将申请转交作出行政拘留决定的公安机关。\"}]}\n",
      "{\"id\": \"ilaw-20\", \"conversations\": [{\"from\": \"human\", \"value\": \"提供一个确切的日期，询问那一天发生了哪些重要的历史事件。\\n1969年7月20日\\n\"}, {\"from\": \"assistant\", \"value\": \"1969年7月20日，美国宇航员阿姆斯特朗首次踏上月球，并发出了“人类历史上的一小步，人类历史上的一大步”的名言。\"}]}\n",
      "{\"id\": \"ilaw-21\", \"conversations\": [{\"from\": \"human\", \"value\": \"向某人推荐一部你最喜欢的电影，并解释推荐的原因。\\n\"}, {\"from\": \"assistant\", \"value\": \"我推荐电影《肖申克的救赎》，因为它是一部经典的心灵成长电影，充满了温情和启示。影片讲述了一位冤狱的银行家如何在狱中与朋友共同生活，并最终通过聪明才智和毅力逃脱囚禁的故事。影片的演员表现也非常出色，特别是主演蒂姆·罗宾斯和摩根·弗里曼的表演。总之，这部电影是一部让人感动和启发人生的佳作。\"}]}\n",
      "{\"id\": \"ilaw-22\", \"conversations\": [{\"from\": \"human\", \"value\": \"根据以下提示，编写一篇短故事\\n题目：失而复得\\n要求：编写一个情节生动、情感丰富的短故事，以“失而复得”为主题。\\n\"}, {\"from\": \"assistant\", \"value\": \"在繁华的城市里，有一位年轻的女孩名叫莉莉。她是一名优秀的记者，专门写关于环保和动物保护的文章，一直为了守护这片土地而奋斗着。 \\n某天，她接到了一份关于一个破坏森林的公司的报道任务，于是她去了森林深处调查采访。在森林中，莉莉遇到了一只小猎豹，它的一只前爪被捕夹夹住了，莉莉决定救助它。在她缓缓地为猎豹处理伤口时，她和小猎豹渐渐地建立起联系。\\n可是任务却没有那么顺利。在拍摄时，她被一些不明身份的人绑架了。她试图逃脱，但徒劳无功。在困顿之际，小猎豹出现了。僵持一会儿后，小猎豹惊人地攻击了那些恶人。莉莉趁机逃脱，但小猎豹也因此慢了半拍受伤了。\\n后来，莉莉真的赢得了法庭的胜诉，成功让那家公司关门了，但她始终思念着失去的伙伴。有一天，她走在回家的路上，脚下突然一滑，她差点跌倒。抬起头，她看见了眼前的小猎豹。这时的小猎豹已经快乐地摇摆着它完好无损的前爪了。\\n“失而复得”这个主题对于莉莉来说早已不止是一个简单的故事。它代表了她和那个曾与她共同战斗的伙伴，经历了失去和重逢的坎坷困难，最终它们再次回到了彼此的身边。\"}]}\n",
      "{\"id\": \"ilaw-23\", \"conversations\": [{\"from\": \"human\", \"value\": \"民法典-物权编:    \\\"第二百四十三条 为了公共利益的需要，依照法律规定的权限和程序可以征收集体所有的土地和组织、个人的房屋以及其他不动产。\\\",\\n民法典-物权编:    \\\"征收集体所有的土地，应当依法及时足额支付土地补偿费、安置补助费以及农村村民住宅、其他地上附着物和青苗等的补偿费用，并安排被征地农民的社会保障费用，保障被征地农民的生活，维护被征地农民的合法权益。\\\",\\n民法典-物权编:    \\\"征收组织、个人的房屋以及其他不动产，应当依法给予征收补偿，维护被征收人的合法权益；征收个人住宅的，还应当保障被征收人的居住条件。\\\",\\n民法典-物权编:    \\\"任何组织或者个人不得贪污、挪用、私分、截留、拖欠征收补偿费等费用。\\\",\\n民法典-物权编:    \\\"第二百四十四条 国家对耕地实行特殊保护，严格限制农用地转为建设用地，控制建设用地总量。不得违反法律规定的权限和程序征收集体所有的土地。\\\",\\n请根据以上法律，选择最合适的法律生成问题的合理答复，问题是：\\n在某县城建设市政项目需要征收该县城一个农村集体所有的土地，该县城政府应该如何征收？\"}, {\"from\": \"assistant\", \"value\": \"根据《民法典-物权编》第二百四十三条规定，为了公共利益的需要，依照法律规定的权限和程序可以征收集体所有的土地。在征收过程中，县城政府应当依法及时足额支付土地补偿费、安置补助费以及农村村民住宅、其他地上附着物和青苗等的补偿费用，并安排被征地农民的社会保障费用，保障被征地农民的生活，维护被征地农民的合法权益。在征收过程中，任何组织或者个人不得贪污、挪用、私分、截留、拖欠征收补偿费等费用。同时，《民法典-物权编》第二百四十四条规定，国家对耕地实行特殊保护，严格限制农用地转为建设用地，控制建设用地总量。因此，县城政府在征收该农村集体所有的土地时，需要严格遵守法律规定，进行合理的补偿，并保障被征地农民的权益，同时不能违反法律规定将农用地转为建设用地。\"}]}\n",
      "{\"id\": \"ilaw-24\", \"conversations\": [{\"from\": \"human\", \"value\": \"给定一组数字，计算它们的平均值。\\n数1: 20\\n数2: 30\\n数3: 40\\n\"}, {\"from\": \"assistant\", \"value\": \"这组数字的平均值为30。\"}]}\n",
      "{\"id\": \"ilaw-25\", \"conversations\": [{\"from\": \"human\", \"value\": \"拖欠地砖工程款2个月怎么处理，流程有哪些？\"}, {\"from\": \"assistant\", \"value\": \"工程款拖欠纠纷的解决方法：（一）与欠款人对工程欠款进行协商，达成还款协议；（二）请求追债公司的帮助；（三）要求建设单位提供按期付款的履约担保；（四）到欠款人住所地或工程地的人民法院提起诉讼。《中华人民共和国民法典》第八百零七条  发包人未按照约定支付价款的，承包人可以催告发包人在合理期限内支付价款。发包人逾期不支付的，除根据建设工程的性质不宜折价、拍卖外，承包人可以与发包人协议将该工程折价，也可以请求人民法院将该工程依法拍卖。建设工程的价款就该工程折价或者拍卖的价款优先受偿。\"}]}\n",
      "{\"id\": \"ilaw-26\", \"conversations\": [{\"from\": \"human\", \"value\": \"从给定的文章中生成一段总结摘要\\n文章：随着数字化技术的日益发展，人们变得更加依赖计算机和互联网。但是，随着依赖程度的加深，网络安全问题也随之增加。信息安全和隐私保护已经成为我们社会和经济生活中面临的重大问题。在这篇文章中，我们将探讨当前互联网环境下的网络安全问题，以及如何针对这些问题采取预防措施。\"}, {\"from\": \"assistant\", \"value\": \"本文讨论了随着数字化技术的发展，人们越来越依赖计算机和互联网所带来的安全和隐私问题，并提供了解决这些问题的一些预防措施。\"}]}\n",
      "{\"id\": \"ilaw-27\", \"conversations\": [{\"from\": \"human\", \"value\": \"上市公司股权分配怎么规定？\"}, {\"from\": \"assistant\", \"value\": \"一般来说，公司股权分配比例按照所持股份进行分配，持股多少，比例多少，持股越多，比例越高，上市公司的股权比例还可以通过收购的方式来不断提升，甚至可以达到控股的股权比例。法律依据：《公司法》第二十八条股东应当按期足额缴纳公司章程中规定的各自所认缴的出资额。股东以货币出资的，应当将货币出资足额存入有限责任公司在银行开设的账户；以非货币财产出资的，应当依法办理其财产权的转移手续。股东不按照前款规定缴纳出资的，除应当向公司足额缴纳外，还应当向已按期足额缴纳出资的股东承担违约责任。\"}]}\n",
      "{\"id\": \"ilaw-28\", \"conversations\": [{\"from\": \"human\", \"value\": \"孩子被送养后与生父母还有抚养关系吗\"}, {\"from\": \"assistant\", \"value\": \"将孩子送养给他人收养后，被收养的子女与原来的生父母已经没有抚养的关系，自收养关系成立之日起，养父母与养子女间的权利义务关系，适用法律关于父母子女关系的规定;养子女与养父母的近亲属间的权利义务关系，适用法律关于子女与父母的近亲属关系的规定。\"}]}\n",
      "{\"id\": \"ilaw-29\", \"conversations\": [{\"from\": \"human\", \"value\": \"残疾人退休年龄怎样规定\"}, {\"from\": \"assistant\", \"value\": \"1、残疾人的退休政策与正常人一样，在退休年龄方面为男60周岁，女工人55岁。如果残疾人想办理提前退休，必须经市级劳动鉴定机构鉴定为完全丧失劳动能力，并且缴费年限满15年，由本人申请，经同级劳动保障行政部门批准，可以办理因病提前退休。\\n2、如果残疾人从事过特殊工种，可以在办理因病提前退休手续时，对1993年底以前实际从事特殊（如高空、井下、高温及有害身体健康）工种工作的年限进行折算，可以视同为缴费年限。\"}]}\n",
      "{\"id\": \"ilaw-30\", \"conversations\": [{\"from\": \"human\", \"value\": \"转换一段短文的过去时为现在时。\\n当我还是个孩子的时候，我总是喜欢在公园里玩耍。我最喜欢的游戏是躲猫猫，经常大声喊着“我不是猫！”。\\n\"}, {\"from\": \"assistant\", \"value\": \"现在时：当我现在还是个孩子时，我总是喜欢在公园里玩耍。我最喜欢的游戏是躲猫猫，经常大声喊着“我不是猫！”。\"}]}\n",
      "{\"id\": \"ilaw-31\", \"conversations\": [{\"from\": \"human\", \"value\": \"我因轻微伤被处罚拘留十日，现在人还没去拘留，要过两天去，如果我去拘留了有可能再重新做一次鉴定吗？?\"}, {\"from\": \"assistant\", \"value\": \"如果您已经被判处拘留十日，表明已经经过了相应的司法程序，如果您去执行拘留，不会再次进行鉴定。但是，如果您对您的案件有异议，可以通过合法手段提出上诉或申请再审。\"}]}\n",
      "{\"id\": \"ilaw-32\", \"conversations\": [{\"from\": \"human\", \"value\": \"《中华人民共和国刑法》第三百八十四条，国家工作人员利用职务上的便利，挪用公款归个人使用，进行非法活动的，或者挪用公款数额较大、进行营利活动的，或者挪用公款数额较大、超过三个月未还的，是挪用公款罪，处五年以下有期徒刑或者拘役；情节严重的，处五年以上有期徒刑。挪用公款数额巨大不退还的，处十年以上有期徒刑或者无期徒刑。\\n挪用用于救灾、抢险、防汛、优抚、扶贫、移民、救济款物归个人使用的，从重处罚。\\n请根据以上法律，选择最合适的法律生成问题的合理答复，问题是：\\n个人在单位借款不能超过多长时间\"}, {\"from\": \"assistant\", \"value\": \"个人在单位借款最长的实际没有具体限制，由当事人之间自行约定，但是如果公司、企业或者其他单位的工作人员利用职务上的便利挪用本单位的资金归个人适用或者借贷给他人的，可能构成挪用公款罪。\"}]}\n",
      "{\"id\": \"ilaw-33\", \"conversations\": [{\"from\": \"human\", \"value\": \"给出一段文本并将其中的每个单词反转。\\n这是一个测试字符串。\\n\"}, {\"from\": \"assistant\", \"value\": \"。串字测试个一个是这\"}]}\n",
      "{\"id\": \"ilaw-34\", \"conversations\": [{\"from\": \"human\", \"value\": \"根据给定的文本，生成一个类似的、不同于原文的段落。\\n人生不止眼前的苟且，还有诗和远方。这句话很有名，你可以根据这句话来生成一个新段落吗？\"}, {\"from\": \"assistant\", \"value\": \"除了繁琐的日常生活，我们的人生还有无限的可能性，比如追求诗歌的美和去远方的冒险。这是我们应该努力去实现和追求的梦想，让自己的人生变得更加有意义和丰富多彩。\"}]}\n",
      "{\"id\": \"ilaw-35\", \"conversations\": [{\"from\": \"human\", \"value\": \"用一句话回答以下问题。\\n根据2019年全球富豪榜，全球最富有的人是谁？\\n\"}, {\"from\": \"assistant\", \"value\": \"2019年全球最富有的人是亚马逊创始人杰夫·贝佐斯。\"}]}\n",
      "{\"id\": \"ilaw-36\", \"conversations\": [{\"from\": \"human\", \"value\": \"民法商法-证券法2019-12-28:    \\\"（九）依法对证券违法行为进行查处；\\\",\\n民法商法-证券法2019-12-28:    \\\"（十）法律、行政法规规定的其他职责。\\\",\\n民法商法-证券法2019-12-28:    \\\"第一百七十条 国务院证券监督管理机构依法履行职责，有权采取下列措施：\\\",\\n民法商法-证券法2019-12-28:    \\\"（一）对证券发行人、证券公司、证券服务机构、证券交易场所、证券登记结算机构进行现场检查；\\\",\\n民法商法-证券法2019-12-28:    \\\"（二）进入涉嫌违法行为发生场所调查取证；\\\",\\n民法商法-证券法2019-12-28:    \\\"（三）询问当事人和与被调查事件有关的单位和个人，要求其对与被调查事件有关的事项作出说明；或者要求其按照指定的方式报送与被调查事件有关的文件和资料；\\\",\\n民法商法-证券法2019-12-28:    \\\"（四）查阅、复制与被调查事件有关的财产权登记、通讯记录等文件和资料；\\\",\\n请根据以上法律，选择最合适的法律生成问题的合理答复，问题是：\\n国务院证券监督管理机构可以采取哪些措施来对证券违法行为进行查处？\"}, {\"from\": \"assistant\", \"value\": \"根据《证券法》第一百七十条规定，国务院证券监督管理机构可以采取现场检查、调查取证、询问当事人和有关单位和个人、要求报送文件和资料以及查阅、复制财产权登记和通讯记录等措施对证券违法行为进行查处。此外，该机构还有其他法律、行政法规规定的职责。\"}]}\n",
      "{\"id\": \"ilaw-37\", \"conversations\": [{\"from\": \"human\", \"value\": \"\\n角色介绍：\\n甲方：小明，一个年轻的创业者，刚刚受到了意外的投资机会。\\n乙方：投资人张先生，一个经验丰富的行业大佬，一直在寻找新的商业机会。\\n请续写他们的对话内容。乙方：不必客气，小明，我听说你有一家很有前途的创业公司，我非常感兴趣。甲方：是的，我所在的公司需要资金支持，同时也需要经验丰富的合作伙伴来帮忙开拓市场。乙方：我对你的公司很有信心，细节上我们可以再讨论。我们可以一起制定一个详细的计划，并共同努力，实现我们的商业愿景。甲方：\"}, {\"from\": \"assistant\", \"value\": \"非常感谢你的支持和信任，我会竭尽全力让这个公司发展得更好。\"}]}\n",
      "{\"id\": \"ilaw-38\", \"conversations\": [{\"from\": \"human\", \"value\": \"《医疗事故技术鉴定暂行办法》\\n第36条，专家鉴定组应当综合分析医疗过失行为在导致医疗事故损害后果中的作用、患者原有疾病状况等因素，判定医疗过失行为的责任程度。\\n请根据以上法律，选择最合适的法律生成问题的合理答复，问题是：\\n医院轻微责任怎么赔偿\"}, {\"from\": \"assistant\", \"value\": \"医院对于轻微责任鉴定为四级医疗事故的，不能按照残疾的标准索赔，只能主张医疗费、误工费、住院伙食补助费、陪护费、交通费和住宿费等必要的合理的费用。\"}]}\n",
      "{\"id\": \"ilaw-39\", \"conversations\": [{\"from\": \"human\", \"value\": \"找到一个翻译工具，将以下短语翻译成西班牙语。\\n“我喜欢吃披萨。”\\n\"}, {\"from\": \"assistant\", \"value\": \"Me gusta comer pizza.\"}]}\n",
      "{\"id\": \"ilaw-40\", \"conversations\": [{\"from\": \"human\", \"value\": \"嘉*区刑事案请律师如何收费 \"}, {\"from\": \"assistant\", \"value\": \"刑事案件请一个律师大概需要多少钱?具体要看律师事务所的收费，如：(一)刑事案件收费按照各办案阶段分别计件确定收费标准。1、侦查阶段，每件收费2###—1####元。2、审查起诉阶段，每件收费2###—1####元。3、一审阶段，每件收费4###—3####元。4、上述收费标准下浮不限。(二)二审、死刑复核、再审、申诉案件以及刑事自诉案件按照一审阶段的收费标准收取律师服务费。(三)一个律师事务所代理一个案件的多个阶段，自第二阶段起酌减收费。(四)被害人提起刑事附带民事诉讼案件的，按照民事诉讼案件收费标准收取律师服务费。(五)犯罪嫌疑人、被告人同时涉及几个罪名或者数起犯罪事实的，可按照所涉罪名或犯罪事实分别计件收取。\"}]}\n",
      "{\"id\": \"ilaw-41\", \"conversations\": [{\"from\": \"human\", \"value\": \"给定两个句子，将它们合并成一个具有逻辑关系的句子。\\n- 句子1: 我今天去购物了。\\n- 句子2: 我买了一些衣服。\\n\"}, {\"from\": \"assistant\", \"value\": \"我今天去购物了，买了一些衣服。\"}]}\n",
      "{\"id\": \"ilaw-42\", \"conversations\": [{\"from\": \"human\", \"value\": \"给定文本，找到最常见的单词并计算它在文本中出现的次数。\\n这是一个简短的句子，包含一些重复的词语。\"}, {\"from\": \"assistant\", \"value\": \"最常见的单词是\\\"的\\\"，它在文本中出现了3次。\"}]}\n",
      "{\"id\": \"ilaw-43\", \"conversations\": [{\"from\": \"human\", \"value\": \"根据所给的文章重写第一段，使其更简短易懂。\\n文章：\\n虽然研究表明，每天多喝一杯牛奶有助于维持健康的骨骼，但是并非所有的人都乐意饮用牛奶，尤其是那些选择避免动物性产品的人群。因此，人们需要寻找其他来源的钙补充剂，以确保他们的身体获得足够的钙质支持。\"}, {\"from\": \"assistant\", \"value\": \"每天多喝牛奶对于维护骨骼健康有帮助，但不是所有人都爱喝牛奶，需要寻找其他钙的来源。\"}]}\n",
      "{\"id\": \"ilaw-44\", \"conversations\": [{\"from\": \"human\", \"value\": \"大学生教师兼职被拖欠工资了，目前陷入僵局怎么办 \"}, {\"from\": \"assistant\", \"value\": \"可以与用人单位进行协商，协商不成的：一、向本单位劳*争议调解委员会申请调解；二、到劳*行政部门举报(通常是劳*管*监*大*)；三、也可以直接申请仲裁；四、如果对仲裁结果不满意可以在拿到仲裁书后15天之内到法院起诉。\"}]}\n",
      "{\"id\": \"ilaw-45\", \"conversations\": [{\"from\": \"human\", \"value\": \"请对以下句子进行语法纠错。\\n我想你的美的眼眸极其迷人。\"}, {\"from\": \"assistant\", \"value\": \"我想给你的美丽眼睛极其赞赏。\"}]}\n",
      "{\"id\": \"ilaw-46\", \"conversations\": [{\"from\": \"human\", \"value\": \"将一段文本翻译成另一种语言。\\n这是要翻译的文本。\"}, {\"from\": \"assistant\", \"value\": \"请提供需要翻译的语言。\"}]}\n",
      "{\"id\": \"ilaw-47\", \"conversations\": [{\"from\": \"human\", \"value\": \"我是一个有前科的人警察有权利随便检查我的手机吗 \"}, {\"from\": \"assistant\", \"value\": \" 不，警察不能随意检查你的手机。根据中国法律，警察只有在执行任务时才能进入私人场所进行检查和调查，并且必须得到相关领导的批准。此外，警察在检查时要遵守规定程序和法律程序，确保检查的合法性和公正性。\"}]}\n",
      "{\"id\": \"ilaw-48\", \"conversations\": [{\"from\": \"human\", \"value\": \"民法商法-海商法1992-11-07:    \\\"第二百二十五条 被保险人对同一保险标的就同一保险事故向几个保险人重复订立合同，而使该保险标的的保险金额总和超过保险标的的价值的，除合同另有约定外，被保险人可以向任何保险人提出赔偿请求。被保险人获得的赔偿金额总和不得超过保险标的的受损价值。各保险人按照其承保的保险金额同保险金额总和的比例承担赔偿责任。任何一个保险人支付的赔偿金额超过其应当承担的赔偿责任的，有权向未按照其应当承担的赔偿责任支付赔偿金额的保险人追偿。\\\",\\n民法商法-海商法1992-11-07:    \\\"第二百二十六条 保险责任开始前，被保险人可以要求解除合同，但是应当向保险人支付手续费，保险人应当退还保险费。\\\",\\n民法商法-海商法1992-11-07:    \\\"第二百二十七条 除合同另有约定外，保险责任开始后，被保险人和保险人均不得解除合同。\\\",\\n民法商法-海商法1992-11-07:    \\\"根据合同约定在保险责任开始后可以解除合同的，被保险人要求解除合同，保险人有权收取自保险责任开始之日起至合同解除之日止的保险费，剩余部分予以退还；保险人要求解除合同，应当将自合同解除之日起至保险期间届满之日止的保险费退还被保险人。\\\",\\n请根据以上法律，选择最合适的法律生成问题的合理答复，问题是：\\n小明在购买船只保险时，先后向两家保险公司重复订立了保险合同，使得保险金额总和超过了船只的实际价值。后来，这艘船在一次海上事故中被损坏，小明向其中一家保险公司提出了赔偿请求。\\n\\n根据相关法规，小明能否向其他保险公司提出赔偿请求？如果小明获得赔偿，赔偿金额是否能够超过船只的实际受损价值？\"}, {\"from\": \"assistant\", \"value\": \"根据《海商法》第225条的规定，被保险人对同一保险标的就同一保险事故向几个保险人重复订立合同，而使该保险标的的保险金额总和超过保险标的的价值的，除合同另有约定外，被保险人可以向任何保险人提出赔偿请求。因此，小明可以向其他保险公司提出赔偿请求。\\n\\n另外，根据同一法律的第225条第二款规定，被保险人获得的赔偿金额总和不得超过保险标的的受损价值。因此，在小明获得赔偿的情况下，赔偿金额不得超过船只的实际受损价值。各保险公司应按照其承保的保险金额同保险金额总和的比例承担赔偿责任，任何一个保险公司支付的赔偿金额超过其应当承担的赔偿责任的，有权向未按照其应当承担的赔偿责任支付赔偿金额的保险公司追偿。\"}]}\n",
      "{\"id\": \"ilaw-49\", \"conversations\": [{\"from\": \"human\", \"value\": \"生成一个回答下列问题的段落：“什么是黑洞？它是如何形成的？它的重力有多强？”。 \\n\"}, {\"from\": \"assistant\", \"value\": \"黑洞是一种极为强大的天体，它的引力非常强大，甚至连光都无法逃逸。黑洞形成于某些恒星的末期，在恒星耗尽燃料后发生引力坍缩。在坍缩过程中，恒星的质量会集中在极小的空间内，形成了一个非常密集的物体，即黑洞。黑洞的重力非常强大，以至于它可以扭曲空间和时间。因此，黑洞的重力是非常强大的。\"}]}\n",
      "{\"id\": \"ilaw-50\", \"conversations\": [{\"from\": \"human\", \"value\": \"社会法-妇女权益保障法2022-10-30:    \\\"第四十四条 用人单位在录（聘）用女职工时，应当依法与其签订劳动（聘用）合同或者服务协议，劳动（聘用）合同或者服务协议中应当具备女职工特殊保护条款，并不得规定限制女职工结婚、生育等内容。\\\",\\n社会法-妇女权益保障法2022-10-30:    \\\"职工一方与用人单位订立的集体合同中应当包含男女平等和女职工权益保护相关内容，也可以就相关内容制定专章、附件或者单独订立女职工权益保护专项集体合同。\\\",\\n社会法-妇女权益保障法2022-10-30:    \\\"第四十五条 实行男女同工同酬。妇女在享受福利待遇方面享有与男子平等的权利。\\\",\\n社会法-妇女权益保障法2022-10-30:    \\\"第四十六条 在晋职、晋级、评聘专业技术职称和职务、培训等方面，应当坚持男女平等的原则，不得歧视妇女。\\\",\\n社会法-妇女权益保障法2022-10-30:    \\\"第四十七条 用人单位应当根据妇女的特点，依法保护妇女在工作和劳动时的安全、健康以及休息的权利。\\\",\\n请根据以上法律，选择最合适的法律生成问题的合理答复，问题是：\\n一家公司在招聘女性员工时，要求其签订“禁止结婚、生育”的合同条款，该做法是否合法？\"}, {\"from\": \"assistant\", \"value\": \"根据《社会法-妇女权益保障法》第四十四条规定，“用人单位在录（聘）用女职工时，应当依法与其签订劳动（聘用）合同或者服务协议，劳动（聘用）合同或者服务协议中应当具备女职工特殊保护条款，并不得规定限制女职工结婚、生育等内容。”因此，在招聘女性员工时，禁止结婚、生育的条款是不合法的。\"}]}\n"
     ]
    }
   ],
   "source": [
    "## 手动上传训练数据后，再查看数据，可以看出，与LLama（Alpaca）不同，Belle开源方案的训练代码，考虑到兼容后续的多轮对话，所以训练数据格式如下：\n",
    "##（第一次先用1000条跑一下，跑通了再改成50多万的）\n",
    "!head -n 50 data/ilaw-dev.json"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7eebb718-fc58-41f7-9025-be3f0ae1bc2e",
   "metadata": {},
   "source": [
    "### 1.1、从HuggingFace Hub上，下载BELLE用200万条中文语料训练的Bloomz7b作为我们的训练基座"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "800ad657-f6ac-4190-9b59-1097e368c7ba",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Looking in indexes: https://pypi.org/simple, https://pip.repos.neuron.amazonaws.com\n",
      "Collecting huggingface_hub\n",
      "  Downloading huggingface_hub-0.14.1-py3-none-any.whl (224 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m224.5/224.5 kB\u001b[0m \u001b[31m6.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: pyyaml>=5.1 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from huggingface_hub) (6.0)\n",
      "Requirement already satisfied: packaging>=20.9 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from huggingface_hub) (21.3)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from huggingface_hub) (4.4.0)\n",
      "Requirement already satisfied: requests in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from huggingface_hub) (2.28.1)\n",
      "Requirement already satisfied: tqdm>=4.42.1 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from huggingface_hub) (4.63.2)\n",
      "Requirement already satisfied: filelock in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from huggingface_hub) (3.6.0)\n",
      "Requirement already satisfied: fsspec in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from huggingface_hub) (2022.11.0)\n",
      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from packaging>=20.9->huggingface_hub) (3.0.9)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from requests->huggingface_hub) (3.4)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from requests->huggingface_hub) (2022.12.7)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from requests->huggingface_hub) (1.26.8)\n",
      "Requirement already satisfied: charset-normalizer<3,>=2 in /home/ec2-user/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages (from requests->huggingface_hub) (2.1.1)\n",
      "Installing collected packages: huggingface_hub\n",
      "Successfully installed huggingface_hub-0.14.1\n"
     ]
    }
   ],
   "source": [
    "#安装huggingface库\n",
    "!pip install huggingface_hub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "6aa7d4a5-ff0b-4dcc-8d5f-95d6577445a4",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.06280732154846191,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": null,
       "postfix": null,
       "prefix": "Fetching 5 files",
       "rate": null,
       "total": 5,
       "unit": "it",
       "unit_divisor": 1000,
       "unit_scale": false
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fcca8147aa804ab9bcbdd030f0f5b805",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Fetching 5 files:   0%|          | 0/5 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.00716710090637207,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": null,
       "postfix": null,
       "prefix": "Downloading (…)00dc10a1/config.json",
       "rate": null,
       "total": 742,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7417737627ad4231854f157bafc43511",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)00dc10a1/config.json:   0%|          | 0.00/742 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.031287431716918945,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": null,
       "postfix": null,
       "prefix": "Downloading (…)okenizer_config.json",
       "rate": null,
       "total": 199,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2f6a25bbe4cf40a4978dacba2ad86d45",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)okenizer_config.json:   0%|          | 0.00/199 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.038210153579711914,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": null,
       "postfix": null,
       "prefix": "Downloading (…)cial_tokens_map.json",
       "rate": null,
       "total": 85,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7c1dfd1f0a144693816b292b566ffd7d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)cial_tokens_map.json:   0%|          | 0.00/85.0 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.007468700408935547,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": null,
       "postfix": null,
       "prefix": "Downloading pytorch_model.bin",
       "rate": null,
       "total": 28276106203,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fa4321d2eeeb46feb3c78330fba77924",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading pytorch_model.bin:   0%|          | 0.00/28.3G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.007685422897338867,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": null,
       "postfix": null,
       "prefix": "Downloading tokenizer.json",
       "rate": null,
       "total": 14500438,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d10c810736ab441f9c3f619618752169",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading tokenizer.json:   0%|          | 0.00/14.5M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from huggingface_hub import snapshot_download\n",
    "from pathlib import Path\n",
    "\n",
    "local_cache_path = Path(\"./model\")\n",
    "local_cache_path.mkdir(exist_ok=True)\n",
    "\n",
    "model_name = \"BelleGroup/BELLE-7B-2M\"\n",
    "\n",
    "# 筛选出和pytorch模型文件相关的文件。\n",
    "allow_patterns = [\"*.json\", \"*.pt\", \"*.bin\", \"*.model\"]\n",
    "\n",
    "model_download_path = snapshot_download(\n",
    "    repo_id=model_name,\n",
    "    cache_dir=local_cache_path,\n",
    "    allow_patterns=allow_patterns,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ec7086e7-4a09-4d10-a92e-89e82efc12c9",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "./model/models--BelleGroup--BELLE-7B-2M/snapshots/a9076d928eff1d94fe6b4372ba2bd3a800dc10a1/config.json\n",
      "./model/models--BelleGroup--BELLE-7B-2M/snapshots/a9076d928eff1d94fe6b4372ba2bd3a800dc10a1/\n"
     ]
    }
   ],
   "source": [
    "# 检查并列出模型文件目录（因为第六步的代码，会下载huggingface上项目整体目录结构，所以要用下面的代码列出模型文件的具体位置）\n",
    "import os\n",
    "from glob import glob\n",
    "\n",
    "local_model_path = None\n",
    "\n",
    "paths = os.walk(r'./model')\n",
    "for root, dirs, files in paths:\n",
    "    for file in files:\n",
    "        if file == 'config.json':\n",
    "            print(os.path.join(root,file))\n",
    "            local_model_path = str(os.path.join(root,file))[0:-11]\n",
    "            print(local_model_path)\n",
    "if local_model_path == None:\n",
    "    print(\"Model download may failed, please check prior step!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "26b39f50-8285-4c70-a799-fbe12e9ab810",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
      "                                 Dload  Upload   Total   Spent    Left  Speed\n",
      "  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0\n",
      "100 4176k  100 4176k    0     0  7656k      0 --:--:-- --:--:-- --:--:-- 50.1M\n"
     ]
    }
   ],
   "source": [
    "# 下载s5cmd命令，这个是文件传输提速用的，后续训练时，用是s5cmd可以节约模型传输时间，达到省钱的目的。\n",
    "!curl -L https://github.com/peak/s5cmd/releases/download/v2.0.0/s5cmd_2.0.0_Linux-64bit.tar.gz | tar -xz"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f7a6fecc-8958-404f-8217-cb90b9baebf3",
   "metadata": {},
   "source": [
    "对于下面的脚本：\n",
    "注意local_model_path要配置为第7行代码曾经输出过的目录地址\n",
    "sagemaker_default_bucket是s3存储桶的地址，需要提前建好存储桶，登陆网址https://s3.console.aws.amazon.com/s3 去建立。\n",
    "还要把sagemaker的s3扩展权限打开，让sagemaker有权访问s3资源\n",
    "然后就可以配置下面的代码，把模型通过s5cmd高速传输命令，将我们之前下载好的模型文件存到s3存储桶中备用了。\n",
    "最后删除notebook空间中的model,原因是后续训练会把根目录下所有文件都推到docker里运行，如果把模型推过去会浪费时间，所以要删除notebook里面的model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "f48d9c1e-4329-4bb8-bcc4-23226437cfd3",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cp model/models--BelleGroup--BELLE-7B-2M/snapshots/a9076d928eff1d94fe6b4372ba2bd3a800dc10a1/config.json s3://zmoolb-ryhrefy1ejk8epxttmfkfagrde4w6usw2a-s3alias/BELLE/pretrain/7B/config.json\n",
      "cp model/models--BelleGroup--BELLE-7B-2M/snapshots/a9076d928eff1d94fe6b4372ba2bd3a800dc10a1/special_tokens_map.json s3://zmoolb-ryhrefy1ejk8epxttmfkfagrde4w6usw2a-s3alias/BELLE/pretrain/7B/special_tokens_map.json\n",
      "cp model/models--BelleGroup--BELLE-7B-2M/snapshots/a9076d928eff1d94fe6b4372ba2bd3a800dc10a1/tokenizer_config.json s3://zmoolb-ryhrefy1ejk8epxttmfkfagrde4w6usw2a-s3alias/BELLE/pretrain/7B/tokenizer_config.json\n",
      "cp model/models--BelleGroup--BELLE-7B-2M/snapshots/a9076d928eff1d94fe6b4372ba2bd3a800dc10a1/tokenizer.json s3://zmoolb-ryhrefy1ejk8epxttmfkfagrde4w6usw2a-s3alias/BELLE/pretrain/7B/tokenizer.json\n",
      "cp model/models--BelleGroup--BELLE-7B-2M/snapshots/a9076d928eff1d94fe6b4372ba2bd3a800dc10a1/pytorch_model.bin s3://zmoolb-ryhrefy1ejk8epxttmfkfagrde4w6usw2a-s3alias/BELLE/pretrain/7B/pytorch_model.bin\n"
     ]
    }
   ],
   "source": [
    "%%script env sagemaker_default_bucket=\"zmoolb-ryhrefy1ejk8epxttmfkfagrde4w6usw2a-s3alias\" local_model_path=\"./model/models--BelleGroup--BELLE-7B-2M/snapshots/a9076d928eff1d94fe6b4372ba2bd3a800dc10a1/\" bash\n",
    "\n",
    "chmod +x ./s5cmd\n",
    "./s5cmd sync ${local_model_path} s3://${sagemaker_default_bucket}/BELLE/pretrain/7B/\n",
    "\n",
    "rm -rf model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d5e2a17c-c88d-49d7-9c54-ec3de3cc916b",
   "metadata": {},
   "source": [
    "## 2、配置并保存docker映像"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9ff0a982-ad44-4b5d-b26c-50682576acf8",
   "metadata": {},
   "source": [
    "这一章主要是配置的docker,后续会推送到集群上，相当于在集群上安装docker映像，然后再docker里面训练。"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2c3899a4-afde-481a-b16e-8635c8363849",
   "metadata": {
    "tags": []
   },
   "source": [
    "### 2.1、配置映像"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "2e439c7a-b905-425d-bf1f-dfd22b95e32d",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Writing Dockerfile\n"
     ]
    }
   ],
   "source": [
    "%%writefile Dockerfile\n",
    "## 注意现在只有us-west-2有机器了，所以配置里已经改为us-west-2\n",
    "From 763104351884.dkr.ecr.us-west-2.amazonaws.com/huggingface-pytorch-training:1.13.1-transformers4.26.0-gpu-py39-cu117-ubuntu20.04 \n",
    "\n",
    "ENV LANG=C.UTF-8\n",
    "ENV PYTHONUNBUFFERED=TRUE\n",
    "ENV PYTHONDONTWRITEBYTECODE=TRUE\n",
    "## 正常流程是把开源项目的环境依赖考过去，然后安装就行了，但是这个开源项目的依赖有毛病，会出现pytorch和cuda版本不匹配的情况，所以还是手工安装一遍更保险。\n",
    "#COPY BELLE/requirements.txt ./\n",
    "#RUN python3 -m pip install -r requirements.txt\n",
    "\n",
    "##作废pip命令默认安装的是旧版，所以要用下面的代码安装特定版本的huggingface transfomers\n",
    "##RUN python3 -m pip install git+https://github.com/huggingface/transformers\n",
    "RUN pip3 install transformers==4.28.1\n",
    "## 安装BELLE开源项目所需的库\n",
    "##RUN pip3 install torch torchvision\n",
    "RUN pip3 uninstall -y deepspeed && pip3 install deepspeed==0.9.0\n",
    "RUN pip3 install datasets==2.10.1\n",
    "RUN pip3 install fire==0.5.0\n",
    "RUN pip3 install accelerate==0.17.1\n",
    "RUN pip3 install numpy\n",
    "RUN pip3 install rouge_score\n",
    "RUN pip3 install gensim==3.8.2\n",
    "RUN pip3 install peft==0.2.0\n",
    "RUN pip3 install bitsandbytes==0.37.1\n",
    "RUN pip3 install tqdm==4.65.0\n",
    "RUN pip3 install huggingface_hub==0.13.1\n",
    "# accelerate==0.17.1\n",
    "# bitsandbytes==0.37.1\n",
    "# datasets==2.10.1\n",
    "# fire==0.5.0\n",
    "# huggingface_hub==0.13.1\n",
    "# torch==1.13.0\n",
    "# tqdm==4.65.0\n",
    "# transformers==4.28.1\n",
    "# deepspeed==0.9.0\n",
    "# gradio\n",
    "\n",
    "## Make all local GPUs visible\n",
    "ENV NVIDIA_VISIBLE_DEVICES=\"all\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "028583de-e8d1-4a0b-92d5-12dc99598545",
   "metadata": {},
   "source": [
    "### 2.2、保存并将映像推送到ECR服务"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "eb00e6e1-06d0-424c-8b1d-695206b51dc9",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING! Your password will be stored unencrypted in /home/ec2-user/.docker/config.json.\n",
      "Configure a credential helper to remove this warning. See\n",
      "https://docs.docker.com/engine/reference/commandline/login/#credentials-store\n",
      "\n",
      "Login Succeeded\n"
     ]
    }
   ],
   "source": [
    "!aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin 763104351884.dkr.ecr.us-west-2.amazonaws.com"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "0f0aace3-9692-4cf2-a911-4e9d05ab2230",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING! Your password will be stored unencrypted in /home/ec2-user/.docker/config.json.\n",
      "Configure a credential helper to remove this warning. See\n",
      "https://docs.docker.com/engine/reference/commandline/login/#credentials-store\n",
      "\n",
      "Login Succeeded\n"
     ]
    }
   ],
   "source": [
    "## 登陆亚马逊ECR(Elastic Container Registry)服务,这个服务可以用来管理docker映像。\n",
    "!aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin 049701616856.dkr.ecr.us-west-2.amazonaws.com"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "4c66ba00-2a8b-41e7-ab8d-2534a991df72",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "## 定义repo name, 文件名要包含*sagemaker*\n",
    "repo_name = \"sagemaker-bloomz-demo\""
   ]
  },
  {
   "cell_type": "raw",
   "id": "ca0b3d03-b209-464f-8c9d-cd4e51376bc7",
   "metadata": {},
   "source": [
    "别再运行下面的代码了，已经打好了。后面直接用就行了，也不用管那个重要步骤了。\n",
    "### 「重要步骤」点击上方的File>new>Terminal\n",
    "然后在命令行里输入sudo chmod -R 777 ./SageMaker 这样的话，才能确保下方的docker build命令有权限打包镜像。\n",
    "如果空间不足，则需要在Terminal执行以下命令迁移docker(不要在notebook里执行):\n",
    "sudo systemctl stop docker\n",
    "sudo systemctl stop docker.socket \n",
    "sudo mv /var/lib/docker /home/ec2-user/SageMaker \n",
    "sudo ln -s /home/ec2-user/SageMaker/docker /var/lib/docker \n",
    "sudo systemctl start docker.socket\n",
    "sudo systemctl start docker\n",
    "sudo chmod -R 777 /home/ec2-user/SageMaker/docker\n",
    "sudo chmod -R 777 /home/ec2-user/SageMaker/docker/tmp/\n",
    "sudo chmod -R 777 /home/ec2-user/SageMaker/\n",
    "##注意下面打包过程必须使用配置高一些的机器，例如m5\n",
    "另外，在当前这个notebook打包会有问题，会出现bug：no basic auth credentials\n",
    "我使用alpaca那个文件夹下的官方教程sagemaker-training的笔记本运行同样的代码就没问题。\n",
    "打包结果如下：\n",
    "The push refers to repository [049701616856.dkr.ecr.us-west-2.amazonaws.com/sagemaker-bloomz-demo]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "b1d5e371-d7c7-4b64-aec2-30dad8ef7869",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Login Succeeded\n",
      "Sending build context to Docker daemon  17.02MB\n",
      "Step 1/17 : From 763104351884.dkr.ecr.us-west-2.amazonaws.com/huggingface-pytorch-training:1.13.1-transformers4.26.0-gpu-py39-cu117-ubuntu20.04\n",
      " ---> c5a6ef695006\n",
      "Step 2/17 : ENV LANG=C.UTF-8\n",
      " ---> Using cache\n",
      " ---> 34558066fc2c\n",
      "Step 3/17 : ENV PYTHONUNBUFFERED=TRUE\n",
      " ---> Using cache\n",
      " ---> 3da9ceb9d715\n",
      "Step 4/17 : ENV PYTHONDONTWRITEBYTECODE=TRUE\n",
      " ---> Using cache\n",
      " ---> d2a73e82a200\n",
      "Step 5/17 : RUN python3 -m pip install git+https://github.com/huggingface/transformers\n",
      " ---> Using cache\n",
      " ---> f451d821d4f3\n",
      "Step 6/17 : RUN pip3 uninstall -y deepspeed && pip3 install deepspeed\n",
      " ---> Using cache\n",
      " ---> c870762c90d4\n",
      "Step 7/17 : RUN pip3 install datasets\n",
      " ---> Using cache\n",
      " ---> e9f6da830e2a\n",
      "Step 8/17 : RUN pip3 install fire\n",
      " ---> Using cache\n",
      " ---> 0e228e136f4f\n",
      "Step 9/17 : RUN pip3 install accelerate\n",
      " ---> Using cache\n",
      " ---> 236b1e438810\n",
      "Step 10/17 : RUN pip3 install numpy\n",
      " ---> Using cache\n",
      " ---> b26fdd31aa1f\n",
      "Step 11/17 : RUN pip3 install rouge_score\n",
      " ---> Using cache\n",
      " ---> fdbc0713c5f4\n",
      "Step 12/17 : RUN pip3 install gensim==3.8.2\n",
      " ---> Using cache\n",
      " ---> 3002ce63c292\n",
      "Step 13/17 : RUN pip3 install peft==0.2.0\n",
      " ---> Using cache\n",
      " ---> f6ee2ba9f514\n",
      "Step 14/17 : RUN pip3 install bitsandbytes\n",
      " ---> Using cache\n",
      " ---> 3dc0fd26e7ad\n",
      "Step 15/17 : RUN pip3 install tqdm\n",
      " ---> Using cache\n",
      " ---> 00c0f2ff60ea\n",
      "Step 16/17 : RUN pip3 install huggingface_hub\n",
      " ---> Using cache\n",
      " ---> cfdddaca9819\n",
      "Step 17/17 : ENV NVIDIA_VISIBLE_DEVICES=\"all\"\n",
      " ---> Using cache\n",
      " ---> e6cdd4fafbba\n",
      "Successfully built e6cdd4fafbba\n",
      "Successfully tagged sagemaker-bloomz7b:latest\n",
      "The push refers to repository [049701616856.dkr.ecr.us-west-2.amazonaws.com/sagemaker-bloomz7b]\n",
      "ec2805d9ec2f: Preparing\n",
      "de6cafca0ee5: Preparing\n",
      "d9802e872659: Preparing\n",
      "649d1f9c1b87: Preparing\n",
      "d093955d9f8c: Preparing\n",
      "8b29cd7fb549: Preparing\n",
      "17826e466735: Preparing\n",
      "032a2f427b32: Preparing\n",
      "912f5547366b: Preparing\n",
      "6403ef3b3b2c: Preparing\n",
      "b8aec32d69e6: Preparing\n",
      "47e8050a4855: Preparing\n",
      "f8dae5c3df1e: Preparing\n",
      "e3221f18601a: Preparing\n",
      "b6f286626882: Preparing\n",
      "76fe97d80cdb: Preparing\n",
      "f5f76489fff8: Preparing\n",
      "17826e466735: Waiting\n",
      "621c3f07daa7: Preparing\n",
      "9b484bb42e11: Preparing\n",
      "032a2f427b32: Waiting\n",
      "54c7c0b58471: Preparing\n",
      "c34adc3ab668: Preparing\n",
      "912f5547366b: Waiting\n",
      "bbf651e48b84: Preparing\n",
      "f61045791108: Preparing\n",
      "4e2ac0cda74a: Preparing\n",
      "658a33d555eb: Preparing\n",
      "bd16d9a61a98: Preparing\n",
      "f0c0cd2accfa: Preparing\n",
      "1275469c066c: Preparing\n",
      "b802dd3babf4: Preparing\n",
      "a3834ec63558: Preparing\n",
      "63edcef6dedf: Preparing\n",
      "0154e84cc2dd: Preparing\n",
      "7085d1c151f6: Preparing\n",
      "a77a2104cfb6: Preparing\n",
      "6808e7f9da2f: Preparing\n",
      "3bc059a9dec6: Preparing\n",
      "de783f3fec23: Preparing\n",
      "18ca52d74b2f: Preparing\n",
      "73df6ccd636c: Preparing\n",
      "6738b73ff7a8: Preparing\n",
      "2a8292d9bfcc: Preparing\n",
      "5b75a5ef32a7: Preparing\n",
      "25a5f55a11f0: Preparing\n",
      "707f484816ae: Preparing\n",
      "0430aa1e47d4: Preparing\n",
      "65448e793131: Preparing\n",
      "15af6e2d42ba: Preparing\n",
      "b46caef92993: Preparing\n",
      "53ce33a12646: Preparing\n",
      "aad68760f4ce: Preparing\n",
      "323d67ab1719: Preparing\n",
      "e72743a0fdfe: Preparing\n",
      "3996353f5820: Preparing\n",
      "ea87e0b9c30f: Preparing\n",
      "af18356cdf10: Preparing\n",
      "f6e30dd4497e: Preparing\n",
      "99832d04a153: Preparing\n",
      "a5981ed7a378: Preparing\n",
      "250519a2f830: Preparing\n",
      "6cadbde53f94: Preparing\n",
      "0002c93bdb37: Preparing\n",
      "6403ef3b3b2c: Waiting\n",
      "b8aec32d69e6: Waiting\n",
      "47e8050a4855: Waiting\n",
      "f8dae5c3df1e: Waiting\n",
      "8b29cd7fb549: Waiting\n",
      "73df6ccd636c: Waiting\n",
      "e3221f18601a: Waiting\n",
      "6738b73ff7a8: Waiting\n",
      "b6f286626882: Waiting\n",
      "2a8292d9bfcc: Waiting\n",
      "76fe97d80cdb: Waiting\n",
      "5b75a5ef32a7: Waiting\n",
      "25a5f55a11f0: Waiting\n",
      "f5f76489fff8: Waiting\n",
      "1275469c066c: Waiting\n",
      "621c3f07daa7: Waiting\n",
      "707f484816ae: Waiting\n",
      "b802dd3babf4: Waiting\n",
      "9b484bb42e11: Waiting\n",
      "0430aa1e47d4: Waiting\n",
      "54c7c0b58471: Waiting\n",
      "a3834ec63558: Waiting\n",
      "c34adc3ab668: Waiting\n",
      "65448e793131: Waiting\n",
      "bbf651e48b84: Waiting\n",
      "63edcef6dedf: Waiting\n",
      "15af6e2d42ba: Waiting\n",
      "f61045791108: Waiting\n",
      "4e2ac0cda74a: Waiting\n",
      "0154e84cc2dd: Waiting\n",
      "658a33d555eb: Waiting\n",
      "7085d1c151f6: Waiting\n",
      "bd16d9a61a98: Waiting\n",
      "a77a2104cfb6: Waiting\n",
      "f0c0cd2accfa: Waiting\n",
      "6808e7f9da2f: Waiting\n",
      "3bc059a9dec6: Waiting\n",
      "b46caef92993: Waiting\n",
      "de783f3fec23: Waiting\n",
      "53ce33a12646: Waiting\n",
      "18ca52d74b2f: Waiting\n",
      "aad68760f4ce: Waiting\n",
      "323d67ab1719: Waiting\n",
      "e72743a0fdfe: Waiting\n",
      "3996353f5820: Waiting\n",
      "ea87e0b9c30f: Waiting\n",
      "af18356cdf10: Waiting\n",
      "f6e30dd4497e: Waiting\n",
      "99832d04a153: Waiting\n",
      "a5981ed7a378: Waiting\n",
      "250519a2f830: Waiting\n",
      "0002c93bdb37: Waiting\n",
      "6cadbde53f94: Waiting\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING! Your password will be stored unencrypted in /home/ec2-user/.docker/config.json.\n",
      "Configure a credential helper to remove this warning. See\n",
      "https://docs.docker.com/engine/reference/commandline/login/#credentials-store\n",
      "\n",
      "no basic auth credentials\n"
     ]
    },
    {
     "ename": "CalledProcessError",
     "evalue": "Command 'b'\\n#!/usr/bin/env bash\\n\\n# This script shows how to build the Docker image and push it to ECR to be ready for use\\n# by SageMaker.\\n\\n# The argument to this script is the image name. This will be used as the image on the local\\n# machine and combined with the account and region to form the repository name for ECR.\\n# The name of our algorithm\\nalgorithm_name=${repo_name}\\n\\naccount=$(aws sts get-caller-identity --query Account --output text)\\n\\n# Get the region defined in the current configuration (default to us-west-2 if none defined)\\nregion=$(aws configure get region)\\nregion=${region:-us-west-2}\\n\\nfullname=\"${account}.dkr.ecr.${region}.amazonaws.com/${algorithm_name}:latest\"\\n\\n# If the repository doesn\\'t exist in ECR, create it.\\naws ecr describe-repositories --repository-names \"${algorithm_name}\" > /dev/null 2>&1\\n\\nif [ $? -ne 0 ]\\nthen\\n    aws ecr create-repository --repository-name \"${algorithm_name}\" > /dev/null\\nfi\\n\\n# Get the login command from ECR and execute it directly\\naws ecr get-login-password --region ${region}|docker login --username AWS --password-stdin ${fullname}\\n\\n# Build the docker image locally with the image name and then push it to ECR\\n# with the full name.\\nsudo docker build -t ${algorithm_name} .\\nsudo docker tag ${algorithm_name} ${fullname}\\n\\nsudo docker push ${fullname}\\n'' returned non-zero exit status 1.",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mCalledProcessError\u001b[0m                        Traceback (most recent call last)",
      "\u001b[0;32m/tmp/ipykernel_25544/1236652822.py\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mget_ipython\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_cell_magic\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'script'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'env repo_name=$repo_name bash'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'\\n#!/usr/bin/env bash\\n\\n# This script shows how to build the Docker image and push it to ECR to be ready for use\\n# by SageMaker.\\n\\n# The argument to this script is the image name. This will be used as the image on the local\\n# machine and combined with the account and region to form the repository name for ECR.\\n# The name of our algorithm\\nalgorithm_name=${repo_name}\\n\\naccount=$(aws sts get-caller-identity --query Account --output text)\\n\\n# Get the region defined in the current configuration (default to us-west-2 if none defined)\\nregion=$(aws configure get region)\\nregion=${region:-us-west-2}\\n\\nfullname=\"${account}.dkr.ecr.${region}.amazonaws.com/${algorithm_name}:latest\"\\n\\n# If the repository doesn\\'t exist in ECR, create it.\\naws ecr describe-repositories --repository-names \"${algorithm_name}\" > /dev/null 2>&1\\n\\nif [ $? -ne 0 ]\\nthen\\n    aws ecr create-repository --repository-name \"${algorithm_name}\" > /dev/null\\nfi\\n\\n# Get the login command from ECR and execute it directly\\naws ecr get-login-password --region ${region}|docker login --username AWS --password-stdin ${fullname}\\n\\n# Build the docker image locally with the image name and then push it to ECR\\n# with the full name.\\nsudo docker build -t ${algorithm_name} .\\nsudo docker tag ${algorithm_name} ${fullname}\\n\\nsudo docker push ${fullname}\\n'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages/IPython/core/interactiveshell.py\u001b[0m in \u001b[0;36mrun_cell_magic\u001b[0;34m(self, magic_name, line, cell)\u001b[0m\n\u001b[1;32m   2460\u001b[0m             \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuiltin_trap\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2461\u001b[0m                 \u001b[0margs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mmagic_arg_s\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcell\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2462\u001b[0;31m                 \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2463\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2464\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages/decorator.py\u001b[0m in \u001b[0;36mfun\u001b[0;34m(*args, **kw)\u001b[0m\n\u001b[1;32m    230\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mkwsyntax\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    231\u001b[0m                 \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkw\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfix\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkw\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msig\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 232\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mcaller\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mextras\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkw\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    233\u001b[0m     \u001b[0mfun\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    234\u001b[0m     \u001b[0mfun\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__doc__\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__doc__\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages/IPython/core/magic.py\u001b[0m in \u001b[0;36m<lambda>\u001b[0;34m(f, *a, **k)\u001b[0m\n\u001b[1;32m    185\u001b[0m     \u001b[0;31m# but it's overkill for just that one bit of state.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    186\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mmagic_deco\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 187\u001b[0;31m         \u001b[0mcall\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mlambda\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    188\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    189\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mcallable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages/IPython/core/magics/script.py\u001b[0m in \u001b[0;36mshebang\u001b[0;34m(self, line, cell)\u001b[0m\n\u001b[1;32m    243\u001b[0m             \u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstderr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mflush\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    244\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mraise_error\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreturncode\u001b[0m\u001b[0;34m!=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 245\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mCalledProcessError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreturncode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcell\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutput\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstderr\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0merr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    246\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    247\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_run_script\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcell\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mto_close\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mCalledProcessError\u001b[0m: Command 'b'\\n#!/usr/bin/env bash\\n\\n# This script shows how to build the Docker image and push it to ECR to be ready for use\\n# by SageMaker.\\n\\n# The argument to this script is the image name. This will be used as the image on the local\\n# machine and combined with the account and region to form the repository name for ECR.\\n# The name of our algorithm\\nalgorithm_name=${repo_name}\\n\\naccount=$(aws sts get-caller-identity --query Account --output text)\\n\\n# Get the region defined in the current configuration (default to us-west-2 if none defined)\\nregion=$(aws configure get region)\\nregion=${region:-us-west-2}\\n\\nfullname=\"${account}.dkr.ecr.${region}.amazonaws.com/${algorithm_name}:latest\"\\n\\n# If the repository doesn\\'t exist in ECR, create it.\\naws ecr describe-repositories --repository-names \"${algorithm_name}\" > /dev/null 2>&1\\n\\nif [ $? -ne 0 ]\\nthen\\n    aws ecr create-repository --repository-name \"${algorithm_name}\" > /dev/null\\nfi\\n\\n# Get the login command from ECR and execute it directly\\naws ecr get-login-password --region ${region}|docker login --username AWS --password-stdin ${fullname}\\n\\n# Build the docker image locally with the image name and then push it to ECR\\n# with the full name.\\nsudo docker build -t ${algorithm_name} .\\nsudo docker tag ${algorithm_name} ${fullname}\\n\\nsudo docker push ${fullname}\\n'' returned non-zero exit status 1."
     ]
    }
   ],
   "source": [
    "%%script env repo_name=$repo_name bash\n",
    "\n",
    "#!/usr/bin/env bash\n",
    "\n",
    "# This script shows how to build the Docker image and push it to ECR to be ready for use\n",
    "# by SageMaker.\n",
    "\n",
    "# The argument to this script is the image name. This will be used as the image on the local\n",
    "# machine and combined with the account and region to form the repository name for ECR.\n",
    "# The name of our algorithm\n",
    "algorithm_name=${repo_name}\n",
    "\n",
    "account=$(aws sts get-caller-identity --query Account --output text)\n",
    "\n",
    "# Get the region defined in the current configuration (default to us-west-2 if none defined)\n",
    "region=$(aws configure get region)\n",
    "region=${region:-us-west-2}\n",
    "\n",
    "fullname=\"${account}.dkr.ecr.${region}.amazonaws.com/${algorithm_name}:latest\"\n",
    "\n",
    "# If the repository doesn't exist in ECR, create it.\n",
    "aws ecr describe-repositories --repository-names \"${algorithm_name}\" > /dev/null 2>&1\n",
    "\n",
    "if [ $? -ne 0 ]\n",
    "then\n",
    "    aws ecr create-repository --repository-name \"${algorithm_name}\" > /dev/null\n",
    "fi\n",
    "\n",
    "# Get the login command from ECR and execute it directly\n",
    "aws ecr get-login-password --region ${region}|docker login --username AWS --password-stdin ${fullname}\n",
    "\n",
    "# Build the docker image locally with the image name and then push it to ECR\n",
    "# with the full name.\n",
    "sudo docker build -t ${algorithm_name} .\n",
    "sudo docker tag ${algorithm_name} ${fullname}\n",
    "\n",
    "sudo docker push ${fullname}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fd2aa3e1-3659-4733-b2d4-5ef085ad84f6",
   "metadata": {},
   "source": [
    "## 3、配置训练参数"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6b29095e-64d0-4d48-8058-a07d25af2afc",
   "metadata": {},
   "source": [
    "### 3.1、配置deepspeed配置文件"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "b5b8de0f-0ba9-483d-ae4f-5c86e9d60542",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "#bloomz不支持半精度fp16加速，禁用半精度加速\n",
    "##这个代码先别执行了，感觉用代码改完后，训练时读取会报错。还是本地用vscode\n",
    "import json\n",
    "\n",
    "ds_config_file = '../BELLE/train/configs/deepspeed_config.json'\n",
    "with open (ds_config_file, 'rb') as f:\n",
    "    ds_config = json.load(f)\n",
    "    f.close()\n",
    "    \n",
    "ds_config['fp16']['enabled'] = False\n",
    "\n",
    "with open(ds_config_file, 'w') as f:\n",
    "    json.dump(ds_config, f, indent=2)\n",
    "    f.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0552055e-cb16-4129-b63a-2f4f1ea11406",
   "metadata": {},
   "source": [
    "### 3.2、Generate training entrypoint script\n",
    "\n",
    "**Note: DO NOT CHANGE BELOW VAlUE OF \"output_dir\" and \"cache_dir\", keep it \"/tmp/llama_out\" and \"/tmp\".**"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "834af1d5-a18f-4f95-a0b1-5a6139bd0007",
   "metadata": {},
   "source": [
    "Below is just a testing to fine-tune on a sample dataset (just 8 samples), you could change ```data_path``` to your dataset for furthur fine tune.\n",
    "$MODEL_S3_BUCKET是我们在之前代码中定义的：zmoolb-ryhrefy1ejk8epxttmfkfagrde4w6usw2a-s3alias,后续会在调用时配置为环境变量\n",
    "For the dataset download, you could follow the way how to download pretrain model:\n",
    "```\n",
    "./s5cmd sync s3://$MODEL_S3_BUCKET/BELLE/pretrain/7B/* /tmp/bloomz_pretrain/\n",
    "```\n",
    "\n",
    "It is recommend to use the folder ```/tmp/dataset/```.\n",
    "第一次测试，先设置成1轮，数据集使用999条训练+30条验证，把流程跑通\n",
    "第二次再用正式数据57w+3000条验证"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5769f03b-56d6-4f0e-a4e7-40d7a4aa1f48",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Writing train.sh\n"
     ]
    }
   ],
   "source": [
    "%%writefile train.sh\n",
    "#!/bin/bash\n",
    "#使用s5cmd命令将我们之前保存在s3桶里的模型文件存到集群的/tmp/BELLEbloomz_pretrain/目录\n",
    "chmod +x ./s5cmd\n",
    "./s5cmd sync s3://$MODEL_S3_BUCKET/BELLE/pretrain/7B/* /tmp/BELLEbloomz_pretrain/\n",
    "#第一次测试，为了快速跑通，先设置为1轮\n",
    "torchrun --nproc_per_node 8 --master_port=12345 BELLE/train/src/train.py \\\n",
    "    --model_name_or_path \"/tmp/BELLEbloomz_pretrain/\" \\\n",
    "    --deepspeed \"/BELLE/train/configs/deepspeed_config.json\" \\\n",
    "    --train_file \"/data/ilaw.json\" \\\n",
    "    --validation_file \"/data/ilaw-dev.json\" \\\n",
    "    --per_device_train_batch_size 16 \\\n",
    "    --per_device_eval_batch_size 16 \\\n",
    "    --gradient_accumulation_steps 4 \\\n",
    "    --num_train_epochs 1 \\\n",
    "    --model_max_length 1024 \\\n",
    "    --save_strategy \"steps\" \\\n",
    "    --save_total_limit 3 \\\n",
    "    --learning_rate 8e-5 \\\n",
    "    --weight_decay 0.00001 \\\n",
    "    --warmup_ratio 0.05 \\\n",
    "    --lr_scheduler_type \"cosine\" \\\n",
    "    --logging_steps 10 \\\n",
    "    --evaluation_strategy \"steps\" \\\n",
    "    --fp16 False \\\n",
    "    --seed 1234 \\\n",
    "    --gradient_checkpointing True \\\n",
    "    --cache_dir \"/tmp\" \\\n",
    "    --output_dir \"/tmp/bloomz_out\"\n",
    "\n",
    "if [ $? -eq 1 ]; then\n",
    "    echo \"Training script error, please check CloudWatch logs\"\n",
    "    exit 1\n",
    "fi\n",
    "\n",
    "./s5cmd sync /tmp/bloomz_out s3://$MODEL_S3_BUCKET/bloomz/output/$(date +%Y-%m-%d-%H-%M-%S)/\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "c2a7ed9c-8888-443e-ba43-056f266e2492",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'049701616856.dkr.ecr.us-west-2.amazonaws.com/sagemaker-bloomz-demo:latest'"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## The image uri which is build and pushed above\n",
    "#核实一下名字是不是之前推送的[049701616856.dkr.ecr.us-west-2.amazonaws.com/sagemaker-bloomz-demo]\n",
    "image_uri = \"{}.dkr.ecr.{}.amazonaws.com/{}:latest\".format(account, region, repo_name)\n",
    "image_uri"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6154eaee-6b7c-4c2d-a621-5638aff03a8f",
   "metadata": {},
   "source": [
    "**The modified training script**"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "52a6584d-2c6c-4961-bb47-beb3667e3e60",
   "metadata": {
    "tags": []
   },
   "source": [
    "Everything is ready, let's launch the training job."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "852f2415-1c84-436b-9312-2c9f988fca16",
   "metadata": {},
   "source": [
    "## Create SageMaker Training Job"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "f577ef6b-544c-46c6-9e7e-65bcfc28bf6e",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:botocore.credentials:Found credentials from IAM Role: BaseNotebookInstanceEc2InstanceRole\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using provided s3_resource\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:sagemaker:Creating training-job with name: bloomz20230523v1-2023-05-23-11-03-19-778\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2023-05-23 11:03:26 Starting - Starting the training job......\n",
      "2023-05-23 11:04:03 Starting - Preparing the instances for training.....................\n",
      "2023-05-23 11:07:43 Downloading - Downloading input data...\n",
      "2023-05-23 11:07:58 Training - Downloading the training image.....................\n",
      "2023-05-23 11:11:30 Training - Training image download completed. Training in progress.....\u001b[34mbash: cannot set terminal process group (-1): Inappropriate ioctl for device\u001b[0m\n",
      "\u001b[34mbash: no job control in this shell\u001b[0m\n",
      "\u001b[34m2023-05-23 11:12:25,603 sagemaker-training-toolkit INFO     Imported framework sagemaker_pytorch_container.training\u001b[0m\n",
      "\u001b[34m2023-05-23 11:12:25,666 sagemaker-training-toolkit INFO     No Neurons detected (normal if no neurons installed)\u001b[0m\n",
      "\u001b[34m2023-05-23 11:12:25,675 sagemaker_pytorch_container.training INFO     Block until all host DNS lookups succeed.\u001b[0m\n",
      "\u001b[34m2023-05-23 11:12:25,677 sagemaker_pytorch_container.training INFO     Invoking user training script.\u001b[0m\n",
      "\u001b[34m2023-05-23 11:12:26,644 sagemaker-training-toolkit INFO     No Neurons detected (normal if no neurons installed)\u001b[0m\n",
      "\u001b[34m2023-05-23 11:12:26,716 sagemaker-training-toolkit INFO     No Neurons detected (normal if no neurons installed)\u001b[0m\n",
      "\u001b[34m2023-05-23 11:12:26,787 sagemaker-training-toolkit INFO     No Neurons detected (normal if no neurons installed)\u001b[0m\n",
      "\u001b[34m2023-05-23 11:12:26,796 sagemaker-training-toolkit INFO     Invoking user script\u001b[0m\n",
      "\u001b[34mTraining Env:\u001b[0m\n",
      "\u001b[34m{\n",
      "    \"additional_framework_parameters\": {},\n",
      "    \"channel_input_dirs\": {},\n",
      "    \"current_host\": \"algo-1\",\n",
      "    \"current_instance_group\": \"homogeneousCluster\",\n",
      "    \"current_instance_group_hosts\": [\n",
      "        \"algo-1\"\n",
      "    ],\n",
      "    \"current_instance_type\": \"ml.p4d.24xlarge\",\n",
      "    \"distribution_hosts\": [],\n",
      "    \"distribution_instance_groups\": [],\n",
      "    \"framework_module\": \"sagemaker_pytorch_container.training:main\",\n",
      "    \"hosts\": [\n",
      "        \"algo-1\"\n",
      "    ],\n",
      "    \"hyperparameters\": {},\n",
      "    \"input_config_dir\": \"/opt/ml/input/config\",\n",
      "    \"input_data_config\": {},\n",
      "    \"input_dir\": \"/opt/ml/input\",\n",
      "    \"instance_groups\": [\n",
      "        \"homogeneousCluster\"\n",
      "    ],\n",
      "    \"instance_groups_dict\": {\n",
      "        \"homogeneousCluster\": {\n",
      "            \"instance_group_name\": \"homogeneousCluster\",\n",
      "            \"instance_type\": \"ml.p4d.24xlarge\",\n",
      "            \"hosts\": [\n",
      "                \"algo-1\"\n",
      "            ]\n",
      "        }\n",
      "    },\n",
      "    \"is_hetero\": false,\n",
      "    \"is_master\": true,\n",
      "    \"is_modelparallel_enabled\": null,\n",
      "    \"is_smddpmprun_installed\": true,\n",
      "    \"job_name\": \"bloomz20230523v1-2023-05-23-11-03-19-778\",\n",
      "    \"log_level\": 20,\n",
      "    \"master_hostname\": \"algo-1\",\n",
      "    \"model_dir\": \"/opt/ml/model\",\n",
      "    \"module_dir\": \"s3://sagemaker-us-west-2-049701616856/bloomz20230523v1-2023-05-23-11-03-19-778/source/sourcedir.tar.gz\",\n",
      "    \"module_name\": \"train.sh\",\n",
      "    \"network_interface_name\": \"eth0\",\n",
      "    \"num_cpus\": 96,\n",
      "    \"num_gpus\": 8,\n",
      "    \"num_neurons\": 0,\n",
      "    \"output_data_dir\": \"/opt/ml/output/data\",\n",
      "    \"output_dir\": \"/opt/ml/output\",\n",
      "    \"output_intermediate_dir\": \"/opt/ml/output/intermediate\",\n",
      "    \"resource_config\": {\n",
      "        \"current_host\": \"algo-1\",\n",
      "        \"current_instance_type\": \"ml.p4d.24xlarge\",\n",
      "        \"current_group_name\": \"homogeneousCluster\",\n",
      "        \"hosts\": [\n",
      "            \"algo-1\"\n",
      "        ],\n",
      "        \"instance_groups\": [\n",
      "            {\n",
      "                \"instance_group_name\": \"homogeneousCluster\",\n",
      "                \"instance_type\": \"ml.p4d.24xlarge\",\n",
      "                \"hosts\": [\n",
      "                    \"algo-1\"\n",
      "                ]\n",
      "            }\n",
      "        ],\n",
      "        \"network_interface_name\": \"eth0\"\n",
      "    },\n",
      "    \"user_entry_point\": \"train.sh\"\u001b[0m\n",
      "\u001b[34m}\u001b[0m\n",
      "\u001b[34mEnvironment variables:\u001b[0m\n",
      "\u001b[34mSM_HOSTS=[\"algo-1\"]\u001b[0m\n",
      "\u001b[34mSM_NETWORK_INTERFACE_NAME=eth0\u001b[0m\n",
      "\u001b[34mSM_HPS={}\u001b[0m\n",
      "\u001b[34mSM_USER_ENTRY_POINT=train.sh\u001b[0m\n",
      "\u001b[34mSM_FRAMEWORK_PARAMS={}\u001b[0m\n",
      "\u001b[34mSM_RESOURCE_CONFIG={\"current_group_name\":\"homogeneousCluster\",\"current_host\":\"algo-1\",\"current_instance_type\":\"ml.p4d.24xlarge\",\"hosts\":[\"algo-1\"],\"instance_groups\":[{\"hosts\":[\"algo-1\"],\"instance_group_name\":\"homogeneousCluster\",\"instance_type\":\"ml.p4d.24xlarge\"}],\"network_interface_name\":\"eth0\"}\u001b[0m\n",
      "\u001b[34mSM_INPUT_DATA_CONFIG={}\u001b[0m\n",
      "\u001b[34mSM_OUTPUT_DATA_DIR=/opt/ml/output/data\u001b[0m\n",
      "\u001b[34mSM_CHANNELS=[]\u001b[0m\n",
      "\u001b[34mSM_CURRENT_HOST=algo-1\u001b[0m\n",
      "\u001b[34mSM_CURRENT_INSTANCE_TYPE=ml.p4d.24xlarge\u001b[0m\n",
      "\u001b[34mSM_CURRENT_INSTANCE_GROUP=homogeneousCluster\u001b[0m\n",
      "\u001b[34mSM_CURRENT_INSTANCE_GROUP_HOSTS=[\"algo-1\"]\u001b[0m\n",
      "\u001b[34mSM_INSTANCE_GROUPS=[\"homogeneousCluster\"]\u001b[0m\n",
      "\u001b[34mSM_INSTANCE_GROUPS_DICT={\"homogeneousCluster\":{\"hosts\":[\"algo-1\"],\"instance_group_name\":\"homogeneousCluster\",\"instance_type\":\"ml.p4d.24xlarge\"}}\u001b[0m\n",
      "\u001b[34mSM_DISTRIBUTION_INSTANCE_GROUPS=[]\u001b[0m\n",
      "\u001b[34mSM_IS_HETERO=false\u001b[0m\n",
      "\u001b[34mSM_MODULE_NAME=train.sh\u001b[0m\n",
      "\u001b[34mSM_LOG_LEVEL=20\u001b[0m\n",
      "\u001b[34mSM_FRAMEWORK_MODULE=sagemaker_pytorch_container.training:main\u001b[0m\n",
      "\u001b[34mSM_INPUT_DIR=/opt/ml/input\u001b[0m\n",
      "\u001b[34mSM_INPUT_CONFIG_DIR=/opt/ml/input/config\u001b[0m\n",
      "\u001b[34mSM_OUTPUT_DIR=/opt/ml/output\u001b[0m\n",
      "\u001b[34mSM_NUM_CPUS=96\u001b[0m\n",
      "\u001b[34mSM_NUM_GPUS=8\u001b[0m\n",
      "\u001b[34mSM_NUM_NEURONS=0\u001b[0m\n",
      "\u001b[34mSM_MODEL_DIR=/opt/ml/model\u001b[0m\n",
      "\u001b[34mSM_MODULE_DIR=s3://sagemaker-us-west-2-049701616856/bloomz20230523v1-2023-05-23-11-03-19-778/source/sourcedir.tar.gz\u001b[0m\n",
      "\u001b[34mSM_TRAINING_ENV={\"additional_framework_parameters\":{},\"channel_input_dirs\":{},\"current_host\":\"algo-1\",\"current_instance_group\":\"homogeneousCluster\",\"current_instance_group_hosts\":[\"algo-1\"],\"current_instance_type\":\"ml.p4d.24xlarge\",\"distribution_hosts\":[],\"distribution_instance_groups\":[],\"framework_module\":\"sagemaker_pytorch_container.training:main\",\"hosts\":[\"algo-1\"],\"hyperparameters\":{},\"input_config_dir\":\"/opt/ml/input/config\",\"input_data_config\":{},\"input_dir\":\"/opt/ml/input\",\"instance_groups\":[\"homogeneousCluster\"],\"instance_groups_dict\":{\"homogeneousCluster\":{\"hosts\":[\"algo-1\"],\"instance_group_name\":\"homogeneousCluster\",\"instance_type\":\"ml.p4d.24xlarge\"}},\"is_hetero\":false,\"is_master\":true,\"is_modelparallel_enabled\":null,\"is_smddpmprun_installed\":true,\"job_name\":\"bloomz20230523v1-2023-05-23-11-03-19-778\",\"log_level\":20,\"master_hostname\":\"algo-1\",\"model_dir\":\"/opt/ml/model\",\"module_dir\":\"s3://sagemaker-us-west-2-049701616856/bloomz20230523v1-2023-05-23-11-03-19-778/source/sourcedir.tar.gz\",\"module_name\":\"train.sh\",\"network_interface_name\":\"eth0\",\"num_cpus\":96,\"num_gpus\":8,\"num_neurons\":0,\"output_data_dir\":\"/opt/ml/output/data\",\"output_dir\":\"/opt/ml/output\",\"output_intermediate_dir\":\"/opt/ml/output/intermediate\",\"resource_config\":{\"current_group_name\":\"homogeneousCluster\",\"current_host\":\"algo-1\",\"current_instance_type\":\"ml.p4d.24xlarge\",\"hosts\":[\"algo-1\"],\"instance_groups\":[{\"hosts\":[\"algo-1\"],\"instance_group_name\":\"homogeneousCluster\",\"instance_type\":\"ml.p4d.24xlarge\"}],\"network_interface_name\":\"eth0\"},\"user_entry_point\":\"train.sh\"}\u001b[0m\n",
      "\u001b[34mSM_USER_ARGS=[]\u001b[0m\n",
      "\u001b[34mSM_OUTPUT_INTERMEDIATE_DIR=/opt/ml/output/intermediate\u001b[0m\n",
      "\u001b[34mPYTHONPATH=/opt/ml/code:/opt/conda/bin:/opt/conda/lib/python39.zip:/opt/conda/lib/python3.9:/opt/conda/lib/python3.9/lib-dynload:/opt/conda/lib/python3.9/site-packages\u001b[0m\n",
      "\u001b[34mInvoking script with the following command:\u001b[0m\n",
      "\u001b[34m/bin/sh -c \"./train.sh \"\u001b[0m\n",
      "\u001b[34m[2023-05-23 11:12:30.904: W smdistributed/modelparallel/torch/nn/predefined_hooks.py:78] Found unsupported HuggingFace version 4.28.1 for automated tensor parallelism. HuggingFace modules will not be automatically distributed. You can use smp.tp_register_with_module API to register desired modules for tensor parallelism, or directly instantiate an smp.nn.DistributedModule. Supported HuggingFace transformers versions for automated tensor parallelism: ['4.17.0', '4.20.1', '4.21.0']\u001b[0m\n",
      "\u001b[34m2023-05-23 11:12:30,908 root         INFO     Using NamedTuple = typing._NamedTuple instead.\u001b[0m\n",
      "\u001b[34m2023-05-23 11:12:30,929 sagemaker-training-toolkit INFO     Exceptions not imported for SageMaker TF as Tensorflow is not installed.\u001b[0m\n",
      "\u001b[34mcp s3://zmoolb-ryhrefy1ejk8epxttmfkfagrde4w6usw2a-s3alias/BELLE/pretrain/7B/config.json /tmp/BELLEbloomz_pretrain/config.json\u001b[0m\n",
      "\u001b[34mcp s3://zmoolb-ryhrefy1ejk8epxttmfkfagrde4w6usw2a-s3alias/BELLE/pretrain/7B/special_tokens_map.json /tmp/BELLEbloomz_pretrain/special_tokens_map.json\u001b[0m\n",
      "\u001b[34mcp s3://zmoolb-ryhrefy1ejk8epxttmfkfagrde4w6usw2a-s3alias/BELLE/pretrain/7B/tokenizer_config.json /tmp/BELLEbloomz_pretrain/tokenizer_config.json\u001b[0m\n",
      "\u001b[34mcp s3://zmoolb-ryhrefy1ejk8epxttmfkfagrde4w6usw2a-s3alias/BELLE/pretrain/7B/tokenizer.json /tmp/BELLEbloomz_pretrain/tokenizer.json\u001b[0m\n",
      "\u001b[34mcp s3://zmoolb-ryhrefy1ejk8epxttmfkfagrde4w6usw2a-s3alias/BELLE/pretrain/7B/pytorch_model.bin /tmp/BELLEbloomz_pretrain/pytorch_model.bin\u001b[0m\n",
      "\u001b[34mWARNING:torch.distributed.run:\u001b[0m\n",
      "\u001b[34m*****************************************\u001b[0m\n",
      "\u001b[34mSetting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. \u001b[0m\n",
      "\u001b[34m*****************************************\u001b[0m\n",
      "\u001b[34m===================================BUG REPORT===================================\u001b[0m\n",
      "\u001b[34mWelcome to bitsandbytes. For bug reports, please submit your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\u001b[0m\n",
      "\u001b[34m================================================================================\u001b[0m\n",
      "\u001b[34m===================================BUG REPORT===================================\u001b[0m\n",
      "\u001b[34mWelcome to bitsandbytes. For bug reports, please submit your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\u001b[0m\n",
      "\u001b[34m================================================================================\u001b[0m\n",
      "\u001b[34m===================================BUG REPORT===================================\u001b[0m\n",
      "\u001b[34mWelcome to bitsandbytes. For bug reports, please submit your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\u001b[0m\n",
      "\u001b[34m================================================================================\u001b[0m\n",
      "\u001b[34m===================================BUG REPORT===================================\u001b[0m\n",
      "\u001b[34mWelcome to bitsandbytes. For bug reports, please submit your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\u001b[0m\n",
      "\u001b[34m================================================================================\u001b[0m\n",
      "\u001b[34m===================================BUG REPORT===================================\u001b[0m\n",
      "\u001b[34mWelcome to bitsandbytes. For bug reports, please submit your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\u001b[0m\n",
      "\u001b[34m================================================================================\u001b[0m\n",
      "\u001b[34m===================================BUG REPORT===================================\u001b[0m\n",
      "\u001b[34mWelcome to bitsandbytes. For bug reports, please submit your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\u001b[0m\n",
      "\u001b[34m================================================================================\u001b[0m\n",
      "\u001b[34m===================================BUG REPORT===================================\u001b[0m\n",
      "\u001b[34mWelcome to bitsandbytes. For bug reports, please submit your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\u001b[0m\n",
      "\u001b[34m================================================================================\u001b[0m\n",
      "\u001b[34m===================================BUG REPORT===================================\u001b[0m\n",
      "\u001b[34mWelcome to bitsandbytes. For bug reports, please submit your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\u001b[0m\n",
      "\u001b[34m================================================================================\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/usr/local/nvidia/lib'), PosixPath('/usr/local/nvidia/lib64')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: /opt/conda/lib/python3.9/site-packages/smdistributed/dataparallel/lib:/opt/amazon/openmpi/lib/:/opt/amazon/efa/lib/:/opt/conda/lib:/usr/local/lib:/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/lib did not contain libcudart.so as expected! Searching further paths...\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/v2/credentials/proxy-ab80705c22f121c87f77cfb9c7d4d6d65fa7e7e7ce69781dc2321ab44dedbba1-customer')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/opt/ml/output/intermediate')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('$(dirname $(which conda))/..')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/opt/conda/lib/python39.zip')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('\"algo-1\",\"current_instance_type\"'), PosixPath('true,\"is_modelparallel_enabled\"'), PosixPath('\"ml.p4d.24xlarge\",\"distribution_hosts\"'), PosixPath('\"/opt/ml/input\",\"instance_groups\"'), PosixPath('\"ml.p4d.24xlarge\"}},\"is_hetero\"'), PosixPath('\"bloomz20230523v1-2023-05-23-11-03-19-778\",\"log_level\"'), PosixPath('\"algo-1\",\"model_dir\"'), PosixPath('\"/opt/ml/model\",\"module_dir\"'), PosixPath('[\"algo-1\"],\"current_instance_type\"'), PosixPath('{\"additional_framework_parameters\"'), PosixPath('//sagemaker-us-west-2-049701616856/bloomz20230523v1-2023-05-23-11-03-19-778/source/sourcedir.tar.gz\",\"module_name\"'), PosixPath('[\"homogeneousCluster\"],\"instance_groups_dict\"'), PosixPath('{},\"input_config_dir\"'), PosixPath('\"/opt/ml/output/data\",\"output_dir\"'), PosixPath('main\",\"hosts\"'), PosixPath('\"ml.p4d.24xlarge\"}],\"network_interface_name\"'), PosixPath('{\"homogeneousCluster\"'), PosixPath('\"train.sh\",\"network_interface_name\"'), PosixPath('[],\"framework_module\"'), PosixPath('null,\"is_smddpmprun_installed\"'), PosixPath('{\"current_group_name\"'), PosixPath('96,\"num_gpus\"'), PosixPath('[\"algo-1\"],\"instance_groups\"'), PosixPath('{},\"channel_input_dirs\"'), PosixPath('\"sagemaker_pytorch_container.training'), PosixPath('true,\"job_name\"'), PosixPath('\"/opt/ml/output/intermediate\",\"resource_config\"'), PosixPath('\"homogeneousCluster\",\"instance_type\"'), PosixPath('\"homogeneousCluster\",\"current_host\"'), PosixPath('\"train.sh\"}'), PosixPath('20,\"master_hostname\"'), PosixPath('false,\"is_master\"'), PosixPath('\"s3'), PosixPath('\"eth0\"},\"user_entry_point\"'), PosixPath('{},\"input_dir\"'), PosixPath('[{\"hosts\"'), PosixPath('\"/opt/ml/input/config\",\"input_data_config\"'), PosixPath('[\"algo-1\"],\"instance_group_name\"'), PosixPath('0,\"output_data_dir\"'), PosixPath('{\"hosts\"'), PosixPath('\"homogeneousCluster\",\"current_instance_group_hosts\"'), PosixPath('\"eth0\",\"num_cpus\"'), PosixPath('{},\"current_host\"'), PosixPath('\"/opt/ml/output\",\"output_intermediate_dir\"'), PosixPath('[\"algo-1\"],\"hyperparameters\"'), PosixPath('[],\"distribution_instance_groups\"'), PosixPath('8,\"num_neurons\"'), PosixPath('\"algo-1\",\"current_instance_group\"'), PosixPath('\"ml.p4d.24xlarge\",\"hosts\"')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('s3'), PosixPath('//sagemaker-us-west-2-049701616856/bloomz20230523v1-2023-05-23-11-03-19-778/source/sourcedir.tar.gz')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('sagemaker'), PosixPath('training-job/bloomz20230523v1-2023-05-23-11-03-19-778'), PosixPath('arn'), PosixPath('049701616856'), PosixPath('aws'), PosixPath('us-west-2')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/tmp/torchelastic__s29sqi9/none_laf3sf0f/attempt_0/0/error.json')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34mCUDA_SETUP: WARNING! libcudart.so not found in any environmental path. Searching /usr/local/cuda/lib64...\u001b[0m\n",
      "\u001b[34mCUDA SETUP: CUDA runtime path found: /usr/local/cuda/lib64/libcudart.so\u001b[0m\n",
      "\u001b[34mCUDA SETUP: Highest compute capability among GPUs detected: 8.0\u001b[0m\n",
      "\u001b[34mCUDA SETUP: Detected CUDA version 117\u001b[0m\n",
      "\u001b[34mCUDA SETUP: Loading binary /opt/conda/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cuda117.so...\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/usr/local/nvidia/lib64'), PosixPath('/usr/local/nvidia/lib')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: /opt/conda/lib/python3.9/site-packages/smdistributed/dataparallel/lib:/opt/amazon/openmpi/lib/:/opt/amazon/efa/lib/:/opt/conda/lib:/usr/local/lib:/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/lib did not contain libcudart.so as expected! Searching further paths...\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/v2/credentials/proxy-ab80705c22f121c87f77cfb9c7d4d6d65fa7e7e7ce69781dc2321ab44dedbba1-customer')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/opt/ml/output/intermediate')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('$(dirname $(which conda))/..')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/opt/conda/lib/python39.zip')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('\"train.sh\"}'), PosixPath('\"ml.p4d.24xlarge\"}},\"is_hetero\"'), PosixPath('[\"homogeneousCluster\"],\"instance_groups_dict\"'), PosixPath('{},\"input_dir\"'), PosixPath('\"/opt/ml/output/intermediate\",\"resource_config\"'), PosixPath('[\"algo-1\"],\"hyperparameters\"'), PosixPath('{\"additional_framework_parameters\"'), PosixPath('{},\"input_config_dir\"'), PosixPath('true,\"job_name\"'), PosixPath('{\"homogeneousCluster\"'), PosixPath('{\"current_group_name\"'), PosixPath('[{\"hosts\"'), PosixPath('\"/opt/ml/output/data\",\"output_dir\"'), PosixPath('\"ml.p4d.24xlarge\"}],\"network_interface_name\"'), PosixPath('true,\"is_modelparallel_enabled\"'), PosixPath('//sagemaker-us-west-2-049701616856/bloomz20230523v1-2023-05-23-11-03-19-778/source/sourcedir.tar.gz\",\"module_name\"'), PosixPath('{},\"channel_input_dirs\"'), PosixPath('\"eth0\",\"num_cpus\"'), PosixPath('\"eth0\"},\"user_entry_point\"'), PosixPath('\"bloomz20230523v1-2023-05-23-11-03-19-778\",\"log_level\"'), PosixPath('\"homogeneousCluster\",\"current_instance_group_hosts\"'), PosixPath('\"ml.p4d.24xlarge\",\"hosts\"'), PosixPath('[\"algo-1\"],\"current_instance_type\"'), PosixPath('\"/opt/ml/input\",\"instance_groups\"'), PosixPath('\"algo-1\",\"model_dir\"'), PosixPath('8,\"num_neurons\"'), PosixPath('\"ml.p4d.24xlarge\",\"distribution_hosts\"'), PosixPath('0,\"output_data_dir\"'), PosixPath('\"train.sh\",\"network_interface_name\"'), PosixPath('\"s3'), PosixPath('\"algo-1\",\"current_instance_type\"'), PosixPath('null,\"is_smddpmprun_installed\"'), PosixPath('{},\"current_host\"'), PosixPath('{\"hosts\"'), PosixPath('96,\"num_gpus\"'), PosixPath('[\"algo-1\"],\"instance_groups\"'), PosixPath('\"/opt/ml/model\",\"module_dir\"'), PosixPath('\"/opt/ml/input/config\",\"input_data_config\"'), PosixPath('[],\"framework_module\"'), PosixPath('[\"algo-1\"],\"instance_group_name\"'), PosixPath('20,\"master_hostname\"'), PosixPath('\"homogeneousCluster\",\"current_host\"'), PosixPath('\"sagemaker_pytorch_container.training'), PosixPath('\"algo-1\",\"current_instance_group\"'), PosixPath('\"homogeneousCluster\",\"instance_type\"'), PosixPath('[],\"distribution_instance_groups\"'), PosixPath('main\",\"hosts\"'), PosixPath('false,\"is_master\"'), PosixPath('\"/opt/ml/output\",\"output_intermediate_dir\"')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('s3'), PosixPath('//sagemaker-us-west-2-049701616856/bloomz20230523v1-2023-05-23-11-03-19-778/source/sourcedir.tar.gz')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('arn'), PosixPath('sagemaker'), PosixPath('training-job/bloomz20230523v1-2023-05-23-11-03-19-778'), PosixPath('049701616856'), PosixPath('aws'), PosixPath('us-west-2')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/tmp/torchelastic__s29sqi9/none_laf3sf0f/attempt_0/5/error.json')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34mCUDA_SETUP: WARNING! libcudart.so not found in any environmental path. Searching /usr/local/cuda/lib64...\u001b[0m\n",
      "\u001b[34mCUDA SETUP: CUDA runtime path found: /usr/local/cuda/lib64/libcudart.so\u001b[0m\n",
      "\u001b[34mCUDA SETUP: Highest compute capability among GPUs detected: 8.0\u001b[0m\n",
      "\u001b[34mCUDA SETUP: Detected CUDA version 117\u001b[0m\n",
      "\u001b[34mCUDA SETUP: Loading binary /opt/conda/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cuda117.so...\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/usr/local/nvidia/lib'), PosixPath('/usr/local/nvidia/lib64')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: /opt/conda/lib/python3.9/site-packages/smdistributed/dataparallel/lib:/opt/amazon/openmpi/lib/:/opt/amazon/efa/lib/:/opt/conda/lib:/usr/local/lib:/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/lib did not contain libcudart.so as expected! Searching further paths...\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/v2/credentials/proxy-ab80705c22f121c87f77cfb9c7d4d6d65fa7e7e7ce69781dc2321ab44dedbba1-customer')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/opt/ml/output/intermediate')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('$(dirname $(which conda))/..')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/opt/conda/lib/python39.zip')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('[\"algo-1\"],\"hyperparameters\"'), PosixPath('null,\"is_smddpmprun_installed\"'), PosixPath('\"bloomz20230523v1-2023-05-23-11-03-19-778\",\"log_level\"'), PosixPath('false,\"is_master\"'), PosixPath('8,\"num_neurons\"'), PosixPath('[\"algo-1\"],\"instance_groups\"'), PosixPath('\"homogeneousCluster\",\"current_host\"'), PosixPath('\"homogeneousCluster\",\"instance_type\"'), PosixPath('\"ml.p4d.24xlarge\",\"hosts\"'), PosixPath('\"train.sh\"}'), PosixPath('\"algo-1\",\"current_instance_group\"'), PosixPath('96,\"num_gpus\"'), PosixPath('\"homogeneousCluster\",\"current_instance_group_hosts\"'), PosixPath('{\"hosts\"'), PosixPath('{\"homogeneousCluster\"'), PosixPath('\"sagemaker_pytorch_container.training'), PosixPath('0,\"output_data_dir\"'), PosixPath('\"ml.p4d.24xlarge\"}},\"is_hetero\"'), PosixPath('[\"homogeneousCluster\"],\"instance_groups_dict\"'), PosixPath('//sagemaker-us-west-2-049701616856/bloomz20230523v1-2023-05-23-11-03-19-778/source/sourcedir.tar.gz\",\"module_name\"'), PosixPath('[],\"distribution_instance_groups\"'), PosixPath('\"/opt/ml/input\",\"instance_groups\"'), PosixPath('\"train.sh\",\"network_interface_name\"'), PosixPath('\"algo-1\",\"current_instance_type\"'), PosixPath('\"/opt/ml/output/data\",\"output_dir\"'), PosixPath('[\"algo-1\"],\"instance_group_name\"'), PosixPath('\"/opt/ml/model\",\"module_dir\"'), PosixPath('\"ml.p4d.24xlarge\",\"distribution_hosts\"'), PosixPath('[],\"framework_module\"'), PosixPath('[{\"hosts\"'), PosixPath('\"/opt/ml/output\",\"output_intermediate_dir\"'), PosixPath('{},\"input_dir\"'), PosixPath('\"eth0\",\"num_cpus\"'), PosixPath('{},\"channel_input_dirs\"'), PosixPath('{},\"current_host\"'), PosixPath('\"eth0\"},\"user_entry_point\"'), PosixPath('{},\"input_config_dir\"'), PosixPath('\"s3'), PosixPath('20,\"master_hostname\"'), PosixPath('[\"algo-1\"],\"current_instance_type\"'), PosixPath('\"/opt/ml/output/intermediate\",\"resource_config\"'), PosixPath('true,\"is_modelparallel_enabled\"'), PosixPath('true,\"job_name\"'), PosixPath('\"/opt/ml/input/config\",\"input_data_config\"'), PosixPath('\"ml.p4d.24xlarge\"}],\"network_interface_name\"'), PosixPath('\"algo-1\",\"model_dir\"'), PosixPath('{\"current_group_name\"'), PosixPath('main\",\"hosts\"'), PosixPath('{\"additional_framework_parameters\"')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('//sagemaker-us-west-2-049701616856/bloomz20230523v1-2023-05-23-11-03-19-778/source/sourcedir.tar.gz'), PosixPath('s3')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('us-west-2'), PosixPath('arn'), PosixPath('sagemaker'), PosixPath('aws'), PosixPath('training-job/bloomz20230523v1-2023-05-23-11-03-19-778'), PosixPath('049701616856')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/tmp/torchelastic__s29sqi9/none_laf3sf0f/attempt_0/3/error.json')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34mCUDA_SETUP: WARNING! libcudart.so not found in any environmental path. Searching /usr/local/cuda/lib64...\u001b[0m\n",
      "\u001b[34mCUDA SETUP: CUDA runtime path found: /usr/local/cuda/lib64/libcudart.so\u001b[0m\n",
      "\u001b[34mCUDA SETUP: Highest compute capability among GPUs detected: 8.0\u001b[0m\n",
      "\u001b[34mCUDA SETUP: Detected CUDA version 117\u001b[0m\n",
      "\u001b[34mCUDA SETUP: Loading binary /opt/conda/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cuda117.so...\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/usr/local/nvidia/lib64'), PosixPath('/usr/local/nvidia/lib')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: /opt/conda/lib/python3.9/site-packages/smdistributed/dataparallel/lib:/opt/amazon/openmpi/lib/:/opt/amazon/efa/lib/:/opt/conda/lib:/usr/local/lib:/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/lib did not contain libcudart.so as expected! Searching further paths...\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/v2/credentials/proxy-ab80705c22f121c87f77cfb9c7d4d6d65fa7e7e7ce69781dc2321ab44dedbba1-customer')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/opt/ml/output/intermediate')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('$(dirname $(which conda))/..')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/opt/conda/lib/python39.zip')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('\"ml.p4d.24xlarge\"}],\"network_interface_name\"'), PosixPath('{\"hosts\"'), PosixPath('\"/opt/ml/input/config\",\"input_data_config\"'), PosixPath('[\"algo-1\"],\"instance_group_name\"'), PosixPath('[\"algo-1\"],\"current_instance_type\"'), PosixPath('{},\"input_dir\"'), PosixPath('\"s3'), PosixPath('\"sagemaker_pytorch_container.training'), PosixPath('\"/opt/ml/output\",\"output_intermediate_dir\"'), PosixPath('\"algo-1\",\"model_dir\"'), PosixPath('true,\"is_modelparallel_enabled\"'), PosixPath('[],\"distribution_instance_groups\"'), PosixPath('\"homogeneousCluster\",\"current_host\"'), PosixPath('\"bloomz20230523v1-2023-05-23-11-03-19-778\",\"log_level\"'), PosixPath('[\"algo-1\"],\"hyperparameters\"'), PosixPath('{\"homogeneousCluster\"'), PosixPath('\"/opt/ml/input\",\"instance_groups\"'), PosixPath('false,\"is_master\"'), PosixPath('//sagemaker-us-west-2-049701616856/bloomz20230523v1-2023-05-23-11-03-19-778/source/sourcedir.tar.gz\",\"module_name\"'), PosixPath('\"eth0\"},\"user_entry_point\"'), PosixPath('{\"additional_framework_parameters\"'), PosixPath('\"homogeneousCluster\",\"current_instance_group_hosts\"'), PosixPath('main\",\"hosts\"'), PosixPath('\"/opt/ml/output/intermediate\",\"resource_config\"'), PosixPath('\"homogeneousCluster\",\"instance_type\"'), PosixPath('\"/opt/ml/model\",\"module_dir\"'), PosixPath('[\"algo-1\"],\"instance_groups\"'), PosixPath('{\"current_group_name\"'), PosixPath('\"train.sh\"}'), PosixPath('0,\"output_data_dir\"'), PosixPath('[\"homogeneousCluster\"],\"instance_groups_dict\"'), PosixPath('20,\"master_hostname\"'), PosixPath('\"algo-1\",\"current_instance_type\"'), PosixPath('{},\"input_config_dir\"'), PosixPath('\"algo-1\",\"current_instance_group\"'), PosixPath('[],\"framework_module\"'), PosixPath('8,\"num_neurons\"'), PosixPath('{},\"channel_input_dirs\"'), PosixPath('[{\"hosts\"'), PosixPath('\"ml.p4d.24xlarge\",\"hosts\"'), PosixPath('{},\"current_host\"'), PosixPath('\"ml.p4d.24xlarge\",\"distribution_hosts\"'), PosixPath('96,\"num_gpus\"'), PosixPath('\"ml.p4d.24xlarge\"}},\"is_hetero\"'), PosixPath('null,\"is_smddpmprun_installed\"'), PosixPath('true,\"job_name\"'), PosixPath('\"/opt/ml/output/data\",\"output_dir\"'), PosixPath('\"train.sh\",\"network_interface_name\"'), PosixPath('\"eth0\",\"num_cpus\"')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('s3'), PosixPath('//sagemaker-us-west-2-049701616856/bloomz20230523v1-2023-05-23-11-03-19-778/source/sourcedir.tar.gz')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('aws'), PosixPath('sagemaker'), PosixPath('training-job/bloomz20230523v1-2023-05-23-11-03-19-778'), PosixPath('049701616856'), PosixPath('arn'), PosixPath('us-west-2')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/tmp/torchelastic__s29sqi9/none_laf3sf0f/attempt_0/4/error.json')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34mCUDA_SETUP: WARNING! libcudart.so not found in any environmental path. Searching /usr/local/cuda/lib64...\u001b[0m\n",
      "\u001b[34mCUDA SETUP: CUDA runtime path found: /usr/local/cuda/lib64/libcudart.so\u001b[0m\n",
      "\u001b[34mCUDA SETUP: Highest compute capability among GPUs detected: 8.0\u001b[0m\n",
      "\u001b[34mCUDA SETUP: Detected CUDA version 117\u001b[0m\n",
      "\u001b[34mCUDA SETUP: Loading binary /opt/conda/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cuda117.so...\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/usr/local/nvidia/lib'), PosixPath('/usr/local/nvidia/lib64')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: /opt/conda/lib/python3.9/site-packages/smdistributed/dataparallel/lib:/opt/amazon/openmpi/lib/:/opt/amazon/efa/lib/:/opt/conda/lib:/usr/local/lib:/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/lib did not contain libcudart.so as expected! Searching further paths...\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/v2/credentials/proxy-ab80705c22f121c87f77cfb9c7d4d6d65fa7e7e7ce69781dc2321ab44dedbba1-customer')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/opt/ml/output/intermediate')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('$(dirname $(which conda))/..')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/opt/conda/lib/python39.zip')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('[\"algo-1\"],\"instance_groups\"'), PosixPath('[\"algo-1\"],\"instance_group_name\"'), PosixPath('false,\"is_master\"'), PosixPath('\"/opt/ml/model\",\"module_dir\"'), PosixPath('\"s3'), PosixPath('\"ml.p4d.24xlarge\",\"distribution_hosts\"'), PosixPath('{\"homogeneousCluster\"'), PosixPath('\"homogeneousCluster\",\"current_host\"'), PosixPath('\"bloomz20230523v1-2023-05-23-11-03-19-778\",\"log_level\"'), PosixPath('\"algo-1\",\"current_instance_group\"'), PosixPath('\"ml.p4d.24xlarge\"}],\"network_interface_name\"'), PosixPath('96,\"num_gpus\"'), PosixPath('{},\"current_host\"'), PosixPath('20,\"master_hostname\"'), PosixPath('0,\"output_data_dir\"'), PosixPath('\"eth0\",\"num_cpus\"'), PosixPath('\"sagemaker_pytorch_container.training'), PosixPath('\"/opt/ml/output/intermediate\",\"resource_config\"'), PosixPath('\"train.sh\",\"network_interface_name\"'), PosixPath('[\"algo-1\"],\"current_instance_type\"'), PosixPath('true,\"job_name\"'), PosixPath('null,\"is_smddpmprun_installed\"'), PosixPath('\"algo-1\",\"model_dir\"'), PosixPath('[],\"framework_module\"'), PosixPath('\"ml.p4d.24xlarge\"}},\"is_hetero\"'), PosixPath('{\"additional_framework_parameters\"'), PosixPath('\"/opt/ml/output/data\",\"output_dir\"'), PosixPath('true,\"is_modelparallel_enabled\"'), PosixPath('\"algo-1\",\"current_instance_type\"'), PosixPath('{\"current_group_name\"'), PosixPath('\"homogeneousCluster\",\"instance_type\"'), PosixPath('{},\"input_config_dir\"'), PosixPath('\"homogeneousCluster\",\"current_instance_group_hosts\"'), PosixPath('//sagemaker-us-west-2-049701616856/bloomz20230523v1-2023-05-23-11-03-19-778/source/sourcedir.tar.gz\",\"module_name\"'), PosixPath('\"/opt/ml/output\",\"output_intermediate_dir\"'), PosixPath('\"/opt/ml/input\",\"instance_groups\"'), PosixPath('\"train.sh\"}'), PosixPath('[],\"distribution_instance_groups\"'), PosixPath('main\",\"hosts\"'), PosixPath('\"/opt/ml/input/config\",\"input_data_config\"'), PosixPath('{\"hosts\"'), PosixPath('[{\"hosts\"'), PosixPath('[\"homogeneousCluster\"],\"instance_groups_dict\"'), PosixPath('\"ml.p4d.24xlarge\",\"hosts\"'), PosixPath('\"eth0\"},\"user_entry_point\"'), PosixPath('8,\"num_neurons\"'), PosixPath('{},\"input_dir\"'), PosixPath('{},\"channel_input_dirs\"'), PosixPath('[\"algo-1\"],\"hyperparameters\"')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('s3'), PosixPath('//sagemaker-us-west-2-049701616856/bloomz20230523v1-2023-05-23-11-03-19-778/source/sourcedir.tar.gz')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('049701616856'), PosixPath('aws'), PosixPath('training-job/bloomz20230523v1-2023-05-23-11-03-19-778'), PosixPath('sagemaker'), PosixPath('us-west-2'), PosixPath('arn')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/tmp/torchelastic__s29sqi9/none_laf3sf0f/attempt_0/7/error.json')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34mCUDA_SETUP: WARNING! libcudart.so not found in any environmental path. Searching /usr/local/cuda/lib64...\u001b[0m\n",
      "\u001b[34mCUDA SETUP: CUDA runtime path found: /usr/local/cuda/lib64/libcudart.so\u001b[0m\n",
      "\u001b[34mCUDA SETUP: Highest compute capability among GPUs detected: 8.0\u001b[0m\n",
      "\u001b[34mCUDA SETUP: Detected CUDA version 117\u001b[0m\n",
      "\u001b[34mCUDA SETUP: Loading binary /opt/conda/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cuda117.so...\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/usr/local/nvidia/lib'), PosixPath('/usr/local/nvidia/lib64')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: /opt/conda/lib/python3.9/site-packages/smdistributed/dataparallel/lib:/opt/amazon/openmpi/lib/:/opt/amazon/efa/lib/:/opt/conda/lib:/usr/local/lib:/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/lib did not contain libcudart.so as expected! Searching further paths...\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/v2/credentials/proxy-ab80705c22f121c87f77cfb9c7d4d6d65fa7e7e7ce69781dc2321ab44dedbba1-customer')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/opt/ml/output/intermediate')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('$(dirname $(which conda))/..')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/opt/conda/lib/python39.zip')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('\"homogeneousCluster\",\"current_host\"'), PosixPath('\"/opt/ml/model\",\"module_dir\"'), PosixPath('{},\"input_dir\"'), PosixPath('{\"hosts\"'), PosixPath('\"/opt/ml/output/data\",\"output_dir\"'), PosixPath('\"train.sh\",\"network_interface_name\"'), PosixPath('//sagemaker-us-west-2-049701616856/bloomz20230523v1-2023-05-23-11-03-19-778/source/sourcedir.tar.gz\",\"module_name\"'), PosixPath('{\"current_group_name\"'), PosixPath('0,\"output_data_dir\"'), PosixPath('\"/opt/ml/output\",\"output_intermediate_dir\"'), PosixPath('{},\"channel_input_dirs\"'), PosixPath('[\"algo-1\"],\"current_instance_type\"'), PosixPath('[],\"distribution_instance_groups\"'), PosixPath('{},\"input_config_dir\"'), PosixPath('[],\"framework_module\"'), PosixPath('\"algo-1\",\"model_dir\"'), PosixPath('\"/opt/ml/output/intermediate\",\"resource_config\"'), PosixPath('[{\"hosts\"'), PosixPath('true,\"job_name\"'), PosixPath('{\"homogeneousCluster\"'), PosixPath('{},\"current_host\"'), PosixPath('[\"homogeneousCluster\"],\"instance_groups_dict\"'), PosixPath('8,\"num_neurons\"'), PosixPath('\"/opt/ml/input/config\",\"input_data_config\"'), PosixPath('\"ml.p4d.24xlarge\",\"distribution_hosts\"'), PosixPath('[\"algo-1\"],\"instance_group_name\"'), PosixPath('main\",\"hosts\"'), PosixPath('\"/opt/ml/input\",\"instance_groups\"'), PosixPath('\"sagemaker_pytorch_container.training'), PosixPath('\"ml.p4d.24xlarge\",\"hosts\"'), PosixPath('\"train.sh\"}'), PosixPath('[\"algo-1\"],\"instance_groups\"'), PosixPath('\"eth0\",\"num_cpus\"'), PosixPath('\"homogeneousCluster\",\"current_instance_group_hosts\"'), PosixPath('\"bloomz20230523v1-2023-05-23-11-03-19-778\",\"log_level\"'), PosixPath('\"algo-1\",\"current_instance_type\"'), PosixPath('\"s3'), PosixPath('null,\"is_smddpmprun_installed\"'), PosixPath('20,\"master_hostname\"'), PosixPath('{\"additional_framework_parameters\"'), PosixPath('96,\"num_gpus\"'), PosixPath('\"ml.p4d.24xlarge\"}],\"network_interface_name\"'), PosixPath('\"ml.p4d.24xlarge\"}},\"is_hetero\"'), PosixPath('[\"algo-1\"],\"hyperparameters\"'), PosixPath('\"homogeneousCluster\",\"instance_type\"'), PosixPath('\"eth0\"},\"user_entry_point\"'), PosixPath('true,\"is_modelparallel_enabled\"'), PosixPath('false,\"is_master\"'), PosixPath('\"algo-1\",\"current_instance_group\"')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('s3'), PosixPath('//sagemaker-us-west-2-049701616856/bloomz20230523v1-2023-05-23-11-03-19-778/source/sourcedir.tar.gz')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('aws'), PosixPath('training-job/bloomz20230523v1-2023-05-23-11-03-19-778'), PosixPath('049701616856'), PosixPath('sagemaker'), PosixPath('arn'), PosixPath('us-west-2')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/tmp/torchelastic__s29sqi9/none_laf3sf0f/attempt_0/1/error.json')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34mCUDA_SETUP: WARNING! libcudart.so not found in any environmental path. Searching /usr/local/cuda/lib64...\u001b[0m\n",
      "\u001b[34mCUDA SETUP: CUDA runtime path found: /usr/local/cuda/lib64/libcudart.so\u001b[0m\n",
      "\u001b[34mCUDA SETUP: Highest compute capability among GPUs detected: 8.0\u001b[0m\n",
      "\u001b[34mCUDA SETUP: Detected CUDA version 117\u001b[0m\n",
      "\u001b[34mCUDA SETUP: Loading binary /opt/conda/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cuda117.so...\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/usr/local/nvidia/lib'), PosixPath('/usr/local/nvidia/lib64')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: /opt/conda/lib/python3.9/site-packages/smdistributed/dataparallel/lib:/opt/amazon/openmpi/lib/:/opt/amazon/efa/lib/:/opt/conda/lib:/usr/local/lib:/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/lib did not contain libcudart.so as expected! Searching further paths...\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/v2/credentials/proxy-ab80705c22f121c87f77cfb9c7d4d6d65fa7e7e7ce69781dc2321ab44dedbba1-customer')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/opt/ml/output/intermediate')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('$(dirname $(which conda))/..')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/opt/conda/lib/python39.zip')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('{},\"input_dir\"'), PosixPath('false,\"is_master\"'), PosixPath('\"/opt/ml/input\",\"instance_groups\"'), PosixPath('\"ml.p4d.24xlarge\",\"distribution_hosts\"'), PosixPath('{},\"input_config_dir\"'), PosixPath('//sagemaker-us-west-2-049701616856/bloomz20230523v1-2023-05-23-11-03-19-778/source/sourcedir.tar.gz\",\"module_name\"'), PosixPath('20,\"master_hostname\"'), PosixPath('\"s3'), PosixPath('[\"algo-1\"],\"hyperparameters\"'), PosixPath('[\"homogeneousCluster\"],\"instance_groups_dict\"'), PosixPath('[\"algo-1\"],\"instance_groups\"'), PosixPath('\"ml.p4d.24xlarge\"}],\"network_interface_name\"'), PosixPath('{\"homogeneousCluster\"'), PosixPath('{},\"current_host\"'), PosixPath('8,\"num_neurons\"'), PosixPath('\"bloomz20230523v1-2023-05-23-11-03-19-778\",\"log_level\"'), PosixPath('[\"algo-1\"],\"instance_group_name\"'), PosixPath('96,\"num_gpus\"'), PosixPath('[{\"hosts\"'), PosixPath('true,\"is_modelparallel_enabled\"'), PosixPath('\"homogeneousCluster\",\"current_instance_group_hosts\"'), PosixPath('true,\"job_name\"'), PosixPath('\"/opt/ml/input/config\",\"input_data_config\"'), PosixPath('0,\"output_data_dir\"'), PosixPath('\"eth0\"},\"user_entry_point\"'), PosixPath('\"homogeneousCluster\",\"current_host\"'), PosixPath('\"algo-1\",\"model_dir\"'), PosixPath('[],\"distribution_instance_groups\"'), PosixPath('\"ml.p4d.24xlarge\"}},\"is_hetero\"'), PosixPath('\"homogeneousCluster\",\"instance_type\"'), PosixPath('{\"current_group_name\"'), PosixPath('null,\"is_smddpmprun_installed\"'), PosixPath('\"/opt/ml/output/data\",\"output_dir\"'), PosixPath('{\"hosts\"'), PosixPath('\"/opt/ml/model\",\"module_dir\"'), PosixPath('[\"algo-1\"],\"current_instance_type\"'), PosixPath('\"/opt/ml/output\",\"output_intermediate_dir\"'), PosixPath('\"algo-1\",\"current_instance_type\"'), PosixPath('\"algo-1\",\"current_instance_group\"'), PosixPath('[],\"framework_module\"'), PosixPath('{},\"channel_input_dirs\"'), PosixPath('{\"additional_framework_parameters\"'), PosixPath('\"train.sh\",\"network_interface_name\"'), PosixPath('\"eth0\",\"num_cpus\"'), PosixPath('\"train.sh\"}'), PosixPath('main\",\"hosts\"'), PosixPath('\"sagemaker_pytorch_container.training'), PosixPath('\"/opt/ml/output/intermediate\",\"resource_config\"'), PosixPath('\"ml.p4d.24xlarge\",\"hosts\"')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('s3'), PosixPath('//sagemaker-us-west-2-049701616856/bloomz20230523v1-2023-05-23-11-03-19-778/source/sourcedir.tar.gz')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('049701616856'), PosixPath('arn'), PosixPath('training-job/bloomz20230523v1-2023-05-23-11-03-19-778'), PosixPath('us-west-2'), PosixPath('aws'), PosixPath('sagemaker')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/tmp/torchelastic__s29sqi9/none_laf3sf0f/attempt_0/6/error.json')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34mCUDA_SETUP: WARNING! libcudart.so not found in any environmental path. Searching /usr/local/cuda/lib64...\u001b[0m\n",
      "\u001b[34mCUDA SETUP: CUDA runtime path found: /usr/local/cuda/lib64/libcudart.so\u001b[0m\n",
      "\u001b[34mCUDA SETUP: Highest compute capability among GPUs detected: 8.0\u001b[0m\n",
      "\u001b[34mCUDA SETUP: Detected CUDA version 117\u001b[0m\n",
      "\u001b[34mCUDA SETUP: Loading binary /opt/conda/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cuda117.so...\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/usr/local/nvidia/lib'), PosixPath('/usr/local/nvidia/lib64')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: /opt/conda/lib/python3.9/site-packages/smdistributed/dataparallel/lib:/opt/amazon/openmpi/lib/:/opt/amazon/efa/lib/:/opt/conda/lib:/usr/local/lib:/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/lib did not contain libcudart.so as expected! Searching further paths...\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/v2/credentials/proxy-ab80705c22f121c87f77cfb9c7d4d6d65fa7e7e7ce69781dc2321ab44dedbba1-customer')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/opt/ml/output/intermediate')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('$(dirname $(which conda))/..')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/opt/conda/lib/python39.zip')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('\"eth0\"},\"user_entry_point\"'), PosixPath('[\"algo-1\"],\"current_instance_type\"'), PosixPath('{\"additional_framework_parameters\"'), PosixPath('20,\"master_hostname\"'), PosixPath('main\",\"hosts\"'), PosixPath('96,\"num_gpus\"'), PosixPath('0,\"output_data_dir\"'), PosixPath('[],\"framework_module\"'), PosixPath('[{\"hosts\"'), PosixPath('\"train.sh\"}'), PosixPath('\"eth0\",\"num_cpus\"'), PosixPath('\"/opt/ml/output/data\",\"output_dir\"'), PosixPath('\"ml.p4d.24xlarge\",\"distribution_hosts\"'), PosixPath('[],\"distribution_instance_groups\"'), PosixPath('\"/opt/ml/output/intermediate\",\"resource_config\"'), PosixPath('[\"algo-1\"],\"instance_groups\"'), PosixPath('true,\"job_name\"'), PosixPath('\"algo-1\",\"current_instance_type\"'), PosixPath('{},\"input_dir\"'), PosixPath('false,\"is_master\"'), PosixPath('true,\"is_modelparallel_enabled\"'), PosixPath('\"ml.p4d.24xlarge\",\"hosts\"'), PosixPath('{},\"input_config_dir\"'), PosixPath('\"/opt/ml/input\",\"instance_groups\"'), PosixPath('\"train.sh\",\"network_interface_name\"'), PosixPath('\"algo-1\",\"current_instance_group\"'), PosixPath('null,\"is_smddpmprun_installed\"'), PosixPath('8,\"num_neurons\"'), PosixPath('\"ml.p4d.24xlarge\"}],\"network_interface_name\"'), PosixPath('\"bloomz20230523v1-2023-05-23-11-03-19-778\",\"log_level\"'), PosixPath('\"/opt/ml/output\",\"output_intermediate_dir\"'), PosixPath('{\"current_group_name\"'), PosixPath('\"/opt/ml/input/config\",\"input_data_config\"'), PosixPath('[\"homogeneousCluster\"],\"instance_groups_dict\"'), PosixPath('{},\"channel_input_dirs\"'), PosixPath('\"homogeneousCluster\",\"current_instance_group_hosts\"'), PosixPath('\"homogeneousCluster\",\"current_host\"'), PosixPath('{\"hosts\"'), PosixPath('\"s3'), PosixPath('{\"homogeneousCluster\"'), PosixPath('\"sagemaker_pytorch_container.training'), PosixPath('//sagemaker-us-west-2-049701616856/bloomz20230523v1-2023-05-23-11-03-19-778/source/sourcedir.tar.gz\",\"module_name\"'), PosixPath('\"ml.p4d.24xlarge\"}},\"is_hetero\"'), PosixPath('[\"algo-1\"],\"instance_group_name\"'), PosixPath('\"homogeneousCluster\",\"instance_type\"'), PosixPath('\"algo-1\",\"model_dir\"'), PosixPath('\"/opt/ml/model\",\"module_dir\"'), PosixPath('[\"algo-1\"],\"hyperparameters\"'), PosixPath('{},\"current_host\"')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('s3'), PosixPath('//sagemaker-us-west-2-049701616856/bloomz20230523v1-2023-05-23-11-03-19-778/source/sourcedir.tar.gz')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('us-west-2'), PosixPath('aws'), PosixPath('training-job/bloomz20230523v1-2023-05-23-11-03-19-778'), PosixPath('sagemaker'), PosixPath('arn'), PosixPath('049701616856')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/tmp/torchelastic__s29sqi9/none_laf3sf0f/attempt_0/2/error.json')}\n",
      "  warn(msg)\u001b[0m\n",
      "\u001b[34mCUDA_SETUP: WARNING! libcudart.so not found in any environmental path. Searching /usr/local/cuda/lib64...\u001b[0m\n",
      "\u001b[34mCUDA SETUP: CUDA runtime path found: /usr/local/cuda/lib64/libcudart.so\u001b[0m\n",
      "\u001b[34mCUDA SETUP: Highest compute capability among GPUs detected: 8.0\u001b[0m\n",
      "\u001b[34mCUDA SETUP: Detected CUDA version 117\u001b[0m\n",
      "\u001b[34mCUDA SETUP: Loading binary /opt/conda/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cuda117.so...\u001b[0m\n",
      "\u001b[34m[2023-05-23 11:13:58,400] [INFO] [comm.py:586:init_distributed] Initializing TorchBackend in DeepSpeed with backend nccl\u001b[0m\n",
      "\u001b[34mUsing the `WANDB_DISABLED` environment variable is deprecated and will be removed in v5. Use the --report_to flag to control the integrations used for logging result (for instance --report_to none).\u001b[0m\n",
      "\u001b[34mUsing the `WANDB_DISABLED` environment variable is deprecated and will be removed in v5. Use the --report_to flag to control the integrations used for logging result (for instance --report_to none).\u001b[0m\n",
      "\u001b[34mUsing the `WANDB_DISABLED` environment variable is deprecated and will be removed in v5. Use the --report_to flag to control the integrations used for logging result (for instance --report_to none).\u001b[0m\n",
      "\u001b[34mUsing the `WANDB_DISABLED` environment variable is deprecated and will be removed in v5. Use the --report_to flag to control the integrations used for logging result (for instance --report_to none).\u001b[0m\n",
      "\u001b[34mUsing the `WANDB_DISABLED` environment variable is deprecated and will be removed in v5. Use the --report_to flag to control the integrations used for logging result (for instance --report_to none).\u001b[0m\n",
      "\u001b[34mUsing the `WANDB_DISABLED` environment variable is deprecated and will be removed in v5. Use the --report_to flag to control the integrations used for logging result (for instance --report_to none).\u001b[0m\n",
      "\u001b[34mUsing the `WANDB_DISABLED` environment variable is deprecated and will be removed in v5. Use the --report_to flag to control the integrations used for logging result (for instance --report_to none).\u001b[0m\n",
      "\u001b[34mUsing the `WANDB_DISABLED` environment variable is deprecated and will be removed in v5. Use the --report_to flag to control the integrations used for logging result (for instance --report_to none).\u001b[0m\n",
      "\u001b[34mUsing the `WANDB_DISABLED` environment variable is deprecated and will be removed in v5. Use the --report_to flag to control the integrations used for logging result (for instance --report_to none).\u001b[0m\n",
      "\u001b[34mUsing the `WANDB_DISABLED` environment variable is deprecated and will be removed in v5. Use the --report_to flag to control the integrations used for logging result (for instance --report_to none).\u001b[0m\n",
      "\u001b[34mUsing the `WANDB_DISABLED` environment variable is deprecated and will be removed in v5. Use the --report_to flag to control the integrations used for logging result (for instance --report_to none).\u001b[0m\n",
      "\u001b[34mUsing the `WANDB_DISABLED` environment variable is deprecated and will be removed in v5. Use the --report_to flag to control the integrations used for logging result (for instance --report_to none).\u001b[0m\n",
      "\u001b[34mUsing the `WANDB_DISABLED` environment variable is deprecated and will be removed in v5. Use the --report_to flag to control the integrations used for logging result (for instance --report_to none).\u001b[0m\n",
      "\u001b[34mUsing the `WANDB_DISABLED` environment variable is deprecated and will be removed in v5. Use the --report_to flag to control the integrations used for logging result (for instance --report_to none).\u001b[0m\n",
      "\u001b[34mUsing the `WANDB_DISABLED` environment variable is deprecated and will be removed in v5. Use the --report_to flag to control the integrations used for logging result (for instance --report_to none).\u001b[0m\n",
      "\u001b[34mUsing the `WANDB_DISABLED` environment variable is deprecated and will be removed in v5. Use the --report_to flag to control the integrations used for logging result (for instance --report_to none).\u001b[0m\n",
      "\u001b[34m╭───────────────────── Traceback (most recent call last) ──────────────────────╮\u001b[0m\n",
      "\u001b[34m│ /opt/ml/code/BELLE/train/src/train.py:394 in <module>                        │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│   391                                                                        │\u001b[0m\n",
      "\u001b[34m│   392                                                                        │\u001b[0m\n",
      "\u001b[34m│   393 if __name__ == \"__main__\":                                             │\u001b[0m\n",
      "\u001b[34m│ ❱ 394 │   main()                                                             │\u001b[0m\n",
      "\u001b[34m│   395                                                                        │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/ml/code/BELLE/train/src/train.py:163 in main                            │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│   160                                                                        │\u001b[0m\n",
      "\u001b[34m│   161 def main():                                                            │\u001b[0m\n",
      "\u001b[34m│   162 │   parser = HfArgumentParser((ModelArguments, DataArguments, Training │\u001b[0m\n",
      "\u001b[34m│ ❱ 163 │   model_args, data_args, training_args = parser.parse_args_into_data │\u001b[0m\n",
      "\u001b[34m│   164 │                                                                      │\u001b[0m\n",
      "\u001b[34m│   165 │   world_size = int(os.environ.get(\"WORLD_SIZE\", 1))                  │\u001b[0m\n",
      "\u001b[34m│   166 │   ddp = world_size != 1                                              │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/conda/lib/python3.9/site-packages/transformers/hf_argparser.py:332 in   │\u001b[0m\n",
      "\u001b[34m│ parse_args_into_dataclasses                                                  │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│   329 │   │   │   inputs = {k: v for k, v in vars(namespace).items() if k in │\u001b[0m\n",
      "\u001b[34m│   330 │   │   │   for k in keys:                                             │\u001b[0m\n",
      "\u001b[34m│   331 │   │   │   │   delattr(namespace, k)                                  │\u001b[0m\n",
      "\u001b[34m│ ❱ 332 │   │   │   obj = dtype(**inputs)                                      │\u001b[0m\n",
      "\u001b[34m│   333 │   │   │   outputs.append(obj)                                        │\u001b[0m\n",
      "\u001b[34m│   334 │   │   if len(namespace.__dict__) > 0:                                │\u001b[0m\n",
      "\u001b[34m│   335 │   │   │   # additional namespace.                                    │\u001b[0m\n",
      "\u001b[34m│ <string>:114 in __init__                                                     │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/conda/lib/python3.9/site-packages/transformers/training_args.py:1446 in │\u001b[0m\n",
      "\u001b[34m│ __post_init__                                                                │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│   1443 │   │   │                                                             │\u001b[0m\n",
      "\u001b[34m│   1444 │   │   │   # will be used later by the Trainer                       │\u001b[0m\n",
      "\u001b[34m│   1445 │   │   │   # note: leave self.deepspeed unmodified in case a user re │\u001b[0m\n",
      "\u001b[34m│ ❱ 1446 │   │   │   self.hf_deepspeed_config = HfTrainerDeepSpeedConfig(self. │\u001b[0m\n",
      "\u001b[34m│   1447 │   │   │   self.hf_deepspeed_config.trainer_config_process(self)     │\u001b[0m\n",
      "\u001b[34m│   1448 │   │                                                                 │\u001b[0m\n",
      "\u001b[34m│   1449 │   │   if self.push_to_hub_token is not None:                        │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/conda/lib/python3.9/site-packages/transformers/deepspeed.py:77 in       │\u001b[0m\n",
      "\u001b[34m│ __init__                                                                     │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│    74 │   \"\"\"                                                                │\u001b[0m\n",
      "\u001b[34m│    75 │                                                                      │\u001b[0m\n",
      "\u001b[34m│    76 │   def __init__(self, config_file_or_dict):                           │\u001b[0m\n",
      "\u001b[34m│ ❱  77 │   │   super().__init__(config_file_or_dict)                          │\u001b[0m\n",
      "\u001b[34m│    78 │   │   self._dtype = None                                             │\u001b[0m\n",
      "\u001b[34m│    79 │   │   self.mismatches = []                                           │\u001b[0m\n",
      "\u001b[34m│    80                                                                        │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/conda/lib/python3.9/site-packages/transformers/deepspeed.py:67 in       │\u001b[0m\n",
      "\u001b[34m│ __init__                                                                     │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│    64 │   │   set_hf_deepspeed_config(self)                                  │\u001b[0m\n",
      "\u001b[34m│    65 │   │   dep_version_check(\"accelerate\")                                │\u001b[0m\n",
      "\u001b[34m│    66 │   │   dep_version_check(\"deepspeed\")                                 │\u001b[0m\n",
      "\u001b[34m│ ❱  67 │   │   super().__init__(config_file_or_dict)                          │\u001b[0m\n",
      "\u001b[34m│    68                                                                        │\u001b[0m\n",
      "\u001b[34m│    69                                                                        │\u001b[0m\n",
      "\u001b[34m│    70 class HfTrainerDeepSpeedConfig(HfDeepSpeedConfig):                     │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/conda/lib/python3.9/site-packages/accelerate/utils/deepspeed.py:52 in   │\u001b[0m\n",
      "\u001b[34m│ __init__                                                                     │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│    49 │   │   │   │   config = json.load(f)                                  │\u001b[0m\n",
      "\u001b[34m│    50 │   │   else:                                                          │\u001b[0m\n",
      "\u001b[34m│    51 │   │   │   try:                                                       │\u001b[0m\n",
      "\u001b[34m│ ❱  52 │   │   │   │   config_decoded = base64.urlsafe_b64decode(config_file_ │\u001b[0m\n",
      "\u001b[34m│    53 │   │   │   │   config = json.loads(config_decoded)                    │\u001b[0m\n",
      "\u001b[34m│    54 │   │   │   except (UnicodeDecodeError, AttributeError):               │\u001b[0m\n",
      "\u001b[34m│    55 │   │   │   │   raise ValueError(                                      │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/conda/lib/python3.9/base64.py:133 in urlsafe_b64decode                  │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│   130 │   \"\"\"                                                                │\u001b[0m\n",
      "\u001b[34m│   131 │   s = _bytes_from_decode_data(s)                                     │\u001b[0m\n",
      "\u001b[34m│   132 │   s = s.translate(_urlsafe_decode_translation)                       │\u001b[0m\n",
      "\u001b[34m│ ❱ 133 │   return b64decode(s)                                                │\u001b[0m\n",
      "\u001b[34m│   134                                                                        │\u001b[0m\n",
      "\u001b[34m│   135                                                                        │\u001b[0m\n",
      "\u001b[34m│   136                                                                        │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/conda/lib/python3.9/base64.py:87 in b64decode                           │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│    84 │   │   s = s.translate(bytes.maketrans(altchars, b'+/'))              │\u001b[0m\n",
      "\u001b[34m│    85 │   if validate and not re.fullmatch(b'[A-Za-z0-9+/]*={0,2}', s):      │\u001b[0m\n",
      "\u001b[34m│    86 │   │   raise binascii.Error('Non-base64 digit found')                 │\u001b[0m\n",
      "\u001b[34m│ ❱  87 │   return binascii.a2b_base64(s)                                      │\u001b[0m\n",
      "\u001b[34m│    88                                                                        │\u001b[0m\n",
      "\u001b[34m│    89                                                                        │\u001b[0m\n",
      "\u001b[34m│    90 def standard_b64encode(s):                                             │\u001b[0m\n",
      "\u001b[34m╰──────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n",
      "\u001b[34mError: Invalid base64-encoded string: number of data characters (41) cannot be 1\u001b[0m\n",
      "\u001b[34mmore than a multiple of 4\u001b[0m\n",
      "\u001b[34m╭───────────────────── Traceback (most recent call last) ──────────────────────╮\u001b[0m\n",
      "\u001b[34m│ /opt/ml/code/BELLE/train/src/train.py:394 in <module>                        │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│   391                                                                        │\u001b[0m\n",
      "\u001b[34m│   392                                                                        │\u001b[0m\n",
      "\u001b[34m│   393 if __name__ == \"__main__\":                                             │\u001b[0m\n",
      "\u001b[34m│ ❱ 394 │   main()                                                             │\u001b[0m\n",
      "\u001b[34m│   395                                                                        │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/ml/code/BELLE/train/src/train.py:163 in main                            │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│   160                                                                        │\u001b[0m\n",
      "\u001b[34m│   161 def main():                                                            │\u001b[0m\n",
      "\u001b[34m│   162 │   parser = HfArgumentParser((ModelArguments, DataArguments, Training │\u001b[0m\n",
      "\u001b[34m│ ❱ 163 │   model_args, data_args, training_args = parser.parse_args_into_data │\u001b[0m\n",
      "\u001b[34m│   164 │                                                                      │\u001b[0m\n",
      "\u001b[34m│   165 │   world_size = int(os.environ.get(\"WORLD_SIZE\", 1))                  │\u001b[0m\n",
      "\u001b[34m│   166 │   ddp = world_size != 1                                              │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/conda/lib/python3.9/site-packages/transformers/hf_argparser.py:332 in   │\u001b[0m\n",
      "\u001b[34m│ parse_args_into_dataclasses                                                  │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│   329 │   │   │   inputs = {k: v for k, v in vars(namespace).items() if k in │\u001b[0m\n",
      "\u001b[34m│   330 │   │   │   for k in keys:                                             │\u001b[0m\n",
      "\u001b[34m│   331 │   │   │   │   delattr(namespace, k)                                  │\u001b[0m\n",
      "\u001b[34m│ ❱ 332 │   │   │   obj = dtype(**inputs)                                      │\u001b[0m\n",
      "\u001b[34m│   333 │   │   │   outputs.append(obj)                                        │\u001b[0m\n",
      "\u001b[34m│   334 │   │   if len(namespace.__dict__) > 0:                                │\u001b[0m\n",
      "\u001b[34m│   335 │   │   │   # additional namespace.                                    │\u001b[0m\n",
      "\u001b[34m│ <string>:114 in __init__                                                     │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/conda/lib/python3.9/site-packages/transformers/training_args.py:1446 in │\u001b[0m\n",
      "\u001b[34m│ __post_init__                                                                │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│   1443 │   │   │                                                             │\u001b[0m\n",
      "\u001b[34m│   1444 │   │   │   # will be used later by the Trainer                       │\u001b[0m\n",
      "\u001b[34m│   1445 │   │   │   # note: leave self.deepspeed unmodified in case a user re │\u001b[0m\n",
      "\u001b[34m│ ❱ 1446 │   │   │   self.hf_deepspeed_config = HfTrainerDeepSpeedConfig(self. │\u001b[0m\n",
      "\u001b[34m│   1447 │   │   │   self.hf_deepspeed_config.trainer_config_process(self)     │\u001b[0m\n",
      "\u001b[34m│   1448 │   │                                                                 │\u001b[0m\n",
      "\u001b[34m│   1449 │   │   if self.push_to_hub_token is not None:                        │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/conda/lib/python3.9/site-packages/transformers/deepspeed.py:77 in       │\u001b[0m\n",
      "\u001b[34m│ __init__                                                                     │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│    74 │   \"\"\"                                                                │\u001b[0m\n",
      "\u001b[34m│    75 │                                                                      │\u001b[0m\n",
      "\u001b[34m│    76 │   def __init__(self, config_file_or_dict):                           │\u001b[0m\n",
      "\u001b[34m│ ❱  77 │   │   super().__init__(config_file_or_dict)                          │\u001b[0m\n",
      "\u001b[34m│    78 │   │   self._dtype = None                                             │\u001b[0m\n",
      "\u001b[34m│    79 │   │   self.mismatches = []                                           │\u001b[0m\n",
      "\u001b[34m│    80                                                                        │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/conda/lib/python3.9/site-packages/transformers/deepspeed.py:67 in       │\u001b[0m\n",
      "\u001b[34m│ __init__                                                                     │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│    64 │   │   set_hf_deepspeed_config(self)                                  │\u001b[0m\n",
      "\u001b[34m│    65 │   │   dep_version_check(\"accelerate\")                                │\u001b[0m\n",
      "\u001b[34m│    66 │   │   dep_version_check(\"deepspeed\")                                 │\u001b[0m\n",
      "\u001b[34m│ ❱  67 │   │   super().__init__(config_file_or_dict)                          │\u001b[0m\n",
      "\u001b[34m│    68                                                                        │\u001b[0m\n",
      "\u001b[34m│    69                                                                        │\u001b[0m\n",
      "\u001b[34m│    70 class HfTrainerDeepSpeedConfig(HfDeepSpeedConfig):                     │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/conda/lib/python3.9/site-packages/accelerate/utils/deepspeed.py:52 in   │\u001b[0m\n",
      "\u001b[34m│ __init__                                                                     │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│    49 │   │   │   │   config = json.load(f)                                  │\u001b[0m\n",
      "\u001b[34m│    50 │   │   else:                                                          │\u001b[0m\n",
      "\u001b[34m│    51 │   │   │   try:                                                       │\u001b[0m\n",
      "\u001b[34m│ ❱  52 │   │   │   │   config_decoded = base64.urlsafe_b64decode(config_file_ │\u001b[0m\n",
      "\u001b[34m│    53 │   │   │   │   config = json.loads(config_decoded)                    │\u001b[0m\n",
      "\u001b[34m│    54 │   │   │   except (UnicodeDecodeError, AttributeError):               │\u001b[0m\n",
      "\u001b[34m│    55 │   │   │   │   raise ValueError(                                      │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/conda/lib/python3.9/base64.py:133 in urlsafe_b64decode                  │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│   130 │   \"\"\"                                                                │\u001b[0m\n",
      "\u001b[34m│   131 │   s = _bytes_from_decode_data(s)                                     │\u001b[0m\n",
      "\u001b[34m│   132 │   s = s.translate(_urlsafe_decode_translation)                       │\u001b[0m\n",
      "\u001b[34m│ ❱ 133 │   return b64decode(s)                                                │\u001b[0m\n",
      "\u001b[34m│   134                                                                        │\u001b[0m\n",
      "\u001b[34m│   135                                                                        │\u001b[0m\n",
      "\u001b[34m│   136                                                                        │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/conda/lib/python3.9/base64.py:87 in b64decode                           │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│    84 │   │   s = s.translate(bytes.maketrans(altchars, b'+/'))              │\u001b[0m\n",
      "\u001b[34m│    85 │   if validate and not re.fullmatch(b'[A-Za-z0-9+/]*={0,2}', s):      │\u001b[0m\n",
      "\u001b[34m│    86 │   │   raise binascii.Error('Non-base64 digit found')                 │\u001b[0m\n",
      "\u001b[34m│ ❱  87 │   return binascii.a2b_base64(s)                                      │\u001b[0m\n",
      "\u001b[34m│    88                                                                        │\u001b[0m\n",
      "\u001b[34m│    89                                                                        │\u001b[0m\n",
      "\u001b[34m│    90 def standard_b64encode(s):                                             │\u001b[0m\n",
      "\u001b[34m╰──────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n",
      "\u001b[34mError: Invalid base64-encoded string: number of data characters (41) cannot be 1\u001b[0m\n",
      "\u001b[34mmore than a multiple of 4\u001b[0m\n",
      "\u001b[34m╭───────────────────── Traceback (most recent call last) ──────────────────────╮\u001b[0m\n",
      "\u001b[34m│ /opt/ml/code/BELLE/train/src/train.py:394 in <module>                        │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│   391                                                                        │\u001b[0m\n",
      "\u001b[34m│   392                                                                        │\u001b[0m\n",
      "\u001b[34m│   393 if __name__ == \"__main__\":                                             │\u001b[0m\n",
      "\u001b[34m│ ❱ 394 │   main()                                                             │\u001b[0m\n",
      "\u001b[34m│   395                                                                        │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/ml/code/BELLE/train/src/train.py:163 in main                            │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│   160                                                                        │\u001b[0m\n",
      "\u001b[34m│   161 def main():                                                            │\u001b[0m\n",
      "\u001b[34m│   162 │   parser = HfArgumentParser((ModelArguments, DataArguments, Training │\u001b[0m\n",
      "\u001b[34m│ ❱ 163 │   model_args, data_args, training_args = parser.parse_args_into_data │\u001b[0m\n",
      "\u001b[34m│   164 │                                                                      │\u001b[0m\n",
      "\u001b[34m│   165 │   world_size = int(os.environ.get(\"WORLD_SIZE\", 1))                  │\u001b[0m\n",
      "\u001b[34m│   166 │   ddp = world_size != 1                                              │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/conda/lib/python3.9/site-packages/transformers/hf_argparser.py:332 in   │\u001b[0m\n",
      "\u001b[34m│ parse_args_into_dataclasses                                                  │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│   329 │   │   │   inputs = {k: v for k, v in vars(namespace).items() if k in │\u001b[0m\n",
      "\u001b[34m│   330 │   │   │   for k in keys:                                             │\u001b[0m\n",
      "\u001b[34m│   331 │   │   │   │   delattr(namespace, k)                                  │\u001b[0m\n",
      "\u001b[34m│ ❱ 332 │   │   │   obj = dtype(**inputs)                                      │\u001b[0m\n",
      "\u001b[34m│   333 │   │   │   outputs.append(obj)                                        │\u001b[0m\n",
      "\u001b[34m│   334 │   │   if len(namespace.__dict__) > 0:                                │\u001b[0m\n",
      "\u001b[34m│   335 │   │   │   # additional namespace.                                    │\u001b[0m\n",
      "\u001b[34m│ <string>:114 in __init__                                                     │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/conda/lib/python3.9/site-packages/transformers/training_args.py:1446 in │\u001b[0m\n",
      "\u001b[34m│ __post_init__                                                                │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│   1443 │   │   │                                                             │\u001b[0m\n",
      "\u001b[34m│   1444 │   │   │   # will be used later by the Trainer                       │\u001b[0m\n",
      "\u001b[34m│   1445 │   │   │   # note: leave self.deepspeed unmodified in case a user re │\u001b[0m\n",
      "\u001b[34m│ ❱ 1446 │   │   │   self.hf_deepspeed_config = HfTrainerDeepSpeedConfig(self. │\u001b[0m\n",
      "\u001b[34m│   1447 │   │   │   self.hf_deepspeed_config.trainer_config_process(self)     │\u001b[0m\n",
      "\u001b[34m│   1448 │   │                                                                 │\u001b[0m\n",
      "\u001b[34m│   1449 │   │   if self.push_to_hub_token is not None:                        │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/conda/lib/python3.9/site-packages/transformers/deepspeed.py:77 in       │\u001b[0m\n",
      "\u001b[34m│ __init__                                                                     │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│    74 │   \"\"\"                                                                │\u001b[0m\n",
      "\u001b[34m│    75 │                                                                      │\u001b[0m\n",
      "\u001b[34m│    76 │   def __init__(self, config_file_or_dict):                           │\u001b[0m\n",
      "\u001b[34m│ ❱  77 │   │   super().__init__(config_file_or_dict)                          │\u001b[0m\n",
      "\u001b[34m│    78 │   │   self._dtype = None                                             │\u001b[0m\n",
      "\u001b[34m│    79 │   │   self.mismatches = []                                           │\u001b[0m\n",
      "\u001b[34m│    80                                                                        │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/conda/lib/python3.9/site-packages/transformers/deepspeed.py:67 in       │\u001b[0m\n",
      "\u001b[34m│ __init__                                                                     │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│    64 │   │   set_hf_deepspeed_config(self)                                  │\u001b[0m\n",
      "\u001b[34m│    65 │   │   dep_version_check(\"accelerate\")                                │\u001b[0m\n",
      "\u001b[34m│    66 │   │   dep_version_check(\"deepspeed\")                                 │\u001b[0m\n",
      "\u001b[34m│ ❱  67 │   │   super().__init__(config_file_or_dict)                          │\u001b[0m\n",
      "\u001b[34m│    68                                                                        │\u001b[0m\n",
      "\u001b[34m│    69                                                                        │\u001b[0m\n",
      "\u001b[34m│    70 class HfTrainerDeepSpeedConfig(HfDeepSpeedConfig):                     │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/conda/lib/python3.9/site-packages/accelerate/utils/deepspeed.py:52 in   │\u001b[0m\n",
      "\u001b[34m│ __init__                                                                     │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│    49 │   │   │   │   config = json.load(f)                                  │\u001b[0m\n",
      "\u001b[34m│    50 │   │   else:                                                          │\u001b[0m\n",
      "\u001b[34m│    51 │   │   │   try:                                                       │\u001b[0m\n",
      "\u001b[34m│ ❱  52 │   │   │   │   config_decoded = base64.urlsafe_b64decode(config_file_ │\u001b[0m\n",
      "\u001b[34m│    53 │   │   │   │   config = json.loads(config_decoded)                    │\u001b[0m\n",
      "\u001b[34m│    54 │   │   │   except (UnicodeDecodeError, AttributeError):               │\u001b[0m\n",
      "\u001b[34m│    55 │   │   │   │   raise ValueError(                                      │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/conda/lib/python3.9/base64.py:133 in urlsafe_b64decode                  │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│   130 │   \"\"\"                                                                │\u001b[0m\n",
      "\u001b[34m│   131 │   s = _bytes_from_decode_data(s)                                     │\u001b[0m\n",
      "\u001b[34m│   132 │   s = s.translate(_urlsafe_decode_translation)                       │\u001b[0m\n",
      "\u001b[34m│ ❱ 133 │   return b64decode(s)                                                │\u001b[0m\n",
      "\u001b[34m│   134                                                                        │\u001b[0m\n",
      "\u001b[34m│   135                                                                        │\u001b[0m\n",
      "\u001b[34m│   136                                                                        │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/conda/lib/python3.9/base64.py:87 in b64decode                           │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│    84 │   │   s = s.translate(bytes.maketrans(altchars, b'+/'))              │\u001b[0m\n",
      "\u001b[34m│    85 │   if validate and not re.fullmatch(b'[A-Za-z0-9+/]*={0,2}', s):      │\u001b[0m\n",
      "\u001b[34m│    86 │   │   raise binascii.Error('Non-base64 digit found')                 │\u001b[0m\n",
      "\u001b[34m│ ❱  87 │   return binascii.a2b_base64(s)                                      │\u001b[0m\n",
      "\u001b[34m│    88                                                                        │\u001b[0m\n",
      "\u001b[34m│    89                                                                        │\u001b[0m\n",
      "\u001b[34m│    90 def standard_b64encode(s):                                             │\u001b[0m\n",
      "\u001b[34m╰──────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n",
      "\u001b[34mError: Invalid base64-encoded string: number of data characters (41) cannot be 1\u001b[0m\n",
      "\u001b[34mmore than a multiple of 4\u001b[0m\n",
      "\u001b[34m╭───────────────────── Traceback (most recent call last) ──────────────────────╮\u001b[0m\n",
      "\u001b[34m│ /opt/ml/code/BELLE/train/src/train.py:394 in <module>                        │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│   391                                                                        │\u001b[0m\n",
      "\u001b[34m│   392                                                                        │\u001b[0m\n",
      "\u001b[34m│   393 if __name__ == \"__main__\":                                             │\u001b[0m\n",
      "\u001b[34m│ ❱ 394 │   main()                                                             │\u001b[0m\n",
      "\u001b[34m│   395                                                                        │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/ml/code/BELLE/train/src/train.py:163 in main                            │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│   160                                                                        │\u001b[0m\n",
      "\u001b[34m│   161 def main():                                                            │\u001b[0m\n",
      "\u001b[34m│   162 │   parser = HfArgumentParser((ModelArguments, DataArguments, Training │\u001b[0m\n",
      "\u001b[34m│ ❱ 163 │   model_args, data_args, training_args = parser.parse_args_into_data │\u001b[0m\n",
      "\u001b[34m│   164 │                                                                      │\u001b[0m\n",
      "\u001b[34m│   165 │   world_size = int(os.environ.get(\"WORLD_SIZE\", 1))                  │\u001b[0m\n",
      "\u001b[34m│   166 │   ddp = world_size != 1                                              │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/conda/lib/python3.9/site-packages/transformers/hf_argparser.py:332 in   │\u001b[0m\n",
      "\u001b[34m│ parse_args_into_dataclasses                                                  │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│   329 │   │   │   inputs = {k: v for k, v in vars(namespace).items() if k in │\u001b[0m\n",
      "\u001b[34m│   330 │   │   │   for k in keys:                                             │\u001b[0m\n",
      "\u001b[34m│   331 │   │   │   │   delattr(namespace, k)                                  │\u001b[0m\n",
      "\u001b[34m│ ❱ 332 │   │   │   obj = dtype(**inputs)                                      │\u001b[0m\n",
      "\u001b[34m│   333 │   │   │   outputs.append(obj)                                        │\u001b[0m\n",
      "\u001b[34m│   334 │   │   if len(namespace.__dict__) > 0:                                │\u001b[0m\n",
      "\u001b[34m│   335 │   │   │   # additional namespace.                                    │\u001b[0m\n",
      "\u001b[34m│ <string>:114 in __init__                                                     │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/conda/lib/python3.9/site-packages/transformers/training_args.py:1446 in │\u001b[0m\n",
      "\u001b[34m│ __post_init__                                                                │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│   1443 │   │   │                                                             │\u001b[0m\n",
      "\u001b[34m│   1444 │   │   │   # will be used later by the Trainer                       │\u001b[0m\n",
      "\u001b[34m│   1445 │   │   │   # note: leave self.deepspeed unmodified in case a user re │\u001b[0m\n",
      "\u001b[34m│ ❱ 1446 │   │   │   self.hf_deepspeed_config = HfTrainerDeepSpeedConfig(self. │\u001b[0m\n",
      "\u001b[34m│   1447 │   │   │   self.hf_deepspeed_config.trainer_config_process(self)     │\u001b[0m\n",
      "\u001b[34m│   1448 │   │                                                                 │\u001b[0m\n",
      "\u001b[34m│   1449 │   │   if self.push_to_hub_token is not None:                        │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/conda/lib/python3.9/site-packages/transformers/deepspeed.py:77 in       │\u001b[0m\n",
      "\u001b[34m│ __init__                                                                     │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│    74 │   \"\"\"                                                                │\u001b[0m\n",
      "\u001b[34m│    75 │                                                                      │\u001b[0m\n",
      "\u001b[34m│    76 │   def __init__(self, config_file_or_dict):                           │\u001b[0m\n",
      "\u001b[34m│ ❱  77 │   │   super().__init__(config_file_or_dict)                          │\u001b[0m\n",
      "\u001b[34m│    78 │   │   self._dtype = None                                             │\u001b[0m\n",
      "\u001b[34m│    79 │   │   self.mismatches = []                                           │\u001b[0m\n",
      "\u001b[34m│    80                                                                        │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/conda/lib/python3.9/site-packages/transformers/deepspeed.py:67 in       │\u001b[0m\n",
      "\u001b[34m│ __init__                                                                     │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│    64 │   │   set_hf_deepspeed_config(self)                                  │\u001b[0m\n",
      "\u001b[34m│    65 │   │   dep_version_check(\"accelerate\")                                │\u001b[0m\n",
      "\u001b[34m│    66 │   │   dep_version_check(\"deepspeed\")                                 │\u001b[0m\n",
      "\u001b[34m│ ❱  67 │   │   super().__init__(config_file_or_dict)                          │\u001b[0m\n",
      "\u001b[34m│    68                                                                        │\u001b[0m\n",
      "\u001b[34m│    69                                                                        │\u001b[0m\n",
      "\u001b[34m│    70 class HfTrainerDeepSpeedConfig(HfDeepSpeedConfig):                     │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/conda/lib/python3.9/site-packages/accelerate/utils/deepspeed.py:52 in   │\u001b[0m\n",
      "\u001b[34m│ __init__                                                                     │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│    49 │   │   │   │   config = json.load(f)                                  │\u001b[0m\n",
      "\u001b[34m│    50 │   │   else:                                                          │\u001b[0m\n",
      "\u001b[34m│    51 │   │   │   try:                                                       │\u001b[0m\n",
      "\u001b[34m│ ❱  52 │   │   │   │   config_decoded = base64.urlsafe_b64decode(config_file_ │\u001b[0m\n",
      "\u001b[34m│    53 │   │   │   │   config = json.loads(config_decoded)                    │\u001b[0m\n",
      "\u001b[34m│    54 │   │   │   except (UnicodeDecodeError, AttributeError):               │\u001b[0m\n",
      "\u001b[34m│    55 │   │   │   │   raise ValueError(                                      │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/conda/lib/python3.9/base64.py:133 in urlsafe_b64decode                  │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│   130 │   \"\"\"                                                                │\u001b[0m\n",
      "\u001b[34m│   131 │   s = _bytes_from_decode_data(s)                                     │\u001b[0m\n",
      "\u001b[34m│   132 │   s = s.translate(_urlsafe_decode_translation)                       │\u001b[0m\n",
      "\u001b[34m│ ❱ 133 │   return b64decode(s)                                                │\u001b[0m\n",
      "\u001b[34m│   134                                                                        │\u001b[0m\n",
      "\u001b[34m│   135                                                                        │\u001b[0m\n",
      "\u001b[34m│   136                                                                        │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/conda/lib/python3.9/base64.py:87 in b64decode                           │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│    84 │   │   s = s.translate(bytes.maketrans(altchars, b'+/'))              │\u001b[0m\n",
      "\u001b[34m│    85 │   if validate and not re.fullmatch(b'[A-Za-z0-9+/]*={0,2}', s):      │\u001b[0m\n",
      "\u001b[34m│    86 │   │   raise binascii.Error('Non-base64 digit found')                 │\u001b[0m\n",
      "\u001b[34m│ ❱  87 │   return binascii.a2b_base64(s)                                      │\u001b[0m\n",
      "\u001b[34m│    88                                                                        │\u001b[0m\n",
      "\u001b[34m│    89                                                                        │\u001b[0m\n",
      "\u001b[34m│    90 def standard_b64encode(s):                                             │\u001b[0m\n",
      "\u001b[34m╰──────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n",
      "\u001b[34mError: Invalid base64-encoded string: number of data characters (41) cannot be 1\u001b[0m\n",
      "\u001b[34mmore than a multiple of 4\u001b[0m\n",
      "\u001b[34m╭───────────────────── Traceback (most recent call last) ──────────────────────╮\u001b[0m\n",
      "\u001b[34m│ /opt/ml/code/BELLE/train/src/train.py:394 in <module>                        │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│   391                                                                        │\u001b[0m\n",
      "\u001b[34m│   392                                                                        │\u001b[0m\n",
      "\u001b[34m│   393 if __name__ == \"__main__\":                                             │\u001b[0m\n",
      "\u001b[34m│ ❱ 394 │   main()                                                             │\u001b[0m\n",
      "\u001b[34m│   395                                                                        │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/ml/code/BELLE/train/src/train.py:163 in main                            │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│   160                                                                        │\u001b[0m\n",
      "\u001b[34m│   161 def main():                                                            │\u001b[0m\n",
      "\u001b[34m│   162 │   parser = HfArgumentParser((ModelArguments, DataArguments, Training │\u001b[0m\n",
      "\u001b[34m│ ❱ 163 │   model_args, data_args, training_args = parser.parse_args_into_data │\u001b[0m\n",
      "\u001b[34m│   164 │                                                                      │\u001b[0m\n",
      "\u001b[34m│   165 │   world_size = int(os.environ.get(\"WORLD_SIZE\", 1))                  │\u001b[0m\n",
      "\u001b[34m│   166 │   ddp = world_size != 1                                              │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/conda/lib/python3.9/site-packages/transformers/hf_argparser.py:332 in   │\u001b[0m\n",
      "\u001b[34m│ parse_args_into_dataclasses                                                  │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│   329 │   │   │   inputs = {k: v for k, v in vars(namespace).items() if k in │\u001b[0m\n",
      "\u001b[34m│   330 │   │   │   for k in keys:                                             │\u001b[0m\n",
      "\u001b[34m│   331 │   │   │   │   delattr(namespace, k)                                  │\u001b[0m\n",
      "\u001b[34m│ ❱ 332 │   │   │   obj = dtype(**inputs)                                      │\u001b[0m\n",
      "\u001b[34m│   333 │   │   │   outputs.append(obj)                                        │\u001b[0m\n",
      "\u001b[34m│   334 │   │   if len(namespace.__dict__) > 0:                                │\u001b[0m\n",
      "\u001b[34m│   335 │   │   │   # additional namespace.                                    │\u001b[0m\n",
      "\u001b[34m│ <string>:114 in __init__                                                     │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/conda/lib/python3.9/site-packages/transformers/training_args.py:1446 in │\u001b[0m\n",
      "\u001b[34m│ __post_init__                                                                │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│   1443 │   │   │                                                             │\u001b[0m\n",
      "\u001b[34m│   1444 │   │   │   # will be used later by the Trainer                       │\u001b[0m\n",
      "\u001b[34m│   1445 │   │   │   # note: leave self.deepspeed unmodified in case a user re │\u001b[0m\n",
      "\u001b[34m│ ❱ 1446 │   │   │   self.hf_deepspeed_config = HfTrainerDeepSpeedConfig(self. │\u001b[0m\n",
      "\u001b[34m│   1447 │   │   │   self.hf_deepspeed_config.trainer_config_process(self)     │\u001b[0m\n",
      "\u001b[34m│   1448 │   │                                                                 │\u001b[0m\n",
      "\u001b[34m│   1449 │   │   if self.push_to_hub_token is not None:                        │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/conda/lib/python3.9/site-packages/transformers/deepspeed.py:77 in       │\u001b[0m\n",
      "\u001b[34m│ __init__                                                                     │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│    74 │   \"\"\"                                                                │\u001b[0m\n",
      "\u001b[34m│    75 │                                                                      │\u001b[0m\n",
      "\u001b[34m│    76 │   def __init__(self, config_file_or_dict):                           │\u001b[0m\n",
      "\u001b[34m│ ❱  77 │   │   super().__init__(config_file_or_dict)                          │\u001b[0m\n",
      "\u001b[34m│    78 │   │   self._dtype = None                                             │\u001b[0m\n",
      "\u001b[34m│    79 │   │   self.mismatches = []                                           │\u001b[0m\n",
      "\u001b[34m│    80                                                                        │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/conda/lib/python3.9/site-packages/transformers/deepspeed.py:67 in       │\u001b[0m\n",
      "\u001b[34m│ __init__                                                                     │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│    64 │   │   set_hf_deepspeed_config(self)                                  │\u001b[0m\n",
      "\u001b[34m│    65 │   │   dep_version_check(\"accelerate\")                                │\u001b[0m\n",
      "\u001b[34m│    66 │   │   dep_version_check(\"deepspeed\")                                 │\u001b[0m\n",
      "\u001b[34m│ ❱  67 │   │   super().__init__(config_file_or_dict)                          │\u001b[0m\n",
      "\u001b[34m│    68                                                                        │\u001b[0m\n",
      "\u001b[34m│    69                                                                        │\u001b[0m\n",
      "\u001b[34m│    70 class HfTrainerDeepSpeedConfig(HfDeepSpeedConfig):                     │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/conda/lib/python3.9/site-packages/accelerate/utils/deepspeed.py:52 in   │\u001b[0m\n",
      "\u001b[34m│ __init__                                                                     │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│    49 │   │   │   │   config = json.load(f)                                  │\u001b[0m\n",
      "\u001b[34m│    50 │   │   else:                                                          │\u001b[0m\n",
      "\u001b[34m│    51 │   │   │   try:                                                       │\u001b[0m\n",
      "\u001b[34m│ ❱  52 │   │   │   │   config_decoded = base64.urlsafe_b64decode(config_file_ │\u001b[0m\n",
      "\u001b[34m│    53 │   │   │   │   config = json.loads(config_decoded)                    │\u001b[0m\n",
      "\u001b[34m│    54 │   │   │   except (UnicodeDecodeError, AttributeError):               │\u001b[0m\n",
      "\u001b[34m│    55 │   │   │   │   raise ValueError(                                      │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/conda/lib/python3.9/base64.py:133 in urlsafe_b64decode                  │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│   130 │   \"\"\"                                                                │\u001b[0m\n",
      "\u001b[34m│   131 │   s = _bytes_from_decode_data(s)                                     │\u001b[0m\n",
      "\u001b[34m│   132 │   s = s.translate(_urlsafe_decode_translation)                       │\u001b[0m\n",
      "\u001b[34m│ ❱ 133 │   return b64decode(s)                                                │\u001b[0m\n",
      "\u001b[34m│   134                                                                        │\u001b[0m\n",
      "\u001b[34m│   135                                                                        │\u001b[0m\n",
      "\u001b[34m│   136                                                                        │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/conda/lib/python3.9/base64.py:87 in b64decode                           │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│    84 │   │   s = s.translate(bytes.maketrans(altchars, b'+/'))              │\u001b[0m\n",
      "\u001b[34m│    85 │   if validate and not re.fullmatch(b'[A-Za-z0-9+/]*={0,2}', s):      │\u001b[0m\n",
      "\u001b[34m│    86 │   │   raise binascii.Error('Non-base64 digit found')                 │\u001b[0m\n",
      "\u001b[34m│ ❱  87 │   return binascii.a2b_base64(s)                                      │\u001b[0m\n",
      "\u001b[34m│    88                                                                        │\u001b[0m\n",
      "\u001b[34m│    89                                                                        │\u001b[0m\n",
      "\u001b[34m│    90 def standard_b64encode(s):                                             │\u001b[0m\n",
      "\u001b[34m╰──────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n",
      "\u001b[34mError: Invalid base64-encoded string: number of data characters (41) cannot be 1\u001b[0m\n",
      "\u001b[34mmore than a multiple of 4\u001b[0m\n",
      "\u001b[34m╭───────────────────── Traceback (most recent call last) ──────────────────────╮\u001b[0m\n",
      "\u001b[34m│ /opt/ml/code/BELLE/train/src/train.py:394 in <module>                        │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│   391                                                                        │\u001b[0m\n",
      "\u001b[34m│   392                                                                        │\u001b[0m\n",
      "\u001b[34m│   393 if __name__ == \"__main__\":                                             │\u001b[0m\n",
      "\u001b[34m│ ❱ 394 │   main()                                                             │\u001b[0m\n",
      "\u001b[34m│   395                                                                        │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/ml/code/BELLE/train/src/train.py:163 in main                            │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│   160                                                                        │\u001b[0m\n",
      "\u001b[34m│   161 def main():                                                            │\u001b[0m\n",
      "\u001b[34m│   162 │   parser = HfArgumentParser((ModelArguments, DataArguments, Training │\u001b[0m\n",
      "\u001b[34m│ ❱ 163 │   model_args, data_args, training_args = parser.parse_args_into_data │\u001b[0m\n",
      "\u001b[34m│   164 │                                                                      │\u001b[0m\n",
      "\u001b[34m│   165 │   world_size = int(os.environ.get(\"WORLD_SIZE\", 1))                  │\u001b[0m\n",
      "\u001b[34m│   166 │   ddp = world_size != 1                                              │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/conda/lib/python3.9/site-packages/transformers/hf_argparser.py:332 in   │\u001b[0m\n",
      "\u001b[34m│ parse_args_into_dataclasses                                                  │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│   329 │   │   │   inputs = {k: v for k, v in vars(namespace).items() if k in │\u001b[0m\n",
      "\u001b[34m│   330 │   │   │   for k in keys:                                             │\u001b[0m\n",
      "\u001b[34m│   331 │   │   │   │   delattr(namespace, k)                                  │\u001b[0m\n",
      "\u001b[34m│ ❱ 332 │   │   │   obj = dtype(**inputs)                                      │\u001b[0m\n",
      "\u001b[34m│   333 │   │   │   outputs.append(obj)                                        │\u001b[0m\n",
      "\u001b[34m│   334 │   │   if len(namespace.__dict__) > 0:                                │\u001b[0m\n",
      "\u001b[34m│   335 │   │   │   # additional namespace.                                    │\u001b[0m\n",
      "\u001b[34m│ <string>:114 in __init__                                                     │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/conda/lib/python3.9/site-packages/transformers/training_args.py:1446 in │\u001b[0m\n",
      "\u001b[34m│ __post_init__                                                                │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│   1443 │   │   │                                                             │\u001b[0m\n",
      "\u001b[34m│   1444 │   │   │   # will be used later by the Trainer                       │\u001b[0m\n",
      "\u001b[34m│   1445 │   │   │   # note: leave self.deepspeed unmodified in case a user re │\u001b[0m\n",
      "\u001b[34m│ ❱ 1446 │   │   │   self.hf_deepspeed_config = HfTrainerDeepSpeedConfig(self. │\u001b[0m\n",
      "\u001b[34m│   1447 │   │   │   self.hf_deepspeed_config.trainer_config_process(self)     │\u001b[0m\n",
      "\u001b[34m│   1448 │   │                                                                 │\u001b[0m\n",
      "\u001b[34m│   1449 │   │   if self.push_to_hub_token is not None:                        │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/conda/lib/python3.9/site-packages/transformers/deepspeed.py:77 in       │\u001b[0m\n",
      "\u001b[34m│ __init__                                                                     │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│    74 │   \"\"\"                                                                │\u001b[0m\n",
      "\u001b[34m│    75 │                                                                      │\u001b[0m\n",
      "\u001b[34m│    76 │   def __init__(self, config_file_or_dict):                           │\u001b[0m\n",
      "\u001b[34m│ ❱  77 │   │   super().__init__(config_file_or_dict)                          │\u001b[0m\n",
      "\u001b[34m│    78 │   │   self._dtype = None                                             │\u001b[0m\n",
      "\u001b[34m│    79 │   │   self.mismatches = []                                           │\u001b[0m\n",
      "\u001b[34m│    80                                                                        │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/conda/lib/python3.9/site-packages/transformers/deepspeed.py:67 in       │\u001b[0m\n",
      "\u001b[34m│ __init__                                                                     │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│    64 │   │   set_hf_deepspeed_config(self)                                  │\u001b[0m\n",
      "\u001b[34m│    65 │   │   dep_version_check(\"accelerate\")                                │\u001b[0m\n",
      "\u001b[34m│    66 │   │   dep_version_check(\"deepspeed\")                                 │\u001b[0m\n",
      "\u001b[34m│ ❱  67 │   │   super().__init__(config_file_or_dict)                          │\u001b[0m\n",
      "\u001b[34m│    68                                                                        │\u001b[0m\n",
      "\u001b[34m│    69                                                                        │\u001b[0m\n",
      "\u001b[34m│    70 class HfTrainerDeepSpeedConfig(HfDeepSpeedConfig):                     │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/conda/lib/python3.9/site-packages/accelerate/utils/deepspeed.py:52 in   │\u001b[0m\n",
      "\u001b[34m│ __init__                                                                     │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│    49 │   │   │   │   config = json.load(f)                                  │\u001b[0m\n",
      "\u001b[34m│    50 │   │   else:                                                          │\u001b[0m\n",
      "\u001b[34m│    51 │   │   │   try:                                                       │\u001b[0m\n",
      "\u001b[34m│ ❱  52 │   │   │   │   config_decoded = base64.urlsafe_b64decode(config_file_ │\u001b[0m\n",
      "\u001b[34m│    53 │   │   │   │   config = json.loads(config_decoded)                    │\u001b[0m\n",
      "\u001b[34m│    54 │   │   │   except (UnicodeDecodeError, AttributeError):               │\u001b[0m\n",
      "\u001b[34m│    55 │   │   │   │   raise ValueError(                                      │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/conda/lib/python3.9/base64.py:133 in urlsafe_b64decode                  │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│   130 │   \"\"\"                                                                │\u001b[0m\n",
      "\u001b[34m│   131 │   s = _bytes_from_decode_data(s)                                     │\u001b[0m\n",
      "\u001b[34m│   132 │   s = s.translate(_urlsafe_decode_translation)                       │\u001b[0m\n",
      "\u001b[34m│ ❱ 133 │   return b64decode(s)                                                │\u001b[0m\n",
      "\u001b[34m│   134                                                                        │\u001b[0m\n",
      "\u001b[34m│   135                                                                        │\u001b[0m\n",
      "\u001b[34m│   136                                                                        │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/conda/lib/python3.9/base64.py:87 in b64decode                           │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│    84 │   │   s = s.translate(bytes.maketrans(altchars, b'+/'))              │\u001b[0m\n",
      "\u001b[34m│    85 │   if validate and not re.fullmatch(b'[A-Za-z0-9+/]*={0,2}', s):      │\u001b[0m\n",
      "\u001b[34m│    86 │   │   raise binascii.Error('Non-base64 digit found')                 │\u001b[0m\n",
      "\u001b[34m│ ❱  87 │   return binascii.a2b_base64(s)                                      │\u001b[0m\n",
      "\u001b[34m│    88                                                                        │\u001b[0m\n",
      "\u001b[34m│    89                                                                        │\u001b[0m\n",
      "\u001b[34m│    90 def standard_b64encode(s):                                             │\u001b[0m\n",
      "\u001b[34m╰──────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n",
      "\u001b[34mError: Invalid base64-encoded string: number of data characters (41) cannot be 1\u001b[0m\n",
      "\u001b[34mmore than a multiple of 4\u001b[0m\n",
      "\u001b[34m╭───────────────────── Traceback (most recent call last) ──────────────────────╮\u001b[0m\n",
      "\u001b[34m│ /opt/ml/code/BELLE/train/src/train.py:394 in <module>                        │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│   391                                                                        │\u001b[0m\n",
      "\u001b[34m│   392                                                                        │\u001b[0m\n",
      "\u001b[34m│   393 if __name__ == \"__main__\":                                             │\u001b[0m\n",
      "\u001b[34m│ ❱ 394 │   main()                                                             │\u001b[0m\n",
      "\u001b[34m│   395                                                                        │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/ml/code/BELLE/train/src/train.py:163 in main                            │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│   160                                                                        │\u001b[0m\n",
      "\u001b[34m│   161 def main():                                                            │\u001b[0m\n",
      "\u001b[34m│   162 │   parser = HfArgumentParser((ModelArguments, DataArguments, Training │\u001b[0m\n",
      "\u001b[34m│ ❱ 163 │   model_args, data_args, training_args = parser.parse_args_into_data │\u001b[0m\n",
      "\u001b[34m│   164 │                                                                      │\u001b[0m\n",
      "\u001b[34m│   165 │   world_size = int(os.environ.get(\"WORLD_SIZE\", 1))                  │\u001b[0m\n",
      "\u001b[34m│   166 │   ddp = world_size != 1                                              │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/conda/lib/python3.9/site-packages/transformers/hf_argparser.py:332 in   │\u001b[0m\n",
      "\u001b[34m│ parse_args_into_dataclasses                                                  │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│   329 │   │   │   inputs = {k: v for k, v in vars(namespace).items() if k in │\u001b[0m\n",
      "\u001b[34m│   330 │   │   │   for k in keys:                                             │\u001b[0m\n",
      "\u001b[34m│   331 │   │   │   │   delattr(namespace, k)                                  │\u001b[0m\n",
      "\u001b[34m│ ❱ 332 │   │   │   obj = dtype(**inputs)                                      │\u001b[0m\n",
      "\u001b[34m│   333 │   │   │   outputs.append(obj)                                        │\u001b[0m\n",
      "\u001b[34m│   334 │   │   if len(namespace.__dict__) > 0:                                │\u001b[0m\n",
      "\u001b[34m│   335 │   │   │   # additional namespace.                                    │\u001b[0m\n",
      "\u001b[34m│ <string>:114 in __init__                                                     │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/conda/lib/python3.9/site-packages/transformers/training_args.py:1446 in │\u001b[0m\n",
      "\u001b[34m│ __post_init__                                                                │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│   1443 │   │   │                                                             │\u001b[0m\n",
      "\u001b[34m│   1444 │   │   │   # will be used later by the Trainer                       │\u001b[0m\n",
      "\u001b[34m│   1445 │   │   │   # note: leave self.deepspeed unmodified in case a user re │\u001b[0m\n",
      "\u001b[34m│ ❱ 1446 │   │   │   self.hf_deepspeed_config = HfTrainerDeepSpeedConfig(self. │\u001b[0m\n",
      "\u001b[34m│   1447 │   │   │   self.hf_deepspeed_config.trainer_config_process(self)     │\u001b[0m\n",
      "\u001b[34m│   1448 │   │                                                                 │\u001b[0m\n",
      "\u001b[34m│   1449 │   │   if self.push_to_hub_token is not None:                        │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/conda/lib/python3.9/site-packages/transformers/deepspeed.py:77 in       │\u001b[0m\n",
      "\u001b[34m│ __init__                                                                     │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│    74 │   \"\"\"                                                                │\u001b[0m\n",
      "\u001b[34m│    75 │                                                                      │\u001b[0m\n",
      "\u001b[34m│    76 │   def __init__(self, config_file_or_dict):                           │\u001b[0m\n",
      "\u001b[34m│ ❱  77 │   │   super().__init__(config_file_or_dict)                          │\u001b[0m\n",
      "\u001b[34m│    78 │   │   self._dtype = None                                             │\u001b[0m\n",
      "\u001b[34m│    79 │   │   self.mismatches = []                                           │\u001b[0m\n",
      "\u001b[34m│    80                                                                        │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/conda/lib/python3.9/site-packages/transformers/deepspeed.py:67 in       │\u001b[0m\n",
      "\u001b[34m│ __init__                                                                     │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│    64 │   │   set_hf_deepspeed_config(self)                                  │\u001b[0m\n",
      "\u001b[34m│    65 │   │   dep_version_check(\"accelerate\")                                │\u001b[0m\n",
      "\u001b[34m│    66 │   │   dep_version_check(\"deepspeed\")                                 │\u001b[0m\n",
      "\u001b[34m│ ❱  67 │   │   super().__init__(config_file_or_dict)                          │\u001b[0m\n",
      "\u001b[34m│    68                                                                        │\u001b[0m\n",
      "\u001b[34m│    69                                                                        │\u001b[0m\n",
      "\u001b[34m│    70 class HfTrainerDeepSpeedConfig(HfDeepSpeedConfig):                     │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/conda/lib/python3.9/site-packages/accelerate/utils/deepspeed.py:52 in   │\u001b[0m\n",
      "\u001b[34m│ __init__                                                                     │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│    49 │   │   │   │   config = json.load(f)                                  │\u001b[0m\n",
      "\u001b[34m│    50 │   │   else:                                                          │\u001b[0m\n",
      "\u001b[34m│    51 │   │   │   try:                                                       │\u001b[0m\n",
      "\u001b[34m│ ❱  52 │   │   │   │   config_decoded = base64.urlsafe_b64decode(config_file_ │\u001b[0m\n",
      "\u001b[34m│    53 │   │   │   │   config = json.loads(config_decoded)                    │\u001b[0m\n",
      "\u001b[34m│    54 │   │   │   except (UnicodeDecodeError, AttributeError):               │\u001b[0m\n",
      "\u001b[34m│    55 │   │   │   │   raise ValueError(                                      │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/conda/lib/python3.9/base64.py:133 in urlsafe_b64decode                  │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│   130 │   \"\"\"                                                                │\u001b[0m\n",
      "\u001b[34m│   131 │   s = _bytes_from_decode_data(s)                                     │\u001b[0m\n",
      "\u001b[34m│   132 │   s = s.translate(_urlsafe_decode_translation)                       │\u001b[0m\n",
      "\u001b[34m│ ❱ 133 │   return b64decode(s)                                                │\u001b[0m\n",
      "\u001b[34m│   134                                                                        │\u001b[0m\n",
      "\u001b[34m│   135                                                                        │\u001b[0m\n",
      "\u001b[34m│   136                                                                        │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/conda/lib/python3.9/base64.py:87 in b64decode                           │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│    84 │   │   s = s.translate(bytes.maketrans(altchars, b'+/'))              │\u001b[0m\n",
      "\u001b[34m│    85 │   if validate and not re.fullmatch(b'[A-Za-z0-9+/]*={0,2}', s):      │\u001b[0m\n",
      "\u001b[34m│    86 │   │   raise binascii.Error('Non-base64 digit found')                 │\u001b[0m\n",
      "\u001b[34m│ ❱  87 │   return binascii.a2b_base64(s)                                      │\u001b[0m\n",
      "\u001b[34m│    88                                                                        │\u001b[0m\n",
      "\u001b[34m│    89                                                                        │\u001b[0m\n",
      "\u001b[34m│    90 def standard_b64encode(s):                                             │\u001b[0m\n",
      "\u001b[34m╰──────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n",
      "\u001b[34mError: Invalid base64-encoded string: number of data characters (41) cannot be 1\u001b[0m\n",
      "\u001b[34mmore than a multiple of 4\u001b[0m\n",
      "\u001b[34m╭───────────────────── Traceback (most recent call last) ──────────────────────╮\u001b[0m\n",
      "\u001b[34m│ /opt/ml/code/BELLE/train/src/train.py:394 in <module>                        │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│   391                                                                        │\u001b[0m\n",
      "\u001b[34m│   392                                                                        │\u001b[0m\n",
      "\u001b[34m│   393 if __name__ == \"__main__\":                                             │\u001b[0m\n",
      "\u001b[34m│ ❱ 394 │   main()                                                             │\u001b[0m\n",
      "\u001b[34m│   395                                                                        │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/ml/code/BELLE/train/src/train.py:163 in main                            │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│   160                                                                        │\u001b[0m\n",
      "\u001b[34m│   161 def main():                                                            │\u001b[0m\n",
      "\u001b[34m│   162 │   parser = HfArgumentParser((ModelArguments, DataArguments, Training │\u001b[0m\n",
      "\u001b[34m│ ❱ 163 │   model_args, data_args, training_args = parser.parse_args_into_data │\u001b[0m\n",
      "\u001b[34m│   164 │                                                                      │\u001b[0m\n",
      "\u001b[34m│   165 │   world_size = int(os.environ.get(\"WORLD_SIZE\", 1))                  │\u001b[0m\n",
      "\u001b[34m│   166 │   ddp = world_size != 1                                              │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/conda/lib/python3.9/site-packages/transformers/hf_argparser.py:332 in   │\u001b[0m\n",
      "\u001b[34m│ parse_args_into_dataclasses                                                  │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│   329 │   │   │   inputs = {k: v for k, v in vars(namespace).items() if k in │\u001b[0m\n",
      "\u001b[34m│   330 │   │   │   for k in keys:                                             │\u001b[0m\n",
      "\u001b[34m│   331 │   │   │   │   delattr(namespace, k)                                  │\u001b[0m\n",
      "\u001b[34m│ ❱ 332 │   │   │   obj = dtype(**inputs)                                      │\u001b[0m\n",
      "\u001b[34m│   333 │   │   │   outputs.append(obj)                                        │\u001b[0m\n",
      "\u001b[34m│   334 │   │   if len(namespace.__dict__) > 0:                                │\u001b[0m\n",
      "\u001b[34m│   335 │   │   │   # additional namespace.                                    │\u001b[0m\n",
      "\u001b[34m│ <string>:114 in __init__                                                     │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/conda/lib/python3.9/site-packages/transformers/training_args.py:1446 in │\u001b[0m\n",
      "\u001b[34m│ __post_init__                                                                │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│   1443 │   │   │                                                             │\u001b[0m\n",
      "\u001b[34m│   1444 │   │   │   # will be used later by the Trainer                       │\u001b[0m\n",
      "\u001b[34m│   1445 │   │   │   # note: leave self.deepspeed unmodified in case a user re │\u001b[0m\n",
      "\u001b[34m│ ❱ 1446 │   │   │   self.hf_deepspeed_config = HfTrainerDeepSpeedConfig(self. │\u001b[0m\n",
      "\u001b[34m│   1447 │   │   │   self.hf_deepspeed_config.trainer_config_process(self)     │\u001b[0m\n",
      "\u001b[34m│   1448 │   │                                                                 │\u001b[0m\n",
      "\u001b[34m│   1449 │   │   if self.push_to_hub_token is not None:                        │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/conda/lib/python3.9/site-packages/transformers/deepspeed.py:77 in       │\u001b[0m\n",
      "\u001b[34m│ __init__                                                                     │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│    74 │   \"\"\"                                                                │\u001b[0m\n",
      "\u001b[34m│    75 │                                                                      │\u001b[0m\n",
      "\u001b[34m│    76 │   def __init__(self, config_file_or_dict):                           │\u001b[0m\n",
      "\u001b[34m│ ❱  77 │   │   super().__init__(config_file_or_dict)                          │\u001b[0m\n",
      "\u001b[34m│    78 │   │   self._dtype = None                                             │\u001b[0m\n",
      "\u001b[34m│    79 │   │   self.mismatches = []                                           │\u001b[0m\n",
      "\u001b[34m│    80                                                                        │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/conda/lib/python3.9/site-packages/transformers/deepspeed.py:67 in       │\u001b[0m\n",
      "\u001b[34m│ __init__                                                                     │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│    64 │   │   set_hf_deepspeed_config(self)                                  │\u001b[0m\n",
      "\u001b[34m│    65 │   │   dep_version_check(\"accelerate\")                                │\u001b[0m\n",
      "\u001b[34m│    66 │   │   dep_version_check(\"deepspeed\")                                 │\u001b[0m\n",
      "\u001b[34m│ ❱  67 │   │   super().__init__(config_file_or_dict)                          │\u001b[0m\n",
      "\u001b[34m│    68                                                                        │\u001b[0m\n",
      "\u001b[34m│    69                                                                        │\u001b[0m\n",
      "\u001b[34m│    70 class HfTrainerDeepSpeedConfig(HfDeepSpeedConfig):                     │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/conda/lib/python3.9/site-packages/accelerate/utils/deepspeed.py:52 in   │\u001b[0m\n",
      "\u001b[34m│ __init__                                                                     │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│    49 │   │   │   │   config = json.load(f)                                  │\u001b[0m\n",
      "\u001b[34m│    50 │   │   else:                                                          │\u001b[0m\n",
      "\u001b[34m│    51 │   │   │   try:                                                       │\u001b[0m\n",
      "\u001b[34m│ ❱  52 │   │   │   │   config_decoded = base64.urlsafe_b64decode(config_file_ │\u001b[0m\n",
      "\u001b[34m│    53 │   │   │   │   config = json.loads(config_decoded)                    │\u001b[0m\n",
      "\u001b[34m│    54 │   │   │   except (UnicodeDecodeError, AttributeError):               │\u001b[0m\n",
      "\u001b[34m│    55 │   │   │   │   raise ValueError(                                      │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/conda/lib/python3.9/base64.py:133 in urlsafe_b64decode                  │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│   130 │   \"\"\"                                                                │\u001b[0m\n",
      "\u001b[34m│   131 │   s = _bytes_from_decode_data(s)                                     │\u001b[0m\n",
      "\u001b[34m│   132 │   s = s.translate(_urlsafe_decode_translation)                       │\u001b[0m\n",
      "\u001b[34m│ ❱ 133 │   return b64decode(s)                                                │\u001b[0m\n",
      "\u001b[34m│   134                                                                        │\u001b[0m\n",
      "\u001b[34m│   135                                                                        │\u001b[0m\n",
      "\u001b[34m│   136                                                                        │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│ /opt/conda/lib/python3.9/base64.py:87 in b64decode                           │\u001b[0m\n",
      "\u001b[34m│                                                                              │\u001b[0m\n",
      "\u001b[34m│    84 │   │   s = s.translate(bytes.maketrans(altchars, b'+/'))              │\u001b[0m\n",
      "\u001b[34m│    85 │   if validate and not re.fullmatch(b'[A-Za-z0-9+/]*={0,2}', s):      │\u001b[0m\n",
      "\u001b[34m│    86 │   │   raise binascii.Error('Non-base64 digit found')                 │\u001b[0m\n",
      "\u001b[34m│ ❱  87 │   return binascii.a2b_base64(s)                                      │\u001b[0m\n",
      "\u001b[34m│    88                                                                        │\u001b[0m\n",
      "\u001b[34m│    89                                                                        │\u001b[0m\n",
      "\u001b[34m│    90 def standard_b64encode(s):                                             │\u001b[0m\n",
      "\u001b[34m╰──────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n",
      "\u001b[34mError: Invalid base64-encoded string: number of data characters (41) cannot be 1\u001b[0m\n",
      "\u001b[34mmore than a multiple of 4\u001b[0m\n",
      "\u001b[34mERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 214) of binary: /opt/conda/bin/python3.9\u001b[0m\n",
      "\u001b[34mTraceback (most recent call last):\n",
      "  File \"/opt/conda/bin/torchrun\", line 8, in <module>\u001b[0m\n",
      "\u001b[34msys.exit(main())\n",
      "  File \"/opt/conda/lib/python3.9/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py\", line 346, in wrapper\u001b[0m\n",
      "\u001b[34mreturn f(*args, **kwargs)\n",
      "  File \"/opt/conda/lib/python3.9/site-packages/torch/distributed/run.py\", line 762, in main\u001b[0m\n",
      "\u001b[34mrun(args)\n",
      "  File \"/opt/conda/lib/python3.9/site-packages/torch/distributed/run.py\", line 753, in run\u001b[0m\n",
      "\u001b[34melastic_launch(\n",
      "  File \"/opt/conda/lib/python3.9/site-packages/torch/distributed/launcher/api.py\", line 132, in __call__\u001b[0m\n",
      "\u001b[34mreturn launch_agent(self._config, self._entrypoint, list(args))\n",
      "  File \"/opt/conda/lib/python3.9/site-packages/torch/distributed/launcher/api.py\", line 246, in launch_agent\u001b[0m\n",
      "\u001b[34mraise ChildFailedError(\u001b[0m\n",
      "\u001b[34mtorch.distributed.elastic.multiprocessing.errors.\u001b[0m\n",
      "\u001b[34mChildFailedError: \u001b[0m\n",
      "\u001b[34m============================================================\u001b[0m\n",
      "\u001b[34mBELLE/train/src/train.py FAILED\u001b[0m\n",
      "\u001b[34m------------------------------------------------------------\u001b[0m\n",
      "\u001b[34mFailures:\u001b[0m\n",
      "\u001b[34m[1]:\n",
      "  time      : 2023-05-23_11:14:03\n",
      "  host      : algo-1\n",
      "  rank      : 1 (local_rank: 1)\n",
      "  exitcode  : 1 (pid: 215)\n",
      "  error_file: <N/A>\n",
      "  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html\u001b[0m\n",
      "\u001b[34m[2]:\n",
      "  time      : 2023-05-23_11:14:03\n",
      "  host      : algo-1\n",
      "  rank      : 2 (local_rank: 2)\n",
      "  exitcode  : 1 (pid: 216)\n",
      "  error_file: <N/A>\n",
      "  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html\u001b[0m\n",
      "\u001b[34m[3]:\n",
      "  time      : 2023-05-23_11:14:03\n",
      "  host      : algo-1\n",
      "  rank      : 3 (local_rank: 3)\n",
      "  exitcode  : 1 (pid: 217)\n",
      "  error_file: <N/A>\n",
      "  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html\u001b[0m\n",
      "\u001b[34m[4]:\n",
      "  time      : 2023-05-23_11:14:03\n",
      "  host      : algo-1\n",
      "  rank      : 4 (local_rank: 4)\n",
      "  exitcode  : 1 (pid: 218)\n",
      "  error_file: <N/A>\n",
      "  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html\u001b[0m\n",
      "\u001b[34m[5]:\n",
      "  time      : 2023-05-23_11:14:03\n",
      "  host      : algo-1\n",
      "  rank      : 5 (local_rank: 5)\n",
      "  exitcode  : 1 (pid: 219)\n",
      "  error_file: <N/A>\n",
      "  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html\u001b[0m\n",
      "\u001b[34m[6]:\n",
      "  time      : 2023-05-23_11:14:03\n",
      "  host      : algo-1\n",
      "  rank      : 6 (local_rank: 6)\n",
      "  exitcode  : 1 (pid: 220)\n",
      "  error_file: <N/A>\n",
      "  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html\u001b[0m\n",
      "\u001b[34m[7]:\n",
      "  time      : 2023-05-23_11:14:03\n",
      "  host      : algo-1\n",
      "  rank      : 7 (local_rank: 7)\n",
      "  exitcode  : 1 (pid: 221)\n",
      "  error_file: <N/A>\n",
      "  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html\u001b[0m\n",
      "\u001b[34m------------------------------------------------------------\u001b[0m\n",
      "\u001b[34mRoot Cause (first observed failure):\u001b[0m\n",
      "\u001b[34m[0]:\n",
      "  time      : 2023-05-23_11:14:03\n",
      "  host      : algo-1\n",
      "  rank      : 0 (local_rank: 0)\n",
      "  exitcode  : 1 (pid: 214)\n",
      "  error_file: <N/A>\n",
      "  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html\u001b[0m\n",
      "\u001b[34m============================================================\u001b[0m\n",
      "\u001b[34mTraining script error, please check CloudWatch logs\u001b[0m\n",
      "\u001b[34m2023-05-23 11:14:03,618 sagemaker-training-toolkit INFO     Waiting for the process to finish and give a return code.\u001b[0m\n",
      "\u001b[34m2023-05-23 11:14:03,618 sagemaker-training-toolkit INFO     Done waiting for a return code. Received 1 from exiting process.\u001b[0m\n",
      "\u001b[34m2023-05-23 11:14:03,619 sagemaker-training-toolkit ERROR    Reporting training FAILURE\u001b[0m\n",
      "\u001b[34m2023-05-23 11:14:03,619 sagemaker-training-toolkit ERROR    ExecuteUserScriptError:\u001b[0m\n",
      "\u001b[34mExitCode 1\u001b[0m\n",
      "\u001b[34mErrorMessage \"│    54 │   │   │   except (UnicodeDecodeError, AttributeError):               │\n",
      " │    55 │   │   │   │   raise ValueError(                                      │\n",
      " │                                                                              │\n",
      " │ /opt/conda/lib/python3.9/base64.py:133 in urlsafe_b64decode                  │\n",
      " │   130 │   \"\"\"                                                                │\n",
      " │   131 │   s = _bytes_from_decode_data(s)                                     │\n",
      " │   132 │   s = s.translate(_urlsafe_decode_translation)                       │\n",
      " │ ❱ 133 │   return b64decode(s)                                                │\n",
      " │   134                                                                        │\n",
      " │   135                                                                        │\n",
      " │   136                                                                        │\n",
      " │ /opt/conda/lib/python3.9/base64.py:87 in b64decode                           │\n",
      " │    84 │   │   s = s.translate(bytes.maketrans(altchars, b'+/'))              │\n",
      " │    85 │   if validate and not re.fullmatch(b'[A-Za-z0-9+/]*={0,2}', s):      │\n",
      " │    86 │   │   raise binascii.Error('Non-base64 digit found')                 │\n",
      " │ ❱  87 │   return binascii.a2b_base64(s)                                      │\n",
      " │    88                                                                        │\n",
      " │    89                                                                        │\n",
      " │    90 def standard_b64encode(s):                                             │\n",
      " ╰──────────────────────────────────────────────────────────────────────────────╯\n",
      " Error: Invalid base64-encoded string: number of data characters (41) cannot be 1\n",
      " more than a multiple of 4\n",
      " ╭───────────────────── Traceback (most recent call last) ──────────────────────╮\n",
      " │ /opt/ml/code/BELLE/train/src/train.py:394 in <module>                        │\n",
      " │   391                                                                        │\n",
      " │   392                                                                        │\n",
      " │   393 if __name__ == \"__main__\":                                             │\n",
      " │ ❱ 394 │   main()                                                             │\n",
      " │   395                                                                        │\n",
      " │ /opt/ml/code/BELLE/train/src/train.py:163 in main                            │\n",
      " │   160                                                                        │\n",
      " │   161 def main():                                                            │\n",
      " │   162 │   parser = HfArgumentParser((ModelArguments, DataArguments, Training │\n",
      " │ ❱ 163 │   model_args, data_args, training_args = parser.parse_args_into_data │\n",
      " │   164 │                                                                      │\n",
      " │   165 │   world_size = int(os.environ.get(\"WORLD_SIZE\", 1))                  │\n",
      " │   166 │   ddp = world_size != 1                                              │\n",
      " │ /opt/conda/lib/python3.9/site-packages/transformers/hf_argparser.py:332 in   │\n",
      " │ parse_args_into_dataclasses                                                  │\n",
      " │   329 │   │   │   inputs = {k: v for k, v in vars(namespace).items() if k in │\n",
      " │   330 │   │   │   for k in keys:                                             │\n",
      " │   331 │   │   │   │   delattr(namespace, k)                                  │\n",
      " │ ❱ 332 │   │   │   obj = dtype(**inputs)                                      │\n",
      " │   333 │   │   │   outputs.append(obj)                                        │\n",
      " │   334 │   │   if len(namespace.__dict__) > 0:                                │\n",
      " │   335 │   │   │   # additional namespace.                                    │\n",
      " │ <string>:114 in __init__                                                     │\n",
      " │ /opt/conda/lib/python3.9/site-packages/transformers/training_args.py:1446 in │\n",
      " │ __post_init__                                                                │\n",
      " │   1443 │   │   │                                                             │\n",
      " │   1444 │   │   │   # will be used later by the Trainer                       │\n",
      " │   1445 │   │   │   # note: leave self.deepspeed unmodified in case a user re │\n",
      " │ ❱ 1446 │   │   │   self.hf_deepspeed_config = HfTrainerDeepSpeedConfig(self. │\n",
      " │   1447 │   │   │   self.hf_deepspeed_config.trainer_config_process(self)     │\n",
      " │   1448 │   │                                                                 │\n",
      " │   1449 │   │   if self.push_to_hub_token is not None:                        │\n",
      " │ /opt/conda/lib/python3.9/site-packages/transformers/deepspeed.py:77 in       │\n",
      " │ __init__                                                                     │\n",
      " │    74 │   \"\"\"                                                                │\n",
      " │    75 │                                                                      │\n",
      " │    76 │   def __init__(self, config_file_or_dict):                           │\n",
      " │ ❱  77 │   │   super().__init__(config_file_or_dict)                          │\n",
      " │    78 │   │   self._dtype = None                                             │\n",
      " │    79 │   │   self.mismatches = []                                           │\n",
      " │    80                                                                        │\n",
      " │ /opt/conda/lib/python3.9/site-packages/transformers/deepspeed.py:67 in       │\n",
      " │    64 │   │   set_hf_deepspeed_config(self)                                  │\n",
      " │    65 │   │   dep_version_check(\"accelerate\")                                │\n",
      " │    66 │   │   dep_version_check(\"deepspeed\")                                 │\n",
      " │ ❱  67 │   │   super().__init__(config_file_or_dict)                          │\n",
      " │    68                                                                        │\n",
      " │    69                                                                        │\n",
      " │    70 class HfTrainerDeepSpeedConfig(HfDeepSpeedConfig):                     │\n",
      " │ /opt/conda/lib/python3.9/site-packages/accelerate/utils/deepspeed.py:52 in   │\n",
      " │    49 │   │   │   │   config = json.load(f)                                  │\n",
      " │    50 │   │   else:                                                          │\n",
      " │    51 │   │   │   try:                                                       │\n",
      " │ ❱  52 │   │   │   │   config_decoded = base64.urlsafe_b64decode(config_file_ │\n",
      " │    53 │   │   │   │   config = json.loads(config_decoded)                    │\n",
      " ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 214) of binary: /opt/conda/bin/python3.9\n",
      " Traceback (most recent call last)\n",
      " File \"/opt/conda/bin/torchrun\", line 8, in <module>\n",
      " sys.exit(main())\n",
      " File \"/opt/conda/lib/python3.9/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py\", line 346, in wrapper\n",
      " return f(*args, **kwargs)\n",
      " File \"/opt/conda/lib/python3.9/site-packages/torch/distributed/run.py\", line 762, in main\n",
      " run(args)\n",
      " File \"/opt/conda/lib/python3.9/site-packages/torch/distributed/run.py\", line 753, in run\n",
      " elastic_launch(\n",
      " File \"/opt/conda/lib/python3.9/site-packages/torch/distributed/launcher/api.py\", line 132, in __call__\n",
      " return launch_agent(self._config, self._entrypoint, list(args))\n",
      " File \"/opt/conda/lib/python3.9/site-packages/torch/distributed/launcher/api.py\", line 246, in launch_agent\n",
      " raise ChildFailedError(\n",
      " torch.distributed.elastic.multiprocessing.errors.\n",
      " ChildFailedError\n",
      " ============================================================\n",
      " BELLE/train/src/train.py FAILED\n",
      " ------------------------------------------------------------\n",
      " Failures\n",
      " [1]\n",
      " time      : 2023-05-23_11:14:03\n",
      " host      : algo-1\n",
      " rank      : 1 (local_rank: 1)\n",
      " exitcode  : 1 (pid: 215)\n",
      " error_file: <N/A>\n",
      " traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html\n",
      " [2]\n",
      " rank      : 2 (local_rank: 2)\n",
      " exitcode  : 1 (pid: 216)\n",
      " [3]\n",
      " rank      : 3 (local_rank: 3)\n",
      " exitcode  : 1 (pid: 217)\n",
      " [4]\n",
      " rank      : 4 (local_rank: 4)\n",
      " exitcode  : 1 (pid: 218)\n",
      " [5]\n",
      " rank      : 5 (local_rank: 5)\n",
      " exitcode  : 1 (pid: 219)\n",
      " [6]\n",
      " rank      : 6 (local_rank: 6)\n",
      " exitcode  : 1 (pid: 220)\n",
      " [7]\n",
      " rank      : 7 (local_rank: 7)\n",
      " exitcode  : 1 (pid: 221)\n",
      " Root Cause (first observed failure)\n",
      " [0]\n",
      " rank      : 0 (local_rank: 0)\n",
      " exitcode  : 1 (pid: 214)\"\u001b[0m\n",
      "\u001b[34mCommand \"/bin/sh -c \"./train.sh \"\"\u001b[0m\n",
      "\u001b[34m2023-05-23 11:14:03,619 sagemaker-training-toolkit ERROR    Encountered exit_code 1\u001b[0m\n",
      "\n",
      "2023-05-23 11:14:17 Uploading - Uploading generated training model\n",
      "2023-05-23 11:14:17 Failed - Training job failed\n"
     ]
    },
    {
     "ename": "UnexpectedStatusException",
     "evalue": "Error for Training job bloomz20230523v1-2023-05-23-11-03-19-778: Failed. Reason: AlgorithmError: ExecuteUserScriptError:\nExitCode 1\nErrorMessage \"│    54 │   │   │   except (UnicodeDecodeError, AttributeError):               │\n │    55 │   │   │   │   raise ValueError(                                      │\n │                                                                              │\n │ /opt/conda/lib/python3.9/base64.py:133 in urlsafe_b64decode                  │\n │   130 │   \"\"\"                                                                │\n │   131 │   s = _bytes_from_decode_data(s)                                     │\n │   132 │   s = s.translate(_urlsafe_decode_translation)                       │\n │ ❱ 133 │   return b64decode(s)                                                │\n │   134                                                                        │\n │   135                                                                        │\n │   136                                                                        │\n │ /, exit code: 1",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mUnexpectedStatusException\u001b[0m                 Traceback (most recent call last)",
      "\u001b[0;32m/tmp/ipykernel_2441/173670645.py\u001b[0m in \u001b[0;36m<cell line: 24>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     22\u001b[0m                       max_run=24*60*60*3)\n\u001b[1;32m     23\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 24\u001b[0;31m \u001b[0mestimator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     25\u001b[0m \u001b[0;31m# estimator.fit(inputs)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages/sagemaker/workflow/pipeline_context.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    298\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0m_StepArguments\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mretrieve_caller_name\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself_instance\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrun_func\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    299\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 300\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mrun_func\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    301\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    302\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages/sagemaker/estimator.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, inputs, wait, logs, job_name, experiment_config)\u001b[0m\n\u001b[1;32m   1219\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjobs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlatest_training_job\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1220\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mwait\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1221\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlatest_training_job\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlogs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1222\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1223\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_compilation_job_name\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages/sagemaker/estimator.py\u001b[0m in \u001b[0;36mwait\u001b[0;34m(self, logs)\u001b[0m\n\u001b[1;32m   2365\u001b[0m         \u001b[0;31m# If logs are requested, call logs_for_jobs.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2366\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlogs\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;34m\"None\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2367\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msagemaker_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlogs_for_job\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjob_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwait\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlog_type\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2368\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2369\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msagemaker_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait_for_job\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjob_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages/sagemaker/session.py\u001b[0m in \u001b[0;36mlogs_for_job\u001b[0;34m(self, job_name, wait, poll, log_type, timeout)\u001b[0m\n\u001b[1;32m   4663\u001b[0m             \u001b[0mexceptions\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mUnexpectedStatusException\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mIf\u001b[0m \u001b[0mwaiting\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mtraining\u001b[0m \u001b[0mjob\u001b[0m \u001b[0mfails\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   4664\u001b[0m         \"\"\"\n\u001b[0;32m-> 4665\u001b[0;31m         \u001b[0m_logs_for_job\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mboto_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mjob_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwait\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpoll\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlog_type\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   4666\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   4667\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mlogs_for_processing_job\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mjob_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwait\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpoll\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages/sagemaker/session.py\u001b[0m in \u001b[0;36m_logs_for_job\u001b[0;34m(boto_session, job_name, wait, poll, log_type, timeout)\u001b[0m\n\u001b[1;32m   6525\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   6526\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mwait\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 6527\u001b[0;31m         \u001b[0m_check_job_status\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjob_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdescription\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"TrainingJobStatus\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   6528\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mdot\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   6529\u001b[0m             \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch_p39/lib/python3.9/site-packages/sagemaker/session.py\u001b[0m in \u001b[0;36m_check_job_status\u001b[0;34m(job, desc, status_key_name)\u001b[0m\n\u001b[1;32m   6578\u001b[0m                 \u001b[0mactual_status\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mstatus\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   6579\u001b[0m             )\n\u001b[0;32m-> 6580\u001b[0;31m         raise exceptions.UnexpectedStatusException(\n\u001b[0m\u001b[1;32m   6581\u001b[0m             \u001b[0mmessage\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   6582\u001b[0m             \u001b[0mallowed_statuses\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"Completed\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"Stopped\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mUnexpectedStatusException\u001b[0m: Error for Training job bloomz20230523v1-2023-05-23-11-03-19-778: Failed. Reason: AlgorithmError: ExecuteUserScriptError:\nExitCode 1\nErrorMessage \"│    54 │   │   │   except (UnicodeDecodeError, AttributeError):               │\n │    55 │   │   │   │   raise ValueError(                                      │\n │                                                                              │\n │ /opt/conda/lib/python3.9/base64.py:133 in urlsafe_b64decode                  │\n │   130 │   \"\"\"                                                                │\n │   131 │   s = _bytes_from_decode_data(s)                                     │\n │   132 │   s = s.translate(_urlsafe_decode_translation)                       │\n │ ❱ 133 │   return b64decode(s)                                                │\n │   134                                                                        │\n │   135                                                                        │\n │   136                                                                        │\n │ /, exit code: 1"
     ]
    }
   ],
   "source": [
    "import time\n",
    "from sagemaker.estimator import Estimator\n",
    "\n",
    "environment = {\n",
    "              'MODEL_S3_BUCKET': 'zmoolb-ryhrefy1ejk8epxttmfkfagrde4w6usw2a-s3alias' # 改一下s3的桶地址\n",
    "}\n",
    "\n",
    "base_job_name = 'bloomz20230523v1'         \n",
    "\n",
    "instance_type = 'ml.p4d.24xlarge'\n",
    "\n",
    "estimator = Estimator(role=role,\n",
    "                      entry_point='train.sh',\n",
    "                      source_dir='./',\n",
    "                      base_job_name=base_job_name,\n",
    "                      instance_count=1,\n",
    "                      instance_type=instance_type,\n",
    "                      image_uri=image_uri,\n",
    "                      environment=environment,\n",
    "                      disable_profiler=True,\n",
    "                      debugger_hook_config=False,\n",
    "                      max_run=24*60*60*3)\n",
    "\n",
    "estimator.fit()\n",
    "# estimator.fit(inputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "56e920de-7dc2-4a39-9291-a4168dd36f9a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_pytorch_p39",
   "language": "python",
   "name": "conda_pytorch_p39"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}