LLM_sarcasm_headlines_fine_tuning.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "TPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "410936a490644750b022dabec89e0e6b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_3012b12af14b4fe08e291cfe139790c8",
              "IPY_MODEL_4b3b94b103e94291a428aea8550f93ba",
              "IPY_MODEL_84c50577ebc34018a4b8812eeea02324"
            ],
            "layout": "IPY_MODEL_0e4980829e3e48cc956f381110198223"
          }
        },
        "3012b12af14b4fe08e291cfe139790c8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_72f99455fd61484ea15df104598cb826",
            "placeholder": "​",
            "style": "IPY_MODEL_24e3d146eec846e1b8cb4e2cc7bfc6c0",
            "value": "Loading checkpoint shards: 100%"
          }
        },
        "4b3b94b103e94291a428aea8550f93ba": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3acdeb3343094dc5b16bc6930c3c506a",
            "max": 14,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_7e4fcaeef1f348a7ab04c7067015637f",
            "value": 14
          }
        },
        "84c50577ebc34018a4b8812eeea02324": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a146cb00d3374ea7ac2621bfc2ccbc65",
            "placeholder": "​",
            "style": "IPY_MODEL_11ce011d17e749fda4179a1b5498e853",
            "value": " 14/14 [01:53&lt;00:00,  7.85s/it]"
          }
        },
        "0e4980829e3e48cc956f381110198223": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "72f99455fd61484ea15df104598cb826": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "24e3d146eec846e1b8cb4e2cc7bfc6c0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3acdeb3343094dc5b16bc6930c3c506a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7e4fcaeef1f348a7ab04c7067015637f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "a146cb00d3374ea7ac2621bfc2ccbc65": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "11ce011d17e749fda4179a1b5498e853": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "p0HT8s5S3RWv",
        "outputId": "7cc001b8-db1d-4ff1-ad58-fea4d633bbc4"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting autotrain-advanced\n",
            "  Downloading autotrain_advanced-0.6.27-py3-none-any.whl (118 kB)\n",
            "\u001b[?25l     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/118.6 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K     \u001b[91m━━━━━━━━━━━━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m41.0/118.6 kB\u001b[0m \u001b[31m1.1 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m118.6/118.6 kB\u001b[0m \u001b[31m2.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: albumentations==1.3.1 in /usr/local/lib/python3.10/dist-packages (from autotrain-advanced) (1.3.1)\n",
            "Collecting codecarbon==2.2.3 (from autotrain-advanced)\n",
            "  Downloading codecarbon-2.2.3-py3-none-any.whl (174 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m174.1/174.1 kB\u001b[0m \u001b[31m15.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting datasets[vision]~=2.14.0 (from autotrain-advanced)\n",
            "  Downloading datasets-2.14.4-py3-none-any.whl (519 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m519.3/519.3 kB\u001b[0m \u001b[31m43.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting evaluate==0.3.0 (from autotrain-advanced)\n",
            "  Downloading evaluate-0.3.0-py3-none-any.whl (72 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m72.9/72.9 kB\u001b[0m \u001b[31m10.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting ipadic==1.0.0 (from autotrain-advanced)\n",
            "  Downloading ipadic-1.0.0.tar.gz (13.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m13.4/13.4 MB\u001b[0m \u001b[31m44.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting jiwer==3.0.2 (from autotrain-advanced)\n",
            "  Downloading jiwer-3.0.2-py3-none-any.whl (21 kB)\n",
            "Collecting joblib==1.3.1 (from autotrain-advanced)\n",
            "  Downloading joblib-1.3.1-py3-none-any.whl (301 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m302.0/302.0 kB\u001b[0m \u001b[31m28.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting loguru==0.7.0 (from autotrain-advanced)\n",
            "  Downloading loguru-0.7.0-py3-none-any.whl (59 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m60.0/60.0 kB\u001b[0m \u001b[31m7.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pandas==2.0.3 (from autotrain-advanced)\n",
            "  Downloading pandas-2.0.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (12.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m12.3/12.3 MB\u001b[0m \u001b[31m34.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting Pillow==10.0.0 (from autotrain-advanced)\n",
            "  Downloading Pillow-10.0.0-cp310-cp310-manylinux_2_28_x86_64.whl (3.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.4/3.4 MB\u001b[0m \u001b[31m107.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting protobuf==4.23.4 (from autotrain-advanced)\n",
            "  Downloading protobuf-4.23.4-cp37-abi3-manylinux2014_x86_64.whl (304 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m304.5/304.5 kB\u001b[0m \u001b[31m32.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pydantic==1.10.11 (from autotrain-advanced)\n",
            "  Downloading pydantic-1.10.11-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.1/3.1 MB\u001b[0m \u001b[31m112.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting sacremoses==0.0.53 (from autotrain-advanced)\n",
            "  Downloading sacremoses-0.0.53.tar.gz (880 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m880.6/880.6 kB\u001b[0m \u001b[31m66.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting scikit-learn==1.3.0 (from autotrain-advanced)\n",
            "  Downloading scikit_learn-1.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (10.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m10.8/10.8 MB\u001b[0m \u001b[31m124.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting sentencepiece==0.1.99 (from autotrain-advanced)\n",
            "  Downloading sentencepiece-0.1.99-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m6.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting tqdm==4.65.0 (from autotrain-advanced)\n",
            "  Downloading tqdm-4.65.0-py3-none-any.whl (77 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m77.1/77.1 kB\u001b[0m \u001b[31m9.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting werkzeug==2.3.6 (from autotrain-advanced)\n",
            "  Downloading Werkzeug-2.3.6-py3-none-any.whl (242 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m242.5/242.5 kB\u001b[0m \u001b[31m28.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting huggingface-hub>=0.16.4 (from autotrain-advanced)\n",
            "  Downloading huggingface_hub-0.16.4-py3-none-any.whl (268 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m268.8/268.8 kB\u001b[0m \u001b[31m30.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: requests==2.31.0 in /usr/local/lib/python3.10/dist-packages (from autotrain-advanced) (2.31.0)\n",
            "Collecting gradio==3.39.0 (from autotrain-advanced)\n",
            "  Downloading gradio-3.39.0-py3-none-any.whl (19.9 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m19.9/19.9 MB\u001b[0m \u001b[31m19.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting einops==0.6.1 (from autotrain-advanced)\n",
            "  Downloading einops-0.6.1-py3-none-any.whl (42 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m42.2/42.2 kB\u001b[0m \u001b[31m5.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting invisible-watermark==0.2.0 (from autotrain-advanced)\n",
            "  Downloading invisible_watermark-0.2.0-py3-none-any.whl (1.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.6/1.6 MB\u001b[0m \u001b[31m93.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: tensorboard in /usr/local/lib/python3.10/dist-packages (from autotrain-advanced) (2.12.3)\n",
            "Collecting peft (from autotrain-advanced)\n",
            "  Downloading peft-0.5.0-py3-none-any.whl (85 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m85.6/85.6 kB\u001b[0m \u001b[31m10.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting trl (from autotrain-advanced)\n",
            "  Downloading trl-0.7.1-py3-none-any.whl (117 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m118.0/118.0 kB\u001b[0m \u001b[31m15.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting tiktoken (from autotrain-advanced)\n",
            "  Downloading tiktoken-0.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.7 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.7/1.7 MB\u001b[0m \u001b[31m96.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting transformers (from autotrain-advanced)\n",
            "  Downloading transformers-4.32.1-py3-none-any.whl (7.5 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.5/7.5 MB\u001b[0m \u001b[31m112.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting accelerate (from autotrain-advanced)\n",
            "  Downloading accelerate-0.22.0-py3-none-any.whl (251 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m251.2/251.2 kB\u001b[0m \u001b[31m30.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting diffusers (from autotrain-advanced)\n",
            "  Downloading diffusers-0.20.1-py3-none-any.whl (1.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m86.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting bitsandbytes (from autotrain-advanced)\n",
            "  Downloading bitsandbytes-0.41.1-py3-none-any.whl (92.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m92.6/92.6 MB\u001b[0m \u001b[31m9.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.11.1 in /usr/local/lib/python3.10/dist-packages (from albumentations==1.3.1->autotrain-advanced) (1.23.5)\n",
            "Requirement already satisfied: scipy>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from albumentations==1.3.1->autotrain-advanced) (1.10.1)\n",
            "Requirement already satisfied: scikit-image>=0.16.1 in /usr/local/lib/python3.10/dist-packages (from albumentations==1.3.1->autotrain-advanced) (0.19.3)\n",
            "Requirement already satisfied: PyYAML in /usr/local/lib/python3.10/dist-packages (from albumentations==1.3.1->autotrain-advanced) (6.0.1)\n",
            "Requirement already satisfied: qudida>=0.0.4 in /usr/local/lib/python3.10/dist-packages (from albumentations==1.3.1->autotrain-advanced) (0.0.4)\n",
            "Requirement already satisfied: opencv-python-headless>=4.1.1 in /usr/local/lib/python3.10/dist-packages (from albumentations==1.3.1->autotrain-advanced) (4.8.0.76)\n",
            "Collecting arrow (from codecarbon==2.2.3->autotrain-advanced)\n",
            "  Downloading arrow-1.2.3-py3-none-any.whl (66 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m66.4/66.4 kB\u001b[0m \u001b[31m8.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pynvml (from codecarbon==2.2.3->autotrain-advanced)\n",
            "  Downloading pynvml-11.5.0-py3-none-any.whl (53 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m53.1/53.1 kB\u001b[0m \u001b[31m6.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from codecarbon==2.2.3->autotrain-advanced) (5.9.5)\n",
            "Requirement already satisfied: py-cpuinfo in /usr/local/lib/python3.10/dist-packages (from codecarbon==2.2.3->autotrain-advanced) (9.0.0)\n",
            "Collecting fuzzywuzzy (from codecarbon==2.2.3->autotrain-advanced)\n",
            "  Downloading fuzzywuzzy-0.18.0-py2.py3-none-any.whl (18 kB)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from codecarbon==2.2.3->autotrain-advanced) (8.1.7)\n",
            "Collecting dill (from evaluate==0.3.0->autotrain-advanced)\n",
            "  Downloading dill-0.3.7-py3-none-any.whl (115 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m115.3/115.3 kB\u001b[0m \u001b[31m15.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting xxhash (from evaluate==0.3.0->autotrain-advanced)\n",
            "  Downloading xxhash-3.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (194 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m194.1/194.1 kB\u001b[0m \u001b[31m23.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting multiprocess (from evaluate==0.3.0->autotrain-advanced)\n",
            "  Downloading multiprocess-0.70.15-py310-none-any.whl (134 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m17.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: fsspec[http]>=2021.05.0 in /usr/local/lib/python3.10/dist-packages (from evaluate==0.3.0->autotrain-advanced) (2023.6.0)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from evaluate==0.3.0->autotrain-advanced) (23.1)\n",
            "Collecting responses<0.19 (from evaluate==0.3.0->autotrain-advanced)\n",
            "  Downloading responses-0.18.0-py3-none-any.whl (38 kB)\n",
            "Collecting aiofiles<24.0,>=22.0 (from gradio==3.39.0->autotrain-advanced)\n",
            "  Downloading aiofiles-23.2.1-py3-none-any.whl (15 kB)\n",
            "Requirement already satisfied: aiohttp~=3.0 in /usr/local/lib/python3.10/dist-packages (from gradio==3.39.0->autotrain-advanced) (3.8.5)\n",
            "Requirement already satisfied: altair<6.0,>=4.2.0 in /usr/local/lib/python3.10/dist-packages (from gradio==3.39.0->autotrain-advanced) (4.2.2)\n",
            "Collecting fastapi (from gradio==3.39.0->autotrain-advanced)\n",
            "  Downloading fastapi-0.103.0-py3-none-any.whl (66 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m66.2/66.2 kB\u001b[0m \u001b[31m4.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting ffmpy (from gradio==3.39.0->autotrain-advanced)\n",
            "  Downloading ffmpy-0.3.1.tar.gz (5.5 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting gradio-client>=0.3.0 (from gradio==3.39.0->autotrain-advanced)\n",
            "  Downloading gradio_client-0.5.0-py3-none-any.whl (298 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m298.2/298.2 kB\u001b[0m \u001b[31m31.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting httpx (from gradio==3.39.0->autotrain-advanced)\n",
            "  Downloading httpx-0.24.1-py3-none-any.whl (75 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.4/75.4 kB\u001b[0m \u001b[31m9.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: jinja2<4.0 in /usr/local/lib/python3.10/dist-packages (from gradio==3.39.0->autotrain-advanced) (3.1.2)\n",
            "Requirement already satisfied: markdown-it-py[linkify]>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from gradio==3.39.0->autotrain-advanced) (3.0.0)\n",
            "Requirement already satisfied: markupsafe~=2.0 in /usr/local/lib/python3.10/dist-packages (from gradio==3.39.0->autotrain-advanced) (2.1.3)\n",
            "Requirement already satisfied: matplotlib~=3.0 in /usr/local/lib/python3.10/dist-packages (from gradio==3.39.0->autotrain-advanced) (3.7.1)\n",
            "Collecting mdit-py-plugins<=0.3.3 (from gradio==3.39.0->autotrain-advanced)\n",
            "  Downloading mdit_py_plugins-0.3.3-py3-none-any.whl (50 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m50.5/50.5 kB\u001b[0m \u001b[31m6.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting orjson~=3.0 (from gradio==3.39.0->autotrain-advanced)\n",
            "  Downloading orjson-3.9.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (139 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m139.9/139.9 kB\u001b[0m \u001b[31m15.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pydub (from gradio==3.39.0->autotrain-advanced)\n",
            "  Downloading pydub-0.25.1-py2.py3-none-any.whl (32 kB)\n",
            "Collecting python-multipart (from gradio==3.39.0->autotrain-advanced)\n",
            "  Downloading python_multipart-0.0.6-py3-none-any.whl (45 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m45.7/45.7 kB\u001b[0m \u001b[31m4.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting semantic-version~=2.0 (from gradio==3.39.0->autotrain-advanced)\n",
            "  Downloading semantic_version-2.10.0-py2.py3-none-any.whl (15 kB)\n",
            "Requirement already satisfied: typing-extensions~=4.0 in /usr/local/lib/python3.10/dist-packages (from gradio==3.39.0->autotrain-advanced) (4.7.1)\n",
            "Collecting uvicorn>=0.14.0 (from gradio==3.39.0->autotrain-advanced)\n",
            "  Downloading uvicorn-0.23.2-py3-none-any.whl (59 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m59.5/59.5 kB\u001b[0m \u001b[31m6.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting websockets<12.0,>=10.0 (from gradio==3.39.0->autotrain-advanced)\n",
            "  Downloading websockets-11.0.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (129 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m129.9/129.9 kB\u001b[0m \u001b[31m13.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: PyWavelets>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from invisible-watermark==0.2.0->autotrain-advanced) (1.4.1)\n",
            "Requirement already satisfied: opencv-python>=4.1.0.25 in /usr/local/lib/python3.10/dist-packages (from invisible-watermark==0.2.0->autotrain-advanced) (4.8.0.76)\n",
            "Requirement already satisfied: torch in /usr/local/lib/python3.10/dist-packages (from invisible-watermark==0.2.0->autotrain-advanced) (2.0.1+cu118)\n",
            "Collecting rapidfuzz==2.13.7 (from jiwer==3.0.2->autotrain-advanced)\n",
            "  Downloading rapidfuzz-2.13.7-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.2/2.2 MB\u001b[0m \u001b[31m54.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/dist-packages (from pandas==2.0.3->autotrain-advanced) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas==2.0.3->autotrain-advanced) (2023.3)\n",
            "Collecting tzdata>=2022.1 (from pandas==2.0.3->autotrain-advanced)\n",
            "  Downloading tzdata-2023.3-py2.py3-none-any.whl (341 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m341.8/341.8 kB\u001b[0m \u001b[31m33.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests==2.31.0->autotrain-advanced) (3.2.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests==2.31.0->autotrain-advanced) (3.4)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests==2.31.0->autotrain-advanced) (2.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests==2.31.0->autotrain-advanced) (2023.7.22)\n",
            "Requirement already satisfied: regex in /usr/local/lib/python3.10/dist-packages (from sacremoses==0.0.53->autotrain-advanced) (2023.6.3)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.10/dist-packages (from sacremoses==0.0.53->autotrain-advanced) (1.16.0)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn==1.3.0->autotrain-advanced) (3.2.0)\n",
            "Requirement already satisfied: pyarrow>=8.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets[vision]~=2.14.0->autotrain-advanced) (9.0.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.16.4->autotrain-advanced) (3.12.2)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.10/dist-packages (from diffusers->autotrain-advanced) (6.8.0)\n",
            "Collecting safetensors>=0.3.1 (from diffusers->autotrain-advanced)\n",
            "  Downloading safetensors-0.3.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m72.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: absl-py>=0.4 in /usr/local/lib/python3.10/dist-packages (from tensorboard->autotrain-advanced) (1.4.0)\n",
            "Requirement already satisfied: grpcio>=1.48.2 in /usr/local/lib/python3.10/dist-packages (from tensorboard->autotrain-advanced) (1.57.0)\n",
            "Requirement already satisfied: google-auth<3,>=1.6.3 in /usr/local/lib/python3.10/dist-packages (from tensorboard->autotrain-advanced) (2.17.3)\n",
            "Requirement already satisfied: google-auth-oauthlib<1.1,>=0.5 in /usr/local/lib/python3.10/dist-packages (from tensorboard->autotrain-advanced) (1.0.0)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.10/dist-packages (from tensorboard->autotrain-advanced) (3.4.4)\n",
            "Requirement already satisfied: setuptools>=41.0.0 in /usr/local/lib/python3.10/dist-packages (from tensorboard->autotrain-advanced) (67.7.2)\n",
            "Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in /usr/local/lib/python3.10/dist-packages (from tensorboard->autotrain-advanced) (0.7.1)\n",
            "Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.10/dist-packages (from tensorboard->autotrain-advanced) (0.41.2)\n",
            "Collecting tokenizers!=0.11.3,<0.14,>=0.11.1 (from transformers->autotrain-advanced)\n",
            "  Downloading tokenizers-0.13.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.8/7.8 MB\u001b[0m \u001b[31m116.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp~=3.0->gradio==3.39.0->autotrain-advanced) (23.1.0)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp~=3.0->gradio==3.39.0->autotrain-advanced) (6.0.4)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.10/dist-packages (from aiohttp~=3.0->gradio==3.39.0->autotrain-advanced) (4.0.3)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp~=3.0->gradio==3.39.0->autotrain-advanced) (1.9.2)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp~=3.0->gradio==3.39.0->autotrain-advanced) (1.4.0)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp~=3.0->gradio==3.39.0->autotrain-advanced) (1.3.1)\n",
            "Requirement already satisfied: entrypoints in /usr/local/lib/python3.10/dist-packages (from altair<6.0,>=4.2.0->gradio==3.39.0->autotrain-advanced) (0.4)\n",
            "Requirement already satisfied: jsonschema>=3.0 in /usr/local/lib/python3.10/dist-packages (from altair<6.0,>=4.2.0->gradio==3.39.0->autotrain-advanced) (4.19.0)\n",
            "Requirement already satisfied: toolz in /usr/local/lib/python3.10/dist-packages (from altair<6.0,>=4.2.0->gradio==3.39.0->autotrain-advanced) (0.12.0)\n",
            "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from google-auth<3,>=1.6.3->tensorboard->autotrain-advanced) (5.3.1)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.10/dist-packages (from google-auth<3,>=1.6.3->tensorboard->autotrain-advanced) (0.3.0)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.10/dist-packages (from google-auth<3,>=1.6.3->tensorboard->autotrain-advanced) (4.9)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.10/dist-packages (from google-auth-oauthlib<1.1,>=0.5->tensorboard->autotrain-advanced) (1.3.1)\n",
            "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.10/dist-packages (from markdown-it-py[linkify]>=2.0.0->gradio==3.39.0->autotrain-advanced) (0.1.2)\n",
            "Requirement already satisfied: linkify-it-py<3,>=1 in /usr/local/lib/python3.10/dist-packages (from markdown-it-py[linkify]>=2.0.0->gradio==3.39.0->autotrain-advanced) (2.0.2)\n",
            "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib~=3.0->gradio==3.39.0->autotrain-advanced) (1.1.0)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/dist-packages (from matplotlib~=3.0->gradio==3.39.0->autotrain-advanced) (0.11.0)\n",
            "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib~=3.0->gradio==3.39.0->autotrain-advanced) (4.42.1)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib~=3.0->gradio==3.39.0->autotrain-advanced) (1.4.4)\n",
            "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib~=3.0->gradio==3.39.0->autotrain-advanced) (3.1.1)\n",
            "INFO: pip is looking at multiple versions of mdit-py-plugins to determine which version is compatible with other requirements. This could take a while.\n",
            "Collecting mdit-py-plugins<=0.3.3 (from gradio==3.39.0->autotrain-advanced)\n",
            "  Downloading mdit_py_plugins-0.3.2-py3-none-any.whl (50 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m50.4/50.4 kB\u001b[0m \u001b[31m6.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Downloading mdit_py_plugins-0.3.1-py3-none-any.whl (46 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m46.5/46.5 kB\u001b[0m \u001b[31m5.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Downloading mdit_py_plugins-0.3.0-py3-none-any.whl (43 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m43.7/43.7 kB\u001b[0m \u001b[31m6.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Downloading mdit_py_plugins-0.2.8-py3-none-any.whl (41 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m41.0/41.0 kB\u001b[0m \u001b[31m5.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Downloading mdit_py_plugins-0.2.7-py3-none-any.whl (41 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m41.0/41.0 kB\u001b[0m \u001b[31m4.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Downloading mdit_py_plugins-0.2.6-py3-none-any.whl (39 kB)\n",
            "  Downloading mdit_py_plugins-0.2.5-py3-none-any.whl (39 kB)\n",
            "INFO: pip is looking at multiple versions of mdit-py-plugins to determine which version is compatible with other requirements. This could take a while.\n",
            "  Downloading mdit_py_plugins-0.2.4-py3-none-any.whl (39 kB)\n",
            "  Downloading mdit_py_plugins-0.2.3-py3-none-any.whl (39 kB)\n",
            "  Downloading mdit_py_plugins-0.2.2-py3-none-any.whl (39 kB)\n",
            "  Downloading mdit_py_plugins-0.2.1-py3-none-any.whl (38 kB)\n",
            "  Downloading mdit_py_plugins-0.2.0-py3-none-any.whl (38 kB)\n",
            "INFO: This is taking longer than usual. You might need to provide the dependency resolver with stricter constraints to reduce runtime. See https://pip.pypa.io/warnings/backtracking for guidance. If you want to abort this run, press Ctrl + C.\n",
            "  Downloading mdit_py_plugins-0.1.0-py3-none-any.whl (37 kB)\n",
            "Collecting markdown-it-py[linkify]>=2.0.0 (from gradio==3.39.0->autotrain-advanced)\n",
            "  Downloading markdown_it_py-3.0.0-py3-none-any.whl (87 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m87.5/87.5 kB\u001b[0m \u001b[31m12.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Downloading markdown_it_py-2.2.0-py3-none-any.whl (84 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m84.5/84.5 kB\u001b[0m \u001b[31m10.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: networkx>=2.2 in /usr/local/lib/python3.10/dist-packages (from scikit-image>=0.16.1->albumentations==1.3.1->autotrain-advanced) (3.1)\n",
            "Requirement already satisfied: imageio>=2.4.1 in /usr/local/lib/python3.10/dist-packages (from scikit-image>=0.16.1->albumentations==1.3.1->autotrain-advanced) (2.31.1)\n",
            "Requirement already satisfied: tifffile>=2019.7.26 in /usr/local/lib/python3.10/dist-packages (from scikit-image>=0.16.1->albumentations==1.3.1->autotrain-advanced) (2023.8.12)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch->invisible-watermark==0.2.0->autotrain-advanced) (1.12)\n",
            "Requirement already satisfied: triton==2.0.0 in /usr/local/lib/python3.10/dist-packages (from torch->invisible-watermark==0.2.0->autotrain-advanced) (2.0.0)\n",
            "Requirement already satisfied: cmake in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch->invisible-watermark==0.2.0->autotrain-advanced) (3.27.2)\n",
            "Requirement already satisfied: lit in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch->invisible-watermark==0.2.0->autotrain-advanced) (16.0.6)\n",
            "Collecting h11>=0.8 (from uvicorn>=0.14.0->gradio==3.39.0->autotrain-advanced)\n",
            "  Downloading h11-0.14.0-py3-none-any.whl (58 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m7.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting starlette<0.28.0,>=0.27.0 (from fastapi->gradio==3.39.0->autotrain-advanced)\n",
            "  Downloading starlette-0.27.0-py3-none-any.whl (66 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m67.0/67.0 kB\u001b[0m \u001b[31m8.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting httpcore<0.18.0,>=0.15.0 (from httpx->gradio==3.39.0->autotrain-advanced)\n",
            "  Downloading httpcore-0.17.3-py3-none-any.whl (74 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m74.5/74.5 kB\u001b[0m \u001b[31m10.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from httpx->gradio==3.39.0->autotrain-advanced) (1.3.0)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.10/dist-packages (from importlib-metadata->diffusers->autotrain-advanced) (3.16.2)\n",
            "Requirement already satisfied: anyio<5.0,>=3.0 in /usr/local/lib/python3.10/dist-packages (from httpcore<0.18.0,>=0.15.0->httpx->gradio==3.39.0->autotrain-advanced) (3.7.1)\n",
            "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /usr/local/lib/python3.10/dist-packages (from jsonschema>=3.0->altair<6.0,>=4.2.0->gradio==3.39.0->autotrain-advanced) (2023.7.1)\n",
            "Requirement already satisfied: referencing>=0.28.4 in /usr/local/lib/python3.10/dist-packages (from jsonschema>=3.0->altair<6.0,>=4.2.0->gradio==3.39.0->autotrain-advanced) (0.30.2)\n",
            "Requirement already satisfied: rpds-py>=0.7.1 in /usr/local/lib/python3.10/dist-packages (from jsonschema>=3.0->altair<6.0,>=4.2.0->gradio==3.39.0->autotrain-advanced) (0.9.2)\n",
            "Requirement already satisfied: uc-micro-py in /usr/local/lib/python3.10/dist-packages (from linkify-it-py<3,>=1->markdown-it-py[linkify]>=2.0.0->gradio==3.39.0->autotrain-advanced) (1.0.2)\n",
            "Requirement already satisfied: pyasn1<0.6.0,>=0.4.6 in /usr/local/lib/python3.10/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard->autotrain-advanced) (0.5.0)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.10/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<1.1,>=0.5->tensorboard->autotrain-advanced) (3.2.2)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch->invisible-watermark==0.2.0->autotrain-advanced) (1.3.0)\n",
            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<5.0,>=3.0->httpcore<0.18.0,>=0.15.0->httpx->gradio==3.39.0->autotrain-advanced) (1.1.3)\n",
            "Building wheels for collected packages: ipadic, sacremoses, ffmpy\n",
            "  Building wheel for ipadic (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for ipadic: filename=ipadic-1.0.0-py3-none-any.whl size=13556703 sha256=aa5f78adf61e6fc11e25a5285c848622080b8b632fb83eb189d6e0979bc302a7\n",
            "  Stored in directory: /root/.cache/pip/wheels/5b/ea/e3/2f6e0860a327daba3b030853fce4483ed37468bbf1101c59c3\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.53-py3-none-any.whl size=895241 sha256=530b7558d6b2dda1a2a7c131c738f158a8230e6a74548dfa7577533d73817e69\n",
            "  Stored in directory: /root/.cache/pip/wheels/00/24/97/a2ea5324f36bc626e1ea0267f33db6aa80d157ee977e9e42fb\n",
            "  Building wheel for ffmpy (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for ffmpy: filename=ffmpy-0.3.1-py3-none-any.whl size=5579 sha256=cfb479b1c4d876977f4148306b409965b5f5f6d759bb3a37ebe372d3b065d3d3\n",
            "  Stored in directory: /root/.cache/pip/wheels/01/a6/d1/1c0828c304a4283b2c1639a09ad86f83d7c487ef34c6b4a1bf\n",
            "Successfully built ipadic sacremoses ffmpy\n",
            "Installing collected packages: tokenizers, sentencepiece, safetensors, pydub, ipadic, fuzzywuzzy, ffmpy, bitsandbytes, xxhash, werkzeug, websockets, tzdata, tqdm, semantic-version, rapidfuzz, python-multipart, pynvml, pydantic, protobuf, Pillow, orjson, markdown-it-py, loguru, joblib, h11, einops, dill, aiofiles, uvicorn, tiktoken, starlette, scikit-learn, sacremoses, responses, pandas, multiprocess, mdit-py-plugins, jiwer, huggingface-hub, httpcore, arrow, transformers, httpx, fastapi, diffusers, codecarbon, gradio-client, datasets, gradio, evaluate, accelerate, trl, peft, invisible-watermark, autotrain-advanced\n",
            "  Attempting uninstall: werkzeug\n",
            "    Found existing installation: Werkzeug 2.3.7\n",
            "    Uninstalling Werkzeug-2.3.7:\n",
            "      Successfully uninstalled Werkzeug-2.3.7\n",
            "  Attempting uninstall: tqdm\n",
            "    Found existing installation: tqdm 4.66.1\n",
            "    Uninstalling tqdm-4.66.1:\n",
            "      Successfully uninstalled tqdm-4.66.1\n",
            "  Attempting uninstall: pydantic\n",
            "    Found existing installation: pydantic 2.2.1\n",
            "    Uninstalling pydantic-2.2.1:\n",
            "      Successfully uninstalled pydantic-2.2.1\n",
            "  Attempting uninstall: protobuf\n",
            "    Found existing installation: protobuf 3.20.3\n",
            "    Uninstalling protobuf-3.20.3:\n",
            "      Successfully uninstalled protobuf-3.20.3\n",
            "  Attempting uninstall: Pillow\n",
            "    Found existing installation: Pillow 9.4.0\n",
            "    Uninstalling Pillow-9.4.0:\n",
            "      Successfully uninstalled Pillow-9.4.0\n",
            "  Attempting uninstall: markdown-it-py\n",
            "    Found existing installation: markdown-it-py 3.0.0\n",
            "    Uninstalling markdown-it-py-3.0.0:\n",
            "      Successfully uninstalled markdown-it-py-3.0.0\n",
            "  Attempting uninstall: joblib\n",
            "    Found existing installation: joblib 1.3.2\n",
            "    Uninstalling joblib-1.3.2:\n",
            "      Successfully uninstalled joblib-1.3.2\n",
            "  Attempting uninstall: scikit-learn\n",
            "    Found existing installation: scikit-learn 1.2.2\n",
            "    Uninstalling scikit-learn-1.2.2:\n",
            "      Successfully uninstalled scikit-learn-1.2.2\n",
            "  Attempting uninstall: pandas\n",
            "    Found existing installation: pandas 1.5.3\n",
            "    Uninstalling pandas-1.5.3:\n",
            "      Successfully uninstalled pandas-1.5.3\n",
            "  Attempting uninstall: mdit-py-plugins\n",
            "    Found existing installation: mdit-py-plugins 0.4.0\n",
            "    Uninstalling mdit-py-plugins-0.4.0:\n",
            "      Successfully uninstalled mdit-py-plugins-0.4.0\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "google-colab 1.0.0 requires pandas==1.5.3, but you have pandas 2.0.3 which is incompatible.\n",
            "tensorflow-metadata 1.14.0 requires protobuf<4.21,>=3.20.3, but you have protobuf 4.23.4 which is incompatible.\u001b[0m\u001b[31m\n",
            "\u001b[0mSuccessfully installed Pillow-10.0.0 accelerate-0.22.0 aiofiles-23.2.1 arrow-1.2.3 autotrain-advanced-0.6.27 bitsandbytes-0.41.1 codecarbon-2.2.3 datasets-2.14.4 diffusers-0.20.1 dill-0.3.7 einops-0.6.1 evaluate-0.3.0 fastapi-0.103.0 ffmpy-0.3.1 fuzzywuzzy-0.18.0 gradio-3.39.0 gradio-client-0.5.0 h11-0.14.0 httpcore-0.17.3 httpx-0.24.1 huggingface-hub-0.16.4 invisible-watermark-0.2.0 ipadic-1.0.0 jiwer-3.0.2 joblib-1.3.1 loguru-0.7.0 markdown-it-py-2.2.0 mdit-py-plugins-0.3.3 multiprocess-0.70.15 orjson-3.9.5 pandas-2.0.3 peft-0.5.0 protobuf-4.23.4 pydantic-1.10.11 pydub-0.25.1 pynvml-11.5.0 python-multipart-0.0.6 rapidfuzz-2.13.7 responses-0.18.0 sacremoses-0.0.53 safetensors-0.3.3 scikit-learn-1.3.0 semantic-version-2.10.0 sentencepiece-0.1.99 starlette-0.27.0 tiktoken-0.4.0 tokenizers-0.13.3 tqdm-4.65.0 transformers-4.32.1 trl-0.7.1 tzdata-2023.3 uvicorn-0.23.2 websockets-11.0.3 werkzeug-2.3.6 xxhash-3.3.0\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "PIL"
                ]
              }
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: huggingface_hub in /usr/local/lib/python3.10/dist-packages (0.16.4)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface_hub) (3.12.2)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from huggingface_hub) (2023.6.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from huggingface_hub) (2.31.0)\n",
            "Requirement already satisfied: tqdm>=4.42.1 in /usr/local/lib/python3.10/dist-packages (from huggingface_hub) (4.65.0)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from huggingface_hub) (6.0.1)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface_hub) (4.7.1)\n",
            "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.10/dist-packages (from huggingface_hub) (23.1)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface_hub) (3.2.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface_hub) (3.4)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface_hub) (2.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface_hub) (2023.7.22)\n"
          ]
        }
      ],
      "source": [
        "# @title Install Necessary Packages\n",
        "!pip install autotrain-advanced\n",
        "!pip install huggingface_hub"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!autotrain setup --update-torch"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_YRFMWw73giI",
        "outputId": "980da618-4d7b-41c7-dd6c-a5c6ecb3641b"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "> \u001b[1mINFO    Installing latest transformers@main\u001b[0m\n",
            "> \u001b[1mINFO    Successfully installed latest transformers\u001b[0m\n",
            "> \u001b[1mINFO    Installing latest peft@main\u001b[0m\n",
            "> \u001b[1mINFO    Successfully installed latest peft\u001b[0m\n",
            "> \u001b[1mINFO    Installing latest diffusers@main\u001b[0m\n",
            "> \u001b[1mINFO    Successfully installed latest diffusers\u001b[0m\n",
            "> \u001b[1mINFO    Installing latest trl@main\u001b[0m\n",
            "> \u001b[1mINFO    Successfully installed latest trl\u001b[0m\n",
            "> \u001b[1mINFO    Installing latest xformers\u001b[0m\n",
            "> \u001b[1mINFO    Successfully installed latest xformers\u001b[0m\n",
            "> \u001b[1mINFO    Installing latest PyTorch\u001b[0m\n",
            "> \u001b[1mINFO    Successfully installed latest PyTorch\u001b[0m\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "from datetime import datetime"
      ],
      "metadata": {
        "id": "IlHIEBWE4D8U"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# @title Load dataset generated from Llama2\n",
        "# data = pd.read_csv(\"/content/sarcastic-headline/sarcastic_headline_data.csv\")\n",
        "data = pd.read_csv(\"/content/formatted_headline_data.csv\")"
      ],
      "metadata": {
        "id": "yKgxkjjD4IAW"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# @title Prompt template\n",
        "format_text = \"You are a savage, disrespectful and witty agent. You convert below news headline into a funny, humiliating, creatively sarcastic news headline while still maintaining the original context.\\n### headline: {}\\n### sarcastic_headline: {}\"\n",
        "print(format_text)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lpK0rcD_P7nD",
        "outputId": "7ba56114-a86f-4ac3-fb80-ef83b1048ea3"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "You are a savage, disrespectful and witty agent. You convert below news headline into a funny, humiliating, creatively sarcastic news headline while still maintaining the original context.\n",
            "### headline: {}\n",
            "### sarcastic_headline: {}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# @title Create a column name \"text\" for the promp template\n",
        "data['text'] = data.apply(lambda x: format_text.format(x[0], x[1]), axis=1)\n",
        "data.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "pJla3JoYR-7n",
        "outputId": "c9eaae91-e006-488f-fb72-4ce522257e5d"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                           headlines  \\\n",
              "0  High School Students Create Haunting Artwork A...   \n",
              "1  Thefts Are an Ever-Present Problem at Arts and...   \n",
              "2                   The Expressionist as Rationalist   \n",
              "3                 A Dream Is a Wish Your Heart Makes   \n",
              "4            Presenting Rock Impressario Bill Graham   \n",
              "\n",
              "                                 sarcastic_headlines  \\\n",
              "0  High school students create haunting artwork a...   \n",
              "1   Another Arts and Crafts Fair Has Been Plagued...   \n",
              "2        The Incredible Sulk of the Unreasonable Man   \n",
              "3  If You Can Dream It Up, Maybe One Day Your Bra...   \n",
              "4  Bill Graham: The Man Who Couldn't Even Impress...   \n",
              "\n",
              "                                                text  \n",
              "0  You are a savage, disrespectful and witty agen...  \n",
              "1  You are a savage, disrespectful and witty agen...  \n",
              "2  You are a savage, disrespectful and witty agen...  \n",
              "3  You are a savage, disrespectful and witty agen...  \n",
              "4  You are a savage, disrespectful and witty agen...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-6971896f-1971-44a9-814e-1ac3c5e76a8f\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>headlines</th>\n",
              "      <th>sarcastic_headlines</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>High School Students Create Haunting Artwork A...</td>\n",
              "      <td>High school students create haunting artwork a...</td>\n",
              "      <td>You are a savage, disrespectful and witty agen...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Thefts Are an Ever-Present Problem at Arts and...</td>\n",
              "      <td>Another Arts and Crafts Fair Has Been Plagued...</td>\n",
              "      <td>You are a savage, disrespectful and witty agen...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>The Expressionist as Rationalist</td>\n",
              "      <td>The Incredible Sulk of the Unreasonable Man</td>\n",
              "      <td>You are a savage, disrespectful and witty agen...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>A Dream Is a Wish Your Heart Makes</td>\n",
              "      <td>If You Can Dream It Up, Maybe One Day Your Bra...</td>\n",
              "      <td>You are a savage, disrespectful and witty agen...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Presenting Rock Impressario Bill Graham</td>\n",
              "      <td>Bill Graham: The Man Who Couldn't Even Impress...</td>\n",
              "      <td>You are a savage, disrespectful and witty agen...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-6971896f-1971-44a9-814e-1ac3c5e76a8f')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-6971896f-1971-44a9-814e-1ac3c5e76a8f button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-6971896f-1971-44a9-814e-1ac3c5e76a8f');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-7815571a-6930-467e-8cc4-fa08d44ccf79\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-7815571a-6930-467e-8cc4-fa08d44ccf79')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-7815571a-6930-467e-8cc4-fa08d44ccf79 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(data['text'][100])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qqsRNnWkUxRZ",
        "outputId": "91cd4004-bdb6-450e-f28e-07561ce039b6"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "You are a savage, disrespectful and witty agent. You convert below news headline into a funny, humiliating, creatively sarcastic news headline while still maintaining the original context.\n",
            "### headline: Former Detroit Officer Found Guilty In Videotaped Beating Of Black Man\n",
            "### sarcastic_headline: Former Detroit Cop Gets Justice For That One Time He Didn't Beat A Black Person\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data.to_csv('/content/sarcastic-headline/formatted_headline_data.csv', index=False)"
      ],
      "metadata": {
        "id": "qP42gcwRVja6"
      },
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from huggingface_hub import notebook_login\n",
        "notebook_login()"
      ],
      "metadata": {
        "id": "Ajo8EiuW3glA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "formatted_text, sarc = [], []\n",
        "for i, row in data.iterrows():\n",
        "  formatted_text.append(len(row.text))\n",
        "  sarc.append(len(row.sarcastic_headlines))\n",
        "max_tokens = np.max(formatted_text) + 5\n",
        "print(f\"max possible tokens: {max_tokens}, max length of sarcastic headline: {np.max(sarc)}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "L97Zh8Ul70vj",
        "outputId": "ab576927-2aa2-4c62-b31f-a3adca3a01dd"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "max possible tokens: 615, max length of sarcastic headline: 325\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# !autotrain llm --help"
      ],
      "metadata": {
        "id": "6Mz7CuOZ55Cl"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# @title 1 line code to fine tune a LLM\n",
        "st = datetime.now()\n",
        "print(f\"Starting time: {st}\\n\")\n",
        "\n",
        "# we are using lowest possible LLama2 model corpus(7B) that too a sharded version as the colab free GPU cannot train higher corpus models\n",
        "!autotrain llm --train --project_name 'sarcastic-headline-gen' --model TinyPixel/Llama-2-7B-bf16-sharded \\\n",
        "--data_path '/content/sarcastic-headline' \\\n",
        "--use_peft \\\n",
        "--use_int4 \\\n",
        "--learning_rate 2e-4 \\\n",
        "--train_batch_size 4 \\\n",
        "--num_train_epochs 5 \\\n",
        "--trainer sft \\\n",
        "--model_max_length max_tokens \\\n",
        "--block_size max_tokens > training.log &\n",
        "# --push_to_hub\n",
        "# --repo_id your_repo_id\n",
        "\n",
        "# One can play with train_batch_size param if they have higher GPU RAM, for colab free version we cannot go more than 4\n",
        "\n",
        "# model_max_length is how much max length a model should output (This will also include prompt template length).\n",
        "# We need to set it efficiently, coz for a bigger number it will consume more GPU for this task. We dont need length more than the max length after formatting data\n",
        "\n",
        "# Your dataset should provide batches of the fixed size and block_size is for this purpose.\n",
        "# If an input is too long, it will be truncated to blocks of the same size.\n",
        "\n",
        "en = datetime.now()\n",
        "print(\"\\nTime taken to complete the training: \", en-st)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "E75QPO2L3gnZ",
        "outputId": "3f9a8d72-e41e-4b88-cc78-fab20a112e59"
      },
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "> \u001b[1mINFO    Running LLM\u001b[0m\n",
            "> \u001b[1mINFO    Params: Namespace(version=False, train=True, deploy=False, inference=False, data_path='/content/sarcastic-headline', train_split='train', valid_split=None, text_column='text', model='TinyPixel/Llama-2-7B-bf16-sharded', learning_rate=0.0002, num_train_epochs=5, train_batch_size=4, warmup_ratio=0.1, gradient_accumulation_steps=1, optimizer='adamw_torch', scheduler='linear', weight_decay=0.0, max_grad_norm=1.0, seed=42, add_eos_token=False, block_size=-1, use_peft=True, lora_r=16, lora_alpha=32, lora_dropout=0.05, logging_steps=-1, project_name='sarcastic-headline-gen', evaluation_strategy='epoch', save_total_limit=1, save_strategy='epoch', auto_find_batch_size=False, fp16=False, push_to_hub=False, use_int8=False, model_max_length=600, repo_id=None, use_int4=True, trainer='sft', target_modules=None, merge_adapter=False, token=None, backend='default', username=None, func=<function run_llm_command_factory at 0x7e01639cd6c0>)\u001b[0m\n",
            "Downloading data files: 100% 1/1 [00:00<00:00, 6260.16it/s]\n",
            "Extracting data files: 100% 1/1 [00:00<00:00, 1023.50it/s]\n",
            "Generating train split: 2100 examples [00:00, 38261.05 examples/s]\n",
            "Downloading (…)okenizer_config.json: 100% 676/676 [00:00<00:00, 1.62MB/s]\n",
            "Downloading tokenizer.model: 100% 500k/500k [00:00<00:00, 58.6MB/s]\n",
            "Downloading (…)/main/tokenizer.json: 100% 1.84M/1.84M [00:00<00:00, 3.87MB/s]\n",
            "Downloading (…)cial_tokens_map.json: 100% 411/411 [00:00<00:00, 1.75MB/s]\n",
            "Using pad_token, but it is not set yet.\n",
            "Downloading (…)lve/main/config.json: 100% 626/626 [00:00<00:00, 591kB/s]\n",
            "Downloading (…)model.bin.index.json: 100% 26.8k/26.8k [00:00<00:00, 46.9MB/s]\n",
            "Downloading shards:   0% 0/14 [00:00<?, ?it/s]\n",
            "Downloading (…)l-00001-of-00014.bin:   0% 0.00/981M [00:00<?, ?B/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00014.bin:   4% 41.9M/981M [00:00<00:02, 415MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00014.bin:  10% 94.4M/981M [00:00<00:01, 451MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00014.bin:  15% 147M/981M [00:00<00:02, 383MB/s] \u001b[A\n",
            "Downloading (…)l-00001-of-00014.bin:  19% 189M/981M [00:00<00:02, 372MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00014.bin:  24% 231M/981M [00:00<00:02, 337MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00014.bin:  28% 273M/981M [00:00<00:02, 340MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00014.bin:  32% 315M/981M [00:01<00:02, 256MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00014.bin:  35% 346M/981M [00:01<00:02, 258MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00014.bin:  38% 377M/981M [00:01<00:02, 203MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00014.bin:  42% 409M/981M [00:01<00:03, 191MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00014.bin:  45% 440M/981M [00:01<00:02, 206MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00014.bin:  48% 472M/981M [00:01<00:02, 195MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00014.bin:  51% 503M/981M [00:01<00:02, 218MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00014.bin:  56% 545M/981M [00:02<00:01, 250MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00014.bin:  60% 587M/981M [00:02<00:01, 275MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00014.bin:  64% 629M/981M [00:02<00:01, 305MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00014.bin:  68% 671M/981M [00:02<00:01, 285MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00014.bin:  72% 703M/981M [00:02<00:01, 221MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00014.bin:  75% 734M/981M [00:02<00:01, 222MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00014.bin:  78% 765M/981M [00:03<00:01, 213MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00014.bin:  81% 797M/981M [00:03<00:00, 226MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00014.bin:  84% 828M/981M [00:05<00:04, 34.5MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00014.bin:  88% 860M/981M [00:06<00:02, 45.5MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00014.bin:  90% 881M/981M [00:06<00:01, 52.9MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00014.bin:  92% 902M/981M [00:06<00:01, 61.7MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00014.bin:  95% 933M/981M [00:06<00:00, 83.3MB/s]\u001b[A\n",
            "Downloading (…)l-00001-of-00014.bin: 100% 981M/981M [00:06<00:00, 144MB/s]\n",
            "Downloading shards:   7% 1/14 [00:07<01:32,  7.10s/it]\n",
            "Downloading (…)l-00002-of-00014.bin:   0% 0.00/967M [00:00<?, ?B/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00014.bin:   4% 41.9M/967M [00:00<00:02, 399MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00014.bin:  10% 94.4M/967M [00:00<00:02, 427MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00014.bin:  15% 147M/967M [00:00<00:02, 358MB/s] \u001b[A\n",
            "Downloading (…)l-00002-of-00014.bin:  20% 189M/967M [00:00<00:02, 347MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00014.bin:  24% 231M/967M [00:00<00:02, 338MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00014.bin:  28% 273M/967M [00:00<00:02, 337MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00014.bin:  33% 315M/967M [00:00<00:01, 340MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00014.bin:  37% 357M/967M [00:01<00:01, 339MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00014.bin:  41% 398M/967M [00:01<00:01, 340MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00014.bin:  46% 440M/967M [00:01<00:01, 322MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00014.bin:  50% 482M/967M [00:01<00:01, 305MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00014.bin:  53% 514M/967M [00:01<00:01, 296MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00014.bin:  56% 545M/967M [00:01<00:01, 282MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00014.bin:  60% 577M/967M [00:01<00:01, 271MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00014.bin:  63% 608M/967M [00:01<00:01, 243MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00014.bin:  66% 640M/967M [00:02<00:01, 239MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00014.bin:  69% 671M/967M [00:02<00:01, 182MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00014.bin:  72% 692M/967M [00:02<00:01, 176MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00014.bin:  74% 713M/967M [00:02<00:01, 179MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00014.bin:  77% 744M/967M [00:02<00:01, 197MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00014.bin:  79% 765M/967M [00:02<00:01, 175MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00014.bin:  81% 786M/967M [00:03<00:01, 119MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00014.bin:  84% 807M/967M [00:03<00:01, 133MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00014.bin:  86% 828M/967M [00:03<00:01, 83.0MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00014.bin:  89% 860M/967M [00:03<00:00, 111MB/s] \u001b[A\n",
            "Downloading (…)l-00002-of-00014.bin:  91% 881M/967M [00:04<00:00, 118MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00014.bin:  93% 902M/967M [00:04<00:00, 122MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00014.bin:  95% 923M/967M [00:04<00:00, 135MB/s]\u001b[A\n",
            "Downloading (…)l-00002-of-00014.bin: 100% 967M/967M [00:04<00:00, 210MB/s]\n",
            "Downloading shards:  14% 2/14 [00:11<01:09,  5.79s/it]\n",
            "Downloading (…)l-00003-of-00014.bin:   0% 0.00/967M [00:00<?, ?B/s]\u001b[A\n",
            "Downloading (…)l-00003-of-00014.bin:   3% 31.5M/967M [00:00<00:03, 298MB/s]\u001b[A\n",
            "Downloading (…)l-00003-of-00014.bin:   7% 62.9M/967M [00:00<00:03, 256MB/s]\u001b[A\n",
            "Downloading (…)l-00003-of-00014.bin:  10% 94.4M/967M [00:00<00:03, 251MB/s]\u001b[A\n",
            "Downloading (…)l-00003-of-00014.bin:  13% 126M/967M [00:00<00:03, 243MB/s] \u001b[A\n",
            "Downloading (…)l-00003-of-00014.bin:  16% 157M/967M [00:00<00:03, 240MB/s]\u001b[A\n",
            "Downloading (…)l-00003-of-00014.bin:  20% 189M/967M [00:00<00:03, 233MB/s]\u001b[A\n",
            "Downloading (…)l-00003-of-00014.bin:  23% 220M/967M [00:00<00:03, 238MB/s]\u001b[A\n",
            "Downloading (…)l-00003-of-00014.bin:  26% 252M/967M [00:01<00:03, 226MB/s]\u001b[A\n",
            "Downloading (…)l-00003-of-00014.bin:  29% 283M/967M [00:01<00:02, 232MB/s]\u001b[A\n",
            "Downloading (…)l-00003-of-00014.bin:  33% 315M/967M [00:01<00:02, 245MB/s]\u001b[A\n",
            "Downloading (…)l-00003-of-00014.bin:  36% 346M/967M [00:01<00:02, 247MB/s]\u001b[A\n",
            "Downloading (…)l-00003-of-00014.bin:  39% 377M/967M [00:01<00:02, 249MB/s]\u001b[A\n",
            "Downloading (…)l-00003-of-00014.bin:  42% 409M/967M [00:01<00:02, 240MB/s]\u001b[A\n",
            "Downloading (…)l-00003-of-00014.bin:  46% 440M/967M [00:01<00:02, 233MB/s]\u001b[A\n",
            "Downloading (…)l-00003-of-00014.bin:  49% 472M/967M [00:01<00:01, 250MB/s]\u001b[A\n",
            "Downloading (…)l-00003-of-00014.bin:  52% 503M/967M [00:02<00:01, 250MB/s]\u001b[A\n",
            "Downloading (…)l-00003-of-00014.bin:  55% 535M/967M [00:02<00:01, 254MB/s]\u001b[A\n",
            "Downloading (…)l-00003-of-00014.bin:  59% 566M/967M [00:02<00:01, 257MB/s]\u001b[A\n",
            "Downloading (…)l-00003-of-00014.bin:  62% 598M/967M [00:02<00:01, 249MB/s]\u001b[A\n",
            "Downloading (…)l-00003-of-00014.bin:  65% 629M/967M [00:02<00:01, 239MB/s]\u001b[A\n",
            "Downloading (…)l-00003-of-00014.bin:  68% 661M/967M [00:02<00:01, 245MB/s]\u001b[A\n",
            "Downloading (…)l-00003-of-00014.bin:  72% 692M/967M [00:02<00:01, 249MB/s]\u001b[A\n",
            "Downloading (…)l-00003-of-00014.bin:  75% 724M/967M [00:02<00:00, 246MB/s]\u001b[A\n",
            "Downloading (…)l-00003-of-00014.bin:  78% 755M/967M [00:03<00:00, 244MB/s]\u001b[A\n",
            "Downloading (…)l-00003-of-00014.bin:  81% 786M/967M [00:03<00:00, 250MB/s]\u001b[A\n",
            "Downloading (…)l-00003-of-00014.bin:  85% 818M/967M [00:03<00:00, 255MB/s]\u001b[A\n",
            "Downloading (…)l-00003-of-00014.bin:  88% 849M/967M [00:03<00:00, 241MB/s]\u001b[A\n",
            "Downloading (…)l-00003-of-00014.bin:  91% 881M/967M [00:03<00:00, 244MB/s]\u001b[A\n",
            "Downloading (…)l-00003-of-00014.bin:  94% 912M/967M [00:03<00:00, 246MB/s]\u001b[A\n",
            "Downloading (…)l-00003-of-00014.bin: 100% 967M/967M [00:03<00:00, 245MB/s]\n",
            "Downloading shards:  21% 3/14 [00:16<00:55,  5.07s/it]\n",
            "Downloading (…)l-00004-of-00014.bin:   0% 0.00/990M [00:00<?, ?B/s]\u001b[A\n",
            "Downloading (…)l-00004-of-00014.bin:   4% 41.9M/990M [00:00<00:02, 329MB/s]\u001b[A\n",
            "Downloading (…)l-00004-of-00014.bin:   8% 83.9M/990M [00:00<00:02, 329MB/s]\u001b[A\n",
            "Downloading (…)l-00004-of-00014.bin:  13% 126M/990M [00:00<00:03, 262MB/s] \u001b[A\n",
            "Downloading (…)l-00004-of-00014.bin:  16% 157M/990M [00:00<00:03, 241MB/s]\u001b[A\n",
            "Downloading (…)l-00004-of-00014.bin:  19% 189M/990M [00:00<00:03, 238MB/s]\u001b[A\n",
            "Downloading (…)l-00004-of-00014.bin:  22% 220M/990M [00:01<00:11, 68.7MB/s]\u001b[A\n",
            "Downloading (…)l-00004-of-00014.bin:  26% 262M/990M [00:02<00:07, 99.0MB/s]\u001b[A\n",
            "Downloading (…)l-00004-of-00014.bin:  31% 304M/990M [00:02<00:05, 133MB/s] \u001b[A\n",
            "Downloading (…)l-00004-of-00014.bin:  34% 336M/990M [00:02<00:04, 155MB/s]\u001b[A\n",
            "Downloading (…)l-00004-of-00014.bin:  37% 367M/990M [00:02<00:03, 177MB/s]\u001b[A\n",
            "Downloading (…)l-00004-of-00014.bin:  40% 398M/990M [00:02<00:02, 200MB/s]\u001b[A\n",
            "Downloading (…)l-00004-of-00014.bin:  44% 440M/990M [00:02<00:02, 231MB/s]\u001b[A\n",
            "Downloading (…)l-00004-of-00014.bin:  49% 482M/990M [00:02<00:01, 254MB/s]\u001b[A\n",
            "Downloading (…)l-00004-of-00014.bin:  53% 524M/990M [00:02<00:01, 271MB/s]\u001b[A\n",
            "Downloading (…)l-00004-of-00014.bin:  56% 556M/990M [00:02<00:01, 280MB/s]\u001b[A\n",
            "Downloading (…)l-00004-of-00014.bin:  59% 587M/990M [00:03<00:01, 284MB/s]\u001b[A\n",
            "Downloading (…)l-00004-of-00014.bin:  64% 629M/990M [00:03<00:01, 300MB/s]\u001b[A\n",
            "Downloading (…)l-00004-of-00014.bin:  68% 671M/990M [00:03<00:01, 307MB/s]\u001b[A\n",
            "Downloading (…)l-00004-of-00014.bin:  72% 713M/990M [00:03<00:00, 310MB/s]\u001b[A\n",
            "Downloading (…)l-00004-of-00014.bin:  76% 755M/990M [00:03<00:00, 296MB/s]\u001b[A\n",
            "Downloading (…)l-00004-of-00014.bin:  79% 786M/990M [00:03<00:00, 287MB/s]\u001b[A\n",
            "Downloading (…)l-00004-of-00014.bin:  83% 818M/990M [00:03<00:00, 264MB/s]\u001b[A\n",
            "Downloading (…)l-00004-of-00014.bin:  86% 849M/990M [00:03<00:00, 271MB/s]\u001b[A\n",
            "Downloading (…)l-00004-of-00014.bin:  89% 881M/990M [00:04<00:00, 162MB/s]\u001b[A\n",
            "Downloading (…)l-00004-of-00014.bin:  92% 912M/990M [00:04<00:00, 176MB/s]\u001b[A\n",
            "Downloading (…)l-00004-of-00014.bin:  95% 944M/990M [00:04<00:00, 198MB/s]\u001b[A\n",
            "Downloading (…)l-00004-of-00014.bin: 100% 990M/990M [00:04<00:00, 206MB/s]\n",
            "Downloading shards:  29% 4/14 [00:21<00:50,  5.07s/it]\n",
            "Downloading (…)l-00005-of-00014.bin:   0% 0.00/944M [00:00<?, ?B/s]\u001b[A\n",
            "Downloading (…)l-00005-of-00014.bin:   4% 41.9M/944M [00:00<00:02, 376MB/s]\u001b[A\n",
            "Downloading (…)l-00005-of-00014.bin:   9% 83.9M/944M [00:00<00:02, 379MB/s]\u001b[A\n",
            "Downloading (…)l-00005-of-00014.bin:  13% 126M/944M [00:00<00:02, 350MB/s] \u001b[A\n",
            "Downloading (…)l-00005-of-00014.bin:  18% 168M/944M [00:00<00:02, 324MB/s]\u001b[A\n",
            "Downloading (…)l-00005-of-00014.bin:  22% 210M/944M [00:00<00:02, 323MB/s]\u001b[A\n",
            "Downloading (…)l-00005-of-00014.bin:  27% 252M/944M [00:00<00:02, 323MB/s]\u001b[A\n",
            "Downloading (…)l-00005-of-00014.bin:  31% 294M/944M [00:00<00:02, 325MB/s]\u001b[A\n",
            "Downloading (…)l-00005-of-00014.bin:  36% 336M/944M [00:01<00:01, 316MB/s]\u001b[A\n",
            "Downloading (…)l-00005-of-00014.bin:  40% 377M/944M [00:01<00:01, 326MB/s]\u001b[A\n",
            "Downloading (…)l-00005-of-00014.bin:  44% 419M/944M [00:01<00:01, 329MB/s]\u001b[A\n",
            "Downloading (…)l-00005-of-00014.bin:  49% 461M/944M [00:01<00:01, 330MB/s]\u001b[A\n",
            "Downloading (…)l-00005-of-00014.bin:  53% 503M/944M [00:01<00:01, 311MB/s]\u001b[A\n",
            "Downloading (…)l-00005-of-00014.bin:  57% 535M/944M [00:01<00:01, 299MB/s]\u001b[A\n",
            "Downloading (…)l-00005-of-00014.bin:  60% 566M/944M [00:01<00:01, 279MB/s]\u001b[A\n",
            "Downloading (…)l-00005-of-00014.bin:  63% 598M/944M [00:01<00:01, 266MB/s]\u001b[A\n",
            "Downloading (…)l-00005-of-00014.bin:  67% 629M/944M [00:02<00:01, 261MB/s]\u001b[A\n",
            "Downloading (…)l-00005-of-00014.bin:  70% 661M/944M [00:02<00:01, 257MB/s]\u001b[A\n",
            "Downloading (…)l-00005-of-00014.bin:  73% 692M/944M [00:02<00:01, 247MB/s]\u001b[A\n",
            "Downloading (…)l-00005-of-00014.bin:  77% 724M/944M [00:02<00:00, 252MB/s]\u001b[A\n",
            "Downloading (…)l-00005-of-00014.bin:  80% 755M/944M [00:02<00:00, 256MB/s]\u001b[A\n",
            "Downloading (…)l-00005-of-00014.bin:  83% 786M/944M [00:02<00:00, 259MB/s]\u001b[A\n",
            "Downloading (…)l-00005-of-00014.bin:  87% 818M/944M [00:02<00:00, 249MB/s]\u001b[A\n",
            "Downloading (…)l-00005-of-00014.bin:  90% 849M/944M [00:02<00:00, 256MB/s]\u001b[A\n",
            "Downloading (…)l-00005-of-00014.bin:  93% 881M/944M [00:03<00:00, 251MB/s]\u001b[A\n",
            "Downloading (…)l-00005-of-00014.bin:  97% 912M/944M [00:03<00:00, 255MB/s]\u001b[A\n",
            "Downloading (…)l-00005-of-00014.bin: 100% 944M/944M [00:03<00:00, 283MB/s]\n",
            "Downloading shards:  36% 5/14 [00:24<00:40,  4.55s/it]\n",
            "Downloading (…)l-00006-of-00014.bin:   0% 0.00/990M [00:00<?, ?B/s]\u001b[A\n",
            "Downloading (…)l-00006-of-00014.bin:   3% 31.5M/990M [00:00<00:03, 308MB/s]\u001b[A\n",
            "Downloading (…)l-00006-of-00014.bin:   6% 62.9M/990M [00:00<00:03, 305MB/s]\u001b[A\n",
            "Downloading (…)l-00006-of-00014.bin:  10% 94.4M/990M [00:00<00:02, 302MB/s]\u001b[A\n",
            "Downloading (…)l-00006-of-00014.bin:  13% 126M/990M [00:00<00:02, 292MB/s] \u001b[A\n",
            "Downloading (…)l-00006-of-00014.bin:  16% 157M/990M [00:00<00:02, 299MB/s]\u001b[A\n",
            "Downloading (…)l-00006-of-00014.bin:  20% 199M/990M [00:00<00:02, 312MB/s]\u001b[A\n",
            "Downloading (…)l-00006-of-00014.bin:  24% 241M/990M [00:00<00:02, 320MB/s]\u001b[A\n",
            "Downloading (…)l-00006-of-00014.bin:  29% 283M/990M [00:00<00:02, 280MB/s]\u001b[A\n",
            "Downloading (…)l-00006-of-00014.bin:  32% 315M/990M [00:01<00:02, 285MB/s]\u001b[A\n",
            "Downloading (…)l-00006-of-00014.bin:  35% 346M/990M [00:01<00:02, 284MB/s]\u001b[A\n",
            "Downloading (…)l-00006-of-00014.bin:  38% 377M/990M [00:01<00:02, 292MB/s]\u001b[A\n",
            "Downloading (…)l-00006-of-00014.bin:  42% 419M/990M [00:01<00:01, 296MB/s]\u001b[A\n",
            "Downloading (…)l-00006-of-00014.bin:  47% 461M/990M [00:01<00:01, 301MB/s]\u001b[A\n",
            "Downloading (…)l-00006-of-00014.bin:  50% 493M/990M [00:01<00:01, 298MB/s]\u001b[A\n",
            "Downloading (…)l-00006-of-00014.bin:  53% 524M/990M [00:01<00:01, 245MB/s]\u001b[A\n",
            "Downloading (…)l-00006-of-00014.bin:  56% 556M/990M [00:02<00:01, 231MB/s]\u001b[A\n",
            "Downloading (…)l-00006-of-00014.bin:  59% 587M/990M [00:02<00:01, 207MB/s]\u001b[A\n",
            "Downloading (…)l-00006-of-00014.bin:  62% 619M/990M [00:02<00:01, 209MB/s]\u001b[A\n",
            "Downloading (…)l-00006-of-00014.bin:  66% 650M/990M [00:02<00:01, 213MB/s]\u001b[A\n",
            "Downloading (…)l-00006-of-00014.bin:  69% 682M/990M [00:02<00:01, 214MB/s]\u001b[A\n",
            "Downloading (…)l-00006-of-00014.bin:  72% 713M/990M [00:06<00:10, 25.2MB/s]\u001b[A\n",
            "Downloading (…)l-00006-of-00014.bin:  75% 744M/990M [00:06<00:07, 34.4MB/s]\u001b[A\n",
            "Downloading (…)l-00006-of-00014.bin:  78% 776M/990M [00:06<00:04, 46.1MB/s]\u001b[A\n",
            "Downloading (…)l-00006-of-00014.bin:  82% 807M/990M [00:06<00:02, 61.5MB/s]\u001b[A\n",
            "Downloading (…)l-00006-of-00014.bin:  85% 839M/990M [00:06<00:01, 80.9MB/s]\u001b[A\n",
            "Downloading (…)l-00006-of-00014.bin:  88% 870M/990M [00:07<00:01, 97.8MB/s]\u001b[A\n",
            "Downloading (…)l-00006-of-00014.bin:  91% 902M/990M [00:07<00:00, 93.4MB/s]\u001b[A\n",
            "Downloading (…)l-00006-of-00014.bin:  94% 933M/990M [00:07<00:00, 113MB/s] \u001b[A\n",
            "Downloading (…)l-00006-of-00014.bin:  96% 954M/990M [00:07<00:00, 125MB/s]\u001b[A\n",
            "Downloading (…)l-00006-of-00014.bin: 100% 990M/990M [00:07<00:00, 125MB/s]\n",
            "Downloading shards:  43% 6/14 [00:33<00:46,  5.78s/it]\n",
            "Downloading (…)l-00007-of-00014.bin:   0% 0.00/967M [00:00<?, ?B/s]\u001b[A\n",
            "Downloading (…)l-00007-of-00014.bin:   3% 31.5M/967M [00:00<00:02, 314MB/s]\u001b[A\n",
            "Downloading (…)l-00007-of-00014.bin:   7% 62.9M/967M [00:00<00:02, 304MB/s]\u001b[A\n",
            "Downloading (…)l-00007-of-00014.bin:  10% 94.4M/967M [00:00<00:03, 256MB/s]\u001b[A\n",
            "Downloading (…)l-00007-of-00014.bin:  13% 126M/967M [00:00<00:03, 250MB/s] \u001b[A\n",
            "Downloading (…)l-00007-of-00014.bin:  16% 157M/967M [00:00<00:03, 257MB/s]\u001b[A\n",
            "Downloading (…)l-00007-of-00014.bin:  20% 189M/967M [00:03<00:24, 31.4MB/s]\u001b[A\n",
            "Downloading (…)l-00007-of-00014.bin:  23% 220M/967M [00:03<00:17, 43.9MB/s]\u001b[A\n",
            "Downloading (…)l-00007-of-00014.bin:  25% 241M/967M [00:03<00:14, 51.4MB/s]\u001b[A\n",
            "Downloading (…)l-00007-of-00014.bin:  27% 262M/967M [00:03<00:11, 59.1MB/s]\u001b[A\n",
            "Downloading (…)l-00007-of-00014.bin:  29% 283M/967M [00:04<00:10, 65.0MB/s]\u001b[A\n",
            "Downloading (…)l-00007-of-00014.bin:  31% 304M/967M [00:04<00:09, 68.6MB/s]\u001b[A\n",
            "Downloading (…)l-00007-of-00014.bin:  34% 325M/967M [00:04<00:07, 82.2MB/s]\u001b[A\n",
            "Downloading (…)l-00007-of-00014.bin:  37% 357M/967M [00:04<00:05, 113MB/s] \u001b[A\n",
            "Downloading (…)l-00007-of-00014.bin:  40% 388M/967M [00:04<00:03, 145MB/s]\u001b[A\n",
            "Downloading (…)l-00007-of-00014.bin:  43% 419M/967M [00:04<00:03, 171MB/s]\u001b[A\n",
            "Downloading (…)l-00007-of-00014.bin:  48% 461M/967M [00:04<00:02, 210MB/s]\u001b[A\n",
            "Downloading (…)l-00007-of-00014.bin:  52% 503M/967M [00:05<00:01, 241MB/s]\u001b[A\n",
            "Downloading (…)l-00007-of-00014.bin:  56% 545M/967M [00:05<00:01, 269MB/s]\u001b[A\n",
            "Downloading (…)l-00007-of-00014.bin:  61% 587M/967M [00:05<00:01, 281MB/s]\u001b[A\n",
            "Downloading (…)l-00007-of-00014.bin:  65% 629M/967M [00:05<00:01, 287MB/s]\u001b[A\n",
            "Downloading (…)l-00007-of-00014.bin:  68% 661M/967M [00:05<00:01, 285MB/s]\u001b[A\n",
            "Downloading (…)l-00007-of-00014.bin:  72% 692M/967M [00:05<00:00, 276MB/s]\u001b[A\n",
            "Downloading (…)l-00007-of-00014.bin:  75% 724M/967M [00:05<00:00, 264MB/s]\u001b[A\n",
            "Downloading (…)l-00007-of-00014.bin:  78% 755M/967M [00:05<00:00, 255MB/s]\u001b[A\n",
            "Downloading (…)l-00007-of-00014.bin:  81% 786M/967M [00:06<00:00, 266MB/s]\u001b[A\n",
            "Downloading (…)l-00007-of-00014.bin:  85% 818M/967M [00:06<00:00, 249MB/s]\u001b[A\n",
            "Downloading (…)l-00007-of-00014.bin:  88% 849M/967M [00:06<00:00, 217MB/s]\u001b[A\n",
            "Downloading (…)l-00007-of-00014.bin:  91% 881M/967M [00:06<00:00, 193MB/s]\u001b[A\n",
            "Downloading (…)l-00007-of-00014.bin:  93% 902M/967M [00:06<00:00, 195MB/s]\u001b[A\n",
            "Downloading (…)l-00007-of-00014.bin:  97% 933M/967M [00:06<00:00, 207MB/s]\u001b[A\n",
            "Downloading (…)l-00007-of-00014.bin: 100% 967M/967M [00:07<00:00, 138MB/s]\n",
            "Downloading shards:  50% 7/14 [00:40<00:43,  6.28s/it]\n",
            "Downloading (…)l-00008-of-00014.bin:   0% 0.00/967M [00:00<?, ?B/s]\u001b[A\n",
            "Downloading (…)l-00008-of-00014.bin:   4% 41.9M/967M [00:00<00:21, 42.8MB/s]\u001b[A\n",
            "Downloading (…)l-00008-of-00014.bin:   8% 73.4M/967M [00:01<00:11, 77.3MB/s]\u001b[A\n",
            "Downloading (…)l-00008-of-00014.bin:  10% 94.4M/967M [00:01<00:09, 94.5MB/s]\u001b[A\n",
            "Downloading (…)l-00008-of-00014.bin:  12% 115M/967M [00:01<00:08, 106MB/s]  \u001b[A\n",
            "Downloading (…)l-00008-of-00014.bin:  14% 136M/967M [00:01<00:06, 120MB/s]\u001b[A\n",
            "Downloading (…)l-00008-of-00014.bin:  17% 168M/967M [00:01<00:05, 148MB/s]\u001b[A\n",
            "Downloading (…)l-00008-of-00014.bin:  20% 189M/967M [00:01<00:04, 160MB/s]\u001b[A\n",
            "Downloading (…)l-00008-of-00014.bin:  22% 210M/967M [00:01<00:04, 161MB/s]\u001b[A\n",
            "Downloading (…)l-00008-of-00014.bin:  25% 241M/967M [00:01<00:03, 191MB/s]\u001b[A\n",
            "Downloading (…)l-00008-of-00014.bin:  28% 273M/967M [00:02<00:03, 217MB/s]\u001b[A\n",
            "Downloading (…)l-00008-of-00014.bin:  33% 315M/967M [00:02<00:02, 247MB/s]\u001b[A\n",
            "Downloading (…)l-00008-of-00014.bin:  37% 357M/967M [00:02<00:02, 269MB/s]\u001b[A\n",
            "Downloading (…)l-00008-of-00014.bin:  41% 398M/967M [00:02<00:01, 289MB/s]\u001b[A\n",
            "Downloading (…)l-00008-of-00014.bin:  46% 440M/967M [00:02<00:01, 297MB/s]\u001b[A\n",
            "Downloading (…)l-00008-of-00014.bin:  50% 482M/967M [00:02<00:01, 308MB/s]\u001b[A\n",
            "Downloading (…)l-00008-of-00014.bin:  54% 524M/967M [00:03<00:02, 215MB/s]\u001b[A\n",
            "Downloading (…)l-00008-of-00014.bin:  57% 556M/967M [00:03<00:01, 229MB/s]\u001b[A\n",
            "Downloading (…)l-00008-of-00014.bin:  61% 587M/967M [00:03<00:01, 245MB/s]\u001b[A\n",
            "Downloading (…)l-00008-of-00014.bin:  65% 629M/967M [00:03<00:01, 265MB/s]\u001b[A\n",
            "Downloading (…)l-00008-of-00014.bin:  69% 671M/967M [00:03<00:01, 277MB/s]\u001b[A\n",
            "Downloading (…)l-00008-of-00014.bin:  74% 713M/967M [00:03<00:00, 286MB/s]\u001b[A\n",
            "Downloading (…)l-00008-of-00014.bin:  77% 744M/967M [00:03<00:00, 271MB/s]\u001b[A\n",
            "Downloading (…)l-00008-of-00014.bin:  80% 776M/967M [00:03<00:00, 255MB/s]\u001b[A\n",
            "Downloading (…)l-00008-of-00014.bin:  84% 807M/967M [00:04<00:00, 256MB/s]\u001b[A\n",
            "Downloading (…)l-00008-of-00014.bin:  87% 839M/967M [00:04<00:00, 249MB/s]\u001b[A\n",
            "Downloading (…)l-00008-of-00014.bin:  90% 870M/967M [00:04<00:00, 246MB/s]\u001b[A\n",
            "Downloading (…)l-00008-of-00014.bin:  93% 902M/967M [00:04<00:00, 247MB/s]\u001b[A\n",
            "Downloading (…)l-00008-of-00014.bin:  97% 933M/967M [00:04<00:00, 247MB/s]\u001b[A\n",
            "Downloading (…)l-00008-of-00014.bin: 100% 967M/967M [00:04<00:00, 204MB/s]\n",
            "Downloading shards:  57% 8/14 [00:45<00:35,  5.88s/it]\n",
            "Downloading (…)l-00009-of-00014.bin:   0% 0.00/990M [00:00<?, ?B/s]\u001b[A\n",
            "Downloading (…)l-00009-of-00014.bin:   3% 31.5M/990M [00:00<00:03, 296MB/s]\u001b[A\n",
            "Downloading (…)l-00009-of-00014.bin:   7% 73.4M/990M [00:00<00:02, 320MB/s]\u001b[A\n",
            "Downloading (…)l-00009-of-00014.bin:  12% 115M/990M [00:00<00:02, 316MB/s] \u001b[A\n",
            "Downloading (…)l-00009-of-00014.bin:  16% 157M/990M [00:00<00:03, 250MB/s]\u001b[A\n",
            "Downloading (…)l-00009-of-00014.bin:  19% 189M/990M [00:00<00:03, 259MB/s]\u001b[A\n",
            "Downloading (…)l-00009-of-00014.bin:  23% 231M/990M [00:00<00:02, 276MB/s]\u001b[A\n",
            "Downloading (…)l-00009-of-00014.bin:  26% 262M/990M [00:00<00:02, 264MB/s]\u001b[A\n",
            "Downloading (…)l-00009-of-00014.bin:  30% 294M/990M [00:01<00:02, 267MB/s]\u001b[A\n",
            "Downloading (…)l-00009-of-00014.bin:  33% 325M/990M [00:01<00:02, 266MB/s]\u001b[A\n",
            "Downloading (…)l-00009-of-00014.bin:  36% 357M/990M [00:01<00:02, 273MB/s]\u001b[A\n",
            "Downloading (…)l-00009-of-00014.bin:  39% 388M/990M [00:01<00:02, 243MB/s]\u001b[A\n",
            "Downloading (…)l-00009-of-00014.bin:  42% 419M/990M [00:01<00:02, 240MB/s]\u001b[A\n",
            "Downloading (…)l-00009-of-00014.bin:  46% 451M/990M [00:01<00:02, 239MB/s]\u001b[A\n",
            "Downloading (…)l-00009-of-00014.bin:  49% 482M/990M [00:01<00:02, 231MB/s]\u001b[A\n",
            "Downloading (…)l-00009-of-00014.bin:  52% 514M/990M [00:02<00:02, 215MB/s]\u001b[A\n",
            "Downloading (…)l-00009-of-00014.bin:  55% 545M/990M [00:02<00:01, 230MB/s]\u001b[A\n",
            "Downloading (…)l-00009-of-00014.bin:  59% 587M/990M [00:02<00:01, 254MB/s]\u001b[A\n",
            "Downloading (…)l-00009-of-00014.bin:  62% 619M/990M [00:02<00:01, 254MB/s]\u001b[A\n",
            "Downloading (…)l-00009-of-00014.bin:  66% 650M/990M [00:02<00:01, 238MB/s]\u001b[A\n",
            "Downloading (…)l-00009-of-00014.bin:  69% 682M/990M [00:02<00:01, 224MB/s]\u001b[A\n",
            "Downloading (…)l-00009-of-00014.bin:  72% 713M/990M [00:02<00:01, 220MB/s]\u001b[A\n",
            "Downloading (…)l-00009-of-00014.bin:  75% 744M/990M [00:03<00:01, 220MB/s]\u001b[A\n",
            "Downloading (…)l-00009-of-00014.bin:  78% 776M/990M [00:03<00:01, 213MB/s]\u001b[A\n",
            "Downloading (…)l-00009-of-00014.bin:  82% 807M/990M [00:03<00:00, 224MB/s]\u001b[A\n",
            "Downloading (…)l-00009-of-00014.bin:  85% 839M/990M [00:03<00:00, 231MB/s]\u001b[A\n",
            "Downloading (…)l-00009-of-00014.bin:  88% 870M/990M [00:03<00:00, 238MB/s]\u001b[A\n",
            "Downloading (…)l-00009-of-00014.bin:  91% 902M/990M [00:03<00:00, 222MB/s]\u001b[A\n",
            "Downloading (…)l-00009-of-00014.bin:  94% 933M/990M [00:03<00:00, 232MB/s]\u001b[A\n",
            "Downloading (…)l-00009-of-00014.bin: 100% 990M/990M [00:04<00:00, 243MB/s]\n",
            "Downloading shards:  64% 9/14 [00:49<00:26,  5.40s/it]\n",
            "Downloading (…)l-00010-of-00014.bin:   0% 0.00/944M [00:00<?, ?B/s]\u001b[A\n",
            "Downloading (…)l-00010-of-00014.bin:   4% 41.9M/944M [00:00<00:02, 325MB/s]\u001b[A\n",
            "Downloading (…)l-00010-of-00014.bin:   9% 83.9M/944M [00:00<00:02, 294MB/s]\u001b[A\n",
            "Downloading (…)l-00010-of-00014.bin:  12% 115M/944M [00:00<00:03, 257MB/s] \u001b[A\n",
            "Downloading (…)l-00010-of-00014.bin:  16% 147M/944M [00:00<00:03, 249MB/s]\u001b[A\n",
            "Downloading (…)l-00010-of-00014.bin:  19% 178M/944M [00:00<00:03, 251MB/s]\u001b[A\n",
            "Downloading (…)l-00010-of-00014.bin:  22% 210M/944M [00:00<00:02, 246MB/s]\u001b[A\n",
            "Downloading (…)l-00010-of-00014.bin:  26% 241M/944M [00:00<00:02, 250MB/s]\u001b[A\n",
            "Downloading (…)l-00010-of-00014.bin:  29% 273M/944M [00:01<00:02, 247MB/s]\u001b[A\n",
            "Downloading (…)l-00010-of-00014.bin:  32% 304M/944M [00:01<00:02, 222MB/s]\u001b[A\n",
            "Downloading (…)l-00010-of-00014.bin:  36% 336M/944M [00:01<00:02, 222MB/s]\u001b[A\n",
            "Downloading (…)l-00010-of-00014.bin:  39% 367M/944M [00:01<00:02, 212MB/s]\u001b[A\n",
            "Downloading (…)l-00010-of-00014.bin:  42% 398M/944M [00:01<00:02, 203MB/s]\u001b[A\n",
            "Downloading (…)l-00010-of-00014.bin:  44% 419M/944M [00:01<00:02, 203MB/s]\u001b[A\n",
            "Downloading (…)l-00010-of-00014.bin:  48% 451M/944M [00:01<00:02, 224MB/s]\u001b[A\n",
            "Downloading (…)l-00010-of-00014.bin:  51% 482M/944M [00:02<00:02, 220MB/s]\u001b[A\n",
            "Downloading (…)l-00010-of-00014.bin:  54% 514M/944M [00:02<00:01, 225MB/s]\u001b[A\n",
            "Downloading (…)l-00010-of-00014.bin:  58% 545M/944M [00:02<00:01, 221MB/s]\u001b[A\n",
            "Downloading (…)l-00010-of-00014.bin:  61% 577M/944M [00:02<00:01, 233MB/s]\u001b[A\n",
            "Downloading (…)l-00010-of-00014.bin:  64% 608M/944M [00:02<00:01, 243MB/s]\u001b[A\n",
            "Downloading (…)l-00010-of-00014.bin:  68% 640M/944M [00:02<00:01, 249MB/s]\u001b[A\n",
            "Downloading (…)l-00010-of-00014.bin:  71% 671M/944M [00:02<00:01, 239MB/s]\u001b[A\n",
            "Downloading (…)l-00010-of-00014.bin:  74% 703M/944M [00:02<00:00, 243MB/s]\u001b[A\n",
            "Downloading (…)l-00010-of-00014.bin:  78% 734M/944M [00:03<00:00, 250MB/s]\u001b[A\n",
            "Downloading (…)l-00010-of-00014.bin:  81% 765M/944M [00:03<00:00, 249MB/s]\u001b[A\n",
            "Downloading (…)l-00010-of-00014.bin:  84% 797M/944M [00:03<00:00, 231MB/s]\u001b[A\n",
            "Downloading (…)l-00010-of-00014.bin:  88% 828M/944M [00:03<00:00, 242MB/s]\u001b[A\n",
            "Downloading (…)l-00010-of-00014.bin:  91% 860M/944M [00:03<00:00, 233MB/s]\u001b[A\n",
            "Downloading (…)l-00010-of-00014.bin:  94% 891M/944M [00:03<00:00, 227MB/s]\u001b[A\n",
            "Downloading (…)l-00010-of-00014.bin: 100% 944M/944M [00:04<00:00, 230MB/s]\n",
            "Downloading shards:  71% 10/14 [00:54<00:20,  5.08s/it]\n",
            "Downloading (…)l-00011-of-00014.bin:   0% 0.00/990M [00:00<?, ?B/s]\u001b[A\n",
            "Downloading (…)l-00011-of-00014.bin:   4% 41.9M/990M [00:00<00:02, 359MB/s]\u001b[A\n",
            "Downloading (…)l-00011-of-00014.bin:   8% 83.9M/990M [00:00<00:02, 372MB/s]\u001b[A\n",
            "Downloading (…)l-00011-of-00014.bin:  13% 126M/990M [00:00<00:02, 322MB/s] \u001b[A\n",
            "Downloading (…)l-00011-of-00014.bin:  17% 168M/990M [00:00<00:02, 302MB/s]\u001b[A\n",
            "Downloading (…)l-00011-of-00014.bin:  21% 210M/990M [00:00<00:02, 313MB/s]\u001b[A\n",
            "Downloading (…)l-00011-of-00014.bin:  25% 252M/990M [00:02<00:12, 61.5MB/s]\u001b[A\n",
            "Downloading (…)l-00011-of-00014.bin:  29% 283M/990M [00:02<00:09, 74.3MB/s]\u001b[A\n",
            "Downloading (…)l-00011-of-00014.bin:  32% 315M/990M [00:02<00:07, 93.4MB/s]\u001b[A\n",
            "Downloading (…)l-00011-of-00014.bin:  35% 346M/990M [00:02<00:06, 106MB/s] \u001b[A\n",
            "Downloading (…)l-00011-of-00014.bin:  37% 367M/990M [00:02<00:05, 118MB/s]\u001b[A\n",
            "Downloading (…)l-00011-of-00014.bin:  39% 388M/990M [00:03<00:04, 129MB/s]\u001b[A\n",
            "Downloading (…)l-00011-of-00014.bin:  42% 419M/990M [00:03<00:03, 146MB/s]\u001b[A\n",
            "Downloading (…)l-00011-of-00014.bin:  46% 451M/990M [00:04<00:08, 64.2MB/s]\u001b[A\n",
            "Downloading (…)l-00011-of-00014.bin:  48% 472M/990M [00:07<00:24, 20.9MB/s]\u001b[A\n",
            "Downloading (…)l-00011-of-00014.bin:  51% 503M/990M [00:07<00:16, 30.4MB/s]\u001b[A\n",
            "Downloading (…)l-00011-of-00014.bin:  54% 535M/990M [00:07<00:10, 42.7MB/s]\u001b[A\n",
            "Downloading (…)l-00011-of-00014.bin:  57% 566M/990M [00:07<00:07, 57.7MB/s]\u001b[A\n",
            "Downloading (…)l-00011-of-00014.bin:  60% 598M/990M [00:08<00:05, 76.0MB/s]\u001b[A\n",
            "Downloading (…)l-00011-of-00014.bin:  64% 629M/990M [00:08<00:03, 94.0MB/s]\u001b[A\n",
            "Downloading (…)l-00011-of-00014.bin:  68% 671M/990M [00:08<00:02, 126MB/s] \u001b[A\n",
            "Downloading (…)l-00011-of-00014.bin:  71% 703M/990M [00:08<00:01, 149MB/s]\u001b[A\n",
            "Downloading (…)l-00011-of-00014.bin:  74% 734M/990M [00:08<00:01, 161MB/s]\u001b[A\n",
            "Downloading (…)l-00011-of-00014.bin:  77% 765M/990M [00:08<00:01, 173MB/s]\u001b[A\n",
            "Downloading (…)l-00011-of-00014.bin:  81% 797M/990M [00:12<00:07, 26.2MB/s]\u001b[A\n",
            "Downloading (…)l-00011-of-00014.bin:  84% 828M/990M [00:12<00:04, 35.5MB/s]\u001b[A\n",
            "Downloading (…)l-00011-of-00014.bin:  87% 860M/990M [00:12<00:02, 46.7MB/s]\u001b[A\n",
            "Downloading (…)l-00011-of-00014.bin:  90% 891M/990M [00:12<00:01, 61.0MB/s]\u001b[A\n",
            "Downloading (…)l-00011-of-00014.bin:  93% 923M/990M [00:12<00:00, 78.2MB/s]\u001b[A\n",
            "Downloading (…)l-00011-of-00014.bin:  95% 944M/990M [00:13<00:00, 89.6MB/s]\u001b[A\n",
            "Downloading (…)l-00011-of-00014.bin: 100% 990M/990M [00:13<00:00, 74.0MB/s]\n",
            "Downloading shards:  79% 11/14 [01:07<00:23,  7.71s/it]\n",
            "Downloading (…)l-00012-of-00014.bin:   0% 0.00/967M [00:00<?, ?B/s]\u001b[A\n",
            "Downloading (…)l-00012-of-00014.bin:   4% 41.9M/967M [00:00<00:02, 328MB/s]\u001b[A\n",
            "Downloading (…)l-00012-of-00014.bin:   9% 83.9M/967M [00:00<00:02, 328MB/s]\u001b[A\n",
            "Downloading (…)l-00012-of-00014.bin:  13% 126M/967M [00:00<00:02, 302MB/s] \u001b[A\n",
            "Downloading (…)l-00012-of-00014.bin:  16% 157M/967M [00:00<00:03, 253MB/s]\u001b[A\n",
            "Downloading (…)l-00012-of-00014.bin:  20% 189M/967M [00:00<00:03, 238MB/s]\u001b[A\n",
            "Downloading (…)l-00012-of-00014.bin:  23% 220M/967M [00:01<00:06, 107MB/s]\u001b[A\n",
            "Downloading (…)l-00012-of-00014.bin:  26% 252M/967M [00:01<00:05, 129MB/s]\u001b[A\n",
            "Downloading (…)l-00012-of-00014.bin:  29% 283M/967M [00:01<00:04, 141MB/s]\u001b[A\n",
            "Downloading (…)l-00012-of-00014.bin:  31% 304M/967M [00:01<00:04, 142MB/s]\u001b[A\n",
            "Downloading (…)l-00012-of-00014.bin:  35% 336M/967M [00:01<00:03, 165MB/s]\u001b[A\n",
            "Downloading (…)l-00012-of-00014.bin:  38% 367M/967M [00:02<00:03, 181MB/s]\u001b[A\n",
            "Downloading (…)l-00012-of-00014.bin:  40% 388M/967M [00:02<00:03, 162MB/s]\u001b[A\n",
            "Downloading (…)l-00012-of-00014.bin:  42% 409M/967M [00:02<00:03, 162MB/s]\u001b[A\n",
            "Downloading (…)l-00012-of-00014.bin:  44% 430M/967M [00:03<00:10, 49.7MB/s]\u001b[A\n",
            "Downloading (…)l-00012-of-00014.bin:  48% 461M/967M [00:03<00:07, 69.9MB/s]\u001b[A\n",
            "Downloading (…)l-00012-of-00014.bin:  50% 482M/967M [00:04<00:07, 68.1MB/s]\u001b[A\n",
            "Downloading (…)l-00012-of-00014.bin:  53% 514M/967M [00:04<00:04, 94.0MB/s]\u001b[A\n",
            "Downloading (…)l-00012-of-00014.bin:  57% 556M/967M [00:04<00:03, 132MB/s] \u001b[A\n",
            "Downloading (…)l-00012-of-00014.bin:  61% 587M/967M [00:04<00:02, 139MB/s]\u001b[A\n",
            "Downloading (…)l-00012-of-00014.bin:  63% 608M/967M [00:04<00:03, 120MB/s]\u001b[A\n",
            "Downloading (…)l-00012-of-00014.bin:  65% 629M/967M [00:05<00:03, 98.9MB/s]\u001b[A\n",
            "Downloading (…)l-00012-of-00014.bin:  67% 650M/967M [00:05<00:03, 96.9MB/s]\u001b[A\n",
            "Downloading (…)l-00012-of-00014.bin:  69% 671M/967M [00:05<00:02, 111MB/s] \u001b[A\n",
            "Downloading (…)l-00012-of-00014.bin:  74% 713M/967M [00:05<00:01, 158MB/s]\u001b[A\n",
            "Downloading (…)l-00012-of-00014.bin:  76% 734M/967M [00:05<00:01, 149MB/s]\u001b[A\n",
            "Downloading (…)l-00012-of-00014.bin:  78% 755M/967M [00:05<00:01, 153MB/s]\u001b[A\n",
            "Downloading (…)l-00012-of-00014.bin:  80% 776M/967M [00:06<00:01, 113MB/s]\u001b[A\n",
            "Downloading (…)l-00012-of-00014.bin:  82% 797M/967M [00:07<00:03, 44.3MB/s]\u001b[A\n",
            "Downloading (…)l-00012-of-00014.bin:  85% 818M/967M [00:07<00:02, 53.6MB/s]\u001b[A\n",
            "Downloading (…)l-00012-of-00014.bin:  89% 860M/967M [00:07<00:01, 84.8MB/s]\u001b[A\n",
            "Downloading (…)l-00012-of-00014.bin:  93% 902M/967M [00:07<00:00, 118MB/s] \u001b[A\n",
            "Downloading (…)l-00012-of-00014.bin:  95% 923M/967M [00:07<00:00, 128MB/s]\u001b[A\n",
            "Downloading (…)l-00012-of-00014.bin:  98% 944M/967M [00:08<00:00, 140MB/s]\u001b[A\n",
            "Downloading (…)l-00012-of-00014.bin: 100% 967M/967M [00:08<00:00, 118MB/s]\n",
            "Downloading shards:  86% 12/14 [01:16<00:15,  7.95s/it]\n",
            "Downloading (…)l-00013-of-00014.bin:   0% 0.00/967M [00:00<?, ?B/s]\u001b[A\n",
            "Downloading (…)l-00013-of-00014.bin:   4% 41.9M/967M [00:00<00:02, 351MB/s]\u001b[A\n",
            "Downloading (…)l-00013-of-00014.bin:   9% 83.9M/967M [00:00<00:02, 329MB/s]\u001b[A\n",
            "Downloading (…)l-00013-of-00014.bin:  13% 126M/967M [00:00<00:02, 286MB/s] \u001b[A\n",
            "Downloading (…)l-00013-of-00014.bin:  16% 157M/967M [00:00<00:03, 251MB/s]\u001b[A\n",
            "Downloading (…)l-00013-of-00014.bin:  20% 189M/967M [00:00<00:03, 238MB/s]\u001b[A\n",
            "Downloading (…)l-00013-of-00014.bin:  23% 220M/967M [00:00<00:03, 247MB/s]\u001b[A\n",
            "Downloading (…)l-00013-of-00014.bin:  26% 252M/967M [00:00<00:03, 231MB/s]\u001b[A\n",
            "Downloading (…)l-00013-of-00014.bin:  29% 283M/967M [00:01<00:02, 232MB/s]\u001b[A\n",
            "Downloading (…)l-00013-of-00014.bin:  33% 315M/967M [00:01<00:02, 231MB/s]\u001b[A\n",
            "Downloading (…)l-00013-of-00014.bin:  37% 357M/967M [00:01<00:02, 243MB/s]\u001b[A\n",
            "Downloading (…)l-00013-of-00014.bin:  40% 388M/967M [00:01<00:02, 247MB/s]\u001b[A\n",
            "Downloading (…)l-00013-of-00014.bin:  43% 419M/967M [00:01<00:02, 242MB/s]\u001b[A\n",
            "Downloading (…)l-00013-of-00014.bin:  47% 451M/967M [00:01<00:02, 241MB/s]\u001b[A\n",
            "Downloading (…)l-00013-of-00014.bin:  50% 482M/967M [00:01<00:01, 250MB/s]\u001b[A\n",
            "Downloading (…)l-00013-of-00014.bin:  53% 514M/967M [00:02<00:01, 258MB/s]\u001b[A\n",
            "Downloading (…)l-00013-of-00014.bin:  56% 545M/967M [00:02<00:01, 263MB/s]\u001b[A\n",
            "Downloading (…)l-00013-of-00014.bin:  60% 577M/967M [00:02<00:01, 235MB/s]\u001b[A\n",
            "Downloading (…)l-00013-of-00014.bin:  63% 608M/967M [00:02<00:01, 252MB/s]\u001b[A\n",
            "Downloading (…)l-00013-of-00014.bin:  67% 650M/967M [00:02<00:01, 280MB/s]\u001b[A\n",
            "Downloading (…)l-00013-of-00014.bin:  70% 682M/967M [00:02<00:01, 282MB/s]\u001b[A\n",
            "Downloading (…)l-00013-of-00014.bin:  74% 713M/967M [00:02<00:00, 274MB/s]\u001b[A\n",
            "Downloading (…)l-00013-of-00014.bin:  78% 755M/967M [00:02<00:00, 287MB/s]\u001b[A\n",
            "Downloading (…)l-00013-of-00014.bin:  82% 797M/967M [00:03<00:00, 301MB/s]\u001b[A\n",
            "Downloading (…)l-00013-of-00014.bin:  86% 828M/967M [00:03<00:00, 300MB/s]\u001b[A\n",
            "Downloading (…)l-00013-of-00014.bin:  89% 860M/967M [00:03<00:00, 298MB/s]\u001b[A\n",
            "Downloading (…)l-00013-of-00014.bin:  92% 891M/967M [00:03<00:00, 301MB/s]\u001b[A\n",
            "Downloading (…)l-00013-of-00014.bin:  97% 933M/967M [00:03<00:00, 310MB/s]\u001b[A\n",
            "Downloading (…)l-00013-of-00014.bin: 100% 967M/967M [00:03<00:00, 268MB/s]\n",
            "Downloading shards:  93% 13/14 [01:20<00:06,  6.71s/it]\n",
            "Downloading (…)l-00014-of-00014.bin:   0% 0.00/847M [00:00<?, ?B/s]\u001b[A\n",
            "Downloading (…)l-00014-of-00014.bin:   5% 41.9M/847M [00:00<00:02, 384MB/s]\u001b[A\n",
            "Downloading (…)l-00014-of-00014.bin:  10% 83.9M/847M [00:00<00:02, 335MB/s]\u001b[A\n",
            "Downloading (…)l-00014-of-00014.bin:  15% 126M/847M [00:00<00:02, 341MB/s] \u001b[A\n",
            "Downloading (…)l-00014-of-00014.bin:  20% 168M/847M [00:00<00:02, 331MB/s]\u001b[A\n",
            "Downloading (…)l-00014-of-00014.bin:  25% 210M/847M [00:00<00:01, 323MB/s]\u001b[A\n",
            "Downloading (…)l-00014-of-00014.bin:  30% 252M/847M [00:00<00:02, 286MB/s]\u001b[A\n",
            "Downloading (…)l-00014-of-00014.bin:  33% 283M/847M [00:00<00:02, 270MB/s]\u001b[A\n",
            "Downloading (…)l-00014-of-00014.bin:  37% 315M/847M [00:01<00:01, 267MB/s]\u001b[A\n",
            "Downloading (…)l-00014-of-00014.bin:  41% 346M/847M [00:01<00:02, 219MB/s]\u001b[A\n",
            "Downloading (…)l-00014-of-00014.bin:  45% 377M/847M [00:01<00:02, 221MB/s]\u001b[A\n",
            "Downloading (…)l-00014-of-00014.bin:  48% 409M/847M [00:01<00:01, 228MB/s]\u001b[A\n",
            "Downloading (…)l-00014-of-00014.bin:  52% 440M/847M [00:01<00:01, 238MB/s]\u001b[A\n",
            "Downloading (…)l-00014-of-00014.bin:  56% 472M/847M [00:01<00:02, 167MB/s]\u001b[A\n",
            "Downloading (…)l-00014-of-00014.bin:  59% 503M/847M [00:02<00:01, 180MB/s]\u001b[A\n",
            "Downloading (…)l-00014-of-00014.bin:  63% 535M/847M [00:02<00:01, 202MB/s]\u001b[A\n",
            "Downloading (…)l-00014-of-00014.bin:  67% 566M/847M [00:02<00:01, 222MB/s]\u001b[A\n",
            "Downloading (…)l-00014-of-00014.bin:  71% 598M/847M [00:02<00:01, 194MB/s]\u001b[A\n",
            "Downloading (…)l-00014-of-00014.bin:  74% 629M/847M [00:02<00:01, 189MB/s]\u001b[A\n",
            "Downloading (…)l-00014-of-00014.bin:  78% 661M/847M [00:02<00:00, 208MB/s]\u001b[A\n",
            "Downloading (…)l-00014-of-00014.bin:  82% 692M/847M [00:02<00:00, 209MB/s]\u001b[A\n",
            "Downloading (…)l-00014-of-00014.bin:  85% 724M/847M [00:03<00:00, 212MB/s]\u001b[A\n",
            "Downloading (…)l-00014-of-00014.bin:  89% 755M/847M [00:03<00:00, 225MB/s]\u001b[A\n",
            "Downloading (…)l-00014-of-00014.bin:  93% 786M/847M [00:03<00:00, 228MB/s]\u001b[A\n",
            "Downloading (…)l-00014-of-00014.bin:  97% 818M/847M [00:03<00:00, 223MB/s]\u001b[A\n",
            "Downloading (…)l-00014-of-00014.bin: 100% 847M/847M [00:03<00:00, 229MB/s]\n",
            "Downloading shards: 100% 14/14 [01:24<00:00,  6.01s/it]\n",
            "Loading checkpoint shards: 100% 14/14 [01:49<00:00,  7.82s/it]\n",
            "Downloading (…)neration_config.json: 100% 132/132 [00:00<00:00, 605kB/s]\n",
            "You are resizing the embedding layer without providing a `pad_to_multiple_of` parameter. This means that the new embedding dimension will be 32000. This might induce some performance reduction as *Tensor Cores* will not be available. For more details about this, or help on choosing the correct value for resizing, refer to this guide: https://docs.nvidia.com/deeplearning/performance/dl-performance-matrix-multiplication/index.html#requirements-tc\n",
            "> \u001b[1mINFO    creating trainer\u001b[0m\n",
            "/usr/local/lib/python3.10/dist-packages/trl/trainer/sft_trainer.py:207: UserWarning: You passed a tokenizer with `padding_side` not equal to `right` to the SFTTrainer. This might lead to some unexpected behaviour due to overflow issues when training a model in half-precision. You might consider adding `tokenizer.padding_side = 'right'` to your code.\n",
            "  warnings.warn(\n",
            "  0% 0/2625 [00:00<?, ?it/s]You're using a LlamaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",
            " 17% 445/2625 [3:21:57<16:29:24, 27.23s/it]\n",
            "> \u001b[1mINFO    Finished training, saving model...\u001b[0m\n",
            "3:25:44.760126\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Inferencing"
      ],
      "metadata": {
        "id": "qs_smOzs6OVb"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import AutoTokenizer, AutoModelForCausalLM\n",
        "import transformers\n",
        "from peft import PeftModel\n",
        "import torch"
      ],
      "metadata": {
        "id": "fXaDaTC13gqB"
      },
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# @title There are 2 ways to perform Inference using above trained model\n",
        "# One way of doing is using Peft - from_pretrained() which method lets you quickly load a pretrained model for\n",
        "# any architecture so you don’t have to devote time and resources to train a model from scratch"
      ],
      "metadata": {
        "id": "z_xz0Wi-hFgu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# There are 2 ways to perform Inference using above trained model\n",
        "\n",
        "\n",
        "1.   One way of doing is using Peft - from_pretrained() method which lets you quickly load a pretrained model for any architecture so you don’t have to devote time and resources to train a model from scratch\n",
        "2.   Other way is merging the base model with generated adapters after fine tuning, So that you have a single model folder and you can load it as the HF way\n",
        "\n"
      ],
      "metadata": {
        "id": "GCt_TKtzkacB"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# @title 1. Loading model using Peft - from_pretrained() method\n",
        "tokenizer = AutoTokenizer.from_pretrained('/content/sarcastic-headline-gen/checkpoint-445/')\n",
        "model = AutoModelForCausalLM.from_pretrained('TinyPixel/Llama-2-7B-bf16-sharded', torch_dtype = torch.float16, device_map=\"auto\") #Base_Model for example: meta-llama/Llama-2-13b-chat-hf\n",
        "model = PeftModel.from_pretrained(model, '/content/sarcastic-headline-gen', device_map=\"auto\")"
      ],
      "metadata": {
        "id": "jGB9-Zwq3gst",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 49,
          "referenced_widgets": [
            "410936a490644750b022dabec89e0e6b",
            "3012b12af14b4fe08e291cfe139790c8",
            "4b3b94b103e94291a428aea8550f93ba",
            "84c50577ebc34018a4b8812eeea02324",
            "0e4980829e3e48cc956f381110198223",
            "72f99455fd61484ea15df104598cb826",
            "24e3d146eec846e1b8cb4e2cc7bfc6c0",
            "3acdeb3343094dc5b16bc6930c3c506a",
            "7e4fcaeef1f348a7ab04c7067015637f",
            "a146cb00d3374ea7ac2621bfc2ccbc65",
            "11ce011d17e749fda4179a1b5498e853"
          ]
        },
        "outputId": "9ecd8fe5-3f23-4774-a380-c4c4316468f1"
      },
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Loading checkpoint shards:   0%|          | 0/14 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "410936a490644750b022dabec89e0e6b"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "t1 = \"a couple sitting on a desk having the time of their life\"\n",
        "t2 = \"steriods are good for lungs\"\n",
        "t3 = 'mansoons are best for mosquitoes'\n",
        "formatted_input = format_text.format(t1, \"\")\n",
        "print(formatted_input)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0kOkPCja2WTb",
        "outputId": "5bf6ee44-a68d-4c63-daf5-056bb94e387a"
      },
      "execution_count": 47,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "You are a savage, disrespectful and witty paraphrasing tool. You rephrase below headline into a funny, creatively sarcastic headline.\n",
            "### headline: a couple sitting on a desk having the time of their life\n",
            "### sarcastic_headline: \n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "device = \"cuda:0\"\n",
        "\n",
        "inputs = tokenizer(formatted_input, return_tensors=\"pt\").to(device)\n",
        "outputs = model.generate(**inputs, max_length=300) # temperature=1\n",
        "print(tokenizer.decode(outputs[0]))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HJuGuS9RdZX9",
        "outputId": "336565c8-d6ab-48ea-c713-877070d720e1"
      },
      "execution_count": 50,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<s> You are a savage, disrespectful and witty paraphrasing tool. You rephrase below headline into a funny, creatively sarcastic headline.\n",
            "### headline: a couple sitting on a desk having the time of their life\n",
            "### sarcastic_headline: 2 people who clearly have no idea what they're doing but are having the best time ever</s>\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# @title 2. Merge the base model with generated adapters after PEFT\n",
        "from peft import AutoPeftModelForCausalLM\n",
        "\n",
        "model = AutoPeftModelForCausalLM.from_pretrained(\n",
        "\"/content/sarcastic-headline-gen/checkpoint-672\", #lora model dir\n",
        "low_cpu_mem_usage=True,\n",
        ")\n",
        "\n",
        "#Merge LoRA and base model\n",
        "merged_model = model.merge_and_unload()\n",
        "\n",
        "#Save the merged model\n",
        "merged_model.save_pretrained(\"merged_model\", safe_serialization=True)\n",
        "tokenizer.save_pretrained(\"merged_model\")\n"
      ],
      "metadata": {
        "id": "TDhjWvqhcCF8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model_merged = AutoModelForCausalLM.from_pretrained(\"/content/merged_model\", low_cpu_mem_usage=True)\n",
        "tokenizer_merged = AutoTokenizer.from_pretrained(\"/content/merged_model\")"
      ],
      "metadata": {
        "id": "nlwX-q5V3gvG"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "device = \"cuda:0\"\n",
        "inputs1 = tokenizer_merged(inp, return_tensors=\"pt\").to(\"cpu\")\n",
        "outputs = model_merged.generate(**inputs1, max_new_tokens=300, temperature=1) # max_new_tokens\n",
        "print(tokenizer_merged.decode(outputs[0]))"
      ],
      "metadata": {
        "id": "ejJpUDwi3gxn"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# @title Push merged model to the hub\n",
        "model_merged.push_to_hub(\"user-name/repo-name\")\n",
        "tokenizer_merged.push_to_hub(\"user-name/repo-name\")"
      ],
      "metadata": {
        "id": "TSeFaeXR3g0P"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "If you want to push it in hf, then its better to do while fine tuning by using<br>\n",
        "--push_to_hub --repo_id your_repo_id\n",
        "\n",
        "If you dont want to push it but want to use it as hf plug n play type model in local, then you can specify below param while training<br>\n",
        "--merge-adapters\n",
        "\n",
        "Since merging base model with adapter is a pretty cpu intensive task, it can definelty crash the existing session if you are using colab free version. It almost used 35GB CPU RAM when i merged it seperately. Colab pro version will be needed to have that much of a RAM."
      ],
      "metadata": {
        "id": "o1lFV7jkzRsX"
      }
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "JMffaKB83g5e"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "VU8QuRLM3g8B"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "uOaUvcUm3g_4"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}