train_ner.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Hz9wfsaS5iMS"
      },
      "source": [
        "#  African NER [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/masakhane-io/masakhane-ner/blob/main/code/train_ner.ipynb)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iFtw4FZP5iMU"
      },
      "source": [
        "This notebook is designed to be able to train a pre-trained model on an African NER dataset.\n",
        "\n",
        "##### Sections:\n",
        "\n",
        "There are four sections in this notebook:\n",
        "\n",
        "1. Installations: this is where we do installation for relevant dependencies\n",
        "2. Imports: here, we perform imports for all the dependencies needed\n",
        "3. Utility Classes and Functions: here, we define utility classes and functions that will help us train\n",
        "4. Training: Here, the actual training process is done\n",
        "\n",
        "### NB: Please run the entire cells in the notebooks as they are. The only section that can be modified is the training section. The parts of the code that can be modified are clearly explained in the the training section.\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RGZEuEqv5iMU"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "z_K11J-Z5iMU"
      },
      "source": [
        "### 1. Installations"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yLeOCgCW5iMU",
        "outputId": "2af8481f-c726-48ad-ac94-df850e1dfaa9"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting transformers\n",
            "  Downloading transformers-4.33.1-py3-none-any.whl (7.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.6/7.6 MB\u001b[0m \u001b[31m30.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.12.3)\n",
            "Collecting huggingface-hub<1.0,>=0.15.1 (from transformers)\n",
            "  Downloading huggingface_hub-0.16.4-py3-none-any.whl (268 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m268.8/268.8 kB\u001b[0m \u001b[31m29.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (1.23.5)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (23.1)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (6.0.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2023.6.3)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.31.0)\n",
            "Collecting tokenizers!=0.11.3,<0.14,>=0.11.1 (from transformers)\n",
            "  Downloading tokenizers-0.13.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.8/7.8 MB\u001b[0m \u001b[31m89.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting safetensors>=0.3.1 (from transformers)\n",
            "  Downloading safetensors-0.3.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m55.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers) (4.66.1)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.15.1->transformers) (2023.6.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.15.1->transformers) (4.7.1)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.2.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.4)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2023.7.22)\n",
            "Installing collected packages: tokenizers, safetensors, huggingface-hub, transformers\n",
            "Successfully installed huggingface-hub-0.16.4 safetensors-0.3.3 tokenizers-0.13.3 transformers-4.33.1\n",
            "Collecting seqeval\n",
            "  Downloading seqeval-1.2.2.tar.gz (43 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m43.6/43.6 kB\u001b[0m \u001b[31m1.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: numpy>=1.14.0 in /usr/local/lib/python3.10/dist-packages (from seqeval) (1.23.5)\n",
            "Requirement already satisfied: scikit-learn>=0.21.3 in /usr/local/lib/python3.10/dist-packages (from seqeval) (1.2.2)\n",
            "Requirement already satisfied: scipy>=1.3.2 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.21.3->seqeval) (1.10.1)\n",
            "Requirement already satisfied: joblib>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.21.3->seqeval) (1.3.2)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.21.3->seqeval) (3.2.0)\n",
            "Building wheels for collected packages: seqeval\n",
            "  Building wheel for seqeval (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for seqeval: filename=seqeval-1.2.2-py3-none-any.whl size=16162 sha256=f6983e120d2491f31b7185d7789ee6caca2ae60e4330105bd414a12cf17e1db2\n",
            "  Stored in directory: /root/.cache/pip/wheels/1a/67/4a/ad4082dd7dfc30f2abfe4d80a2ed5926a506eb8a972b4767fa\n",
            "Successfully built seqeval\n",
            "Installing collected packages: seqeval\n",
            "Successfully installed seqeval-1.2.2\n",
            "Collecting ptvsd\n",
            "  Downloading ptvsd-4.3.2-py2.py3-none-any.whl (4.9 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m4.9/4.9 MB\u001b[0m \u001b[31m22.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: ptvsd\n",
            "Successfully installed ptvsd-4.3.2\n"
          ]
        }
      ],
      "source": [
        "!pip install transformers\n",
        "!pip install seqeval\n",
        "!pip install ptvsd"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "l5BJkE1i5iMV"
      },
      "source": [
        "### 2. Imports"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "cEb50dAP5iMV"
      },
      "outputs": [],
      "source": [
        "import argparse\n",
        "import glob\n",
        "import logging\n",
        "import os\n",
        "import random\n",
        "from collections import defaultdict, Counter\n",
        "\n",
        "import torch\n",
        "import numpy as np\n",
        "\n",
        "from scipy.sparse import save_npz, load_npz\n",
        "from seqeval.metrics import f1_score, precision_score, recall_score\n",
        "from torch import LongTensor\n",
        "from torch import nn, optim\n",
        "from torch.nn import CrossEntropyLoss\n",
        "from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence\n",
        "from torch.nn import functional as F\n",
        "from torch.utils.data import Dataset, DataLoader, RandomSampler, SequentialSampler, TensorDataset\n",
        "from torch.utils.data.distributed import DistributedSampler\n",
        "from tqdm import tqdm, trange\n",
        "\n",
        "from transformers import (\n",
        "    WEIGHTS_NAME,\n",
        "    AutoModelForSequenceClassification,\n",
        "    AutoModelForTokenClassification,\n",
        "    AutoConfig,\n",
        "    AutoTokenizer,\n",
        "    AutoModel,\n",
        "    AdamW,\n",
        "    BertConfig,\n",
        "    BertForTokenClassification,\n",
        "    BertTokenizer,\n",
        "    CamembertConfig,\n",
        "    CamembertForTokenClassification,\n",
        "    CamembertTokenizer,\n",
        "    DistilBertConfig,\n",
        "    DistilBertForTokenClassification,\n",
        "    DistilBertTokenizer,\n",
        "    RobertaConfig,\n",
        "    RobertaForTokenClassification,\n",
        "    RobertaTokenizer,\n",
        "    XLMRobertaConfig,\n",
        "    XLMRobertaForTokenClassification,\n",
        "    XLMRobertaTokenizer,\n",
        "    get_linear_schedule_with_warmup,\n",
        "    get_constant_schedule_with_warmup,\n",
        ")\n",
        "\n",
        "try:\n",
        "    from torch.utils.tensorboard import SummaryWriter\n",
        "except ImportError:\n",
        "    from tensorboardX import SummaryWriter\n",
        "\n",
        "logger = logging.getLogger(\"Afri_NER_Log\")\n",
        "logging.basicConfig(level=logging.DEBUG)\n",
        "\n",
        "MODEL_CLASSES = {\n",
        "    \"bert\": (BertConfig, BertForTokenClassification, BertTokenizer),\n",
        "    \"roberta\": (RobertaConfig, RobertaForTokenClassification, RobertaTokenizer),\n",
        "    \"distilbert\": (DistilBertConfig, DistilBertForTokenClassification, DistilBertTokenizer),\n",
        "    \"camembert\": (CamembertConfig, CamembertForTokenClassification, CamembertTokenizer),\n",
        "    \"xlmroberta\": (XLMRobertaConfig, XLMRobertaForTokenClassification, XLMRobertaTokenizer),\n",
        "}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "5Pk0IM2p5iMW"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "t7eSJUyA5iMW"
      },
      "source": [
        "### 3. Utility classes and functions"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YoiYUU8R5iMW"
      },
      "source": [
        "Here, we write utility classes and functions that we will use for training. You can just run all the cells below.\n",
        "\n",
        "**PLEASE DO NOT MAKE ANY CHANGES IN THIS SECTION**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vVvAgXY15iMW"
      },
      "source": [
        "We begin by writing custom datasets for our NER task"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "scrolled": true,
        "id": "_pQ_dDUG5iMW"
      },
      "outputs": [],
      "source": [
        "class InputExample(object):\n",
        "    \"\"\"A single training/test example for token classification.\"\"\"\n",
        "\n",
        "    def __init__(self, guid, words, labels):\n",
        "        \"\"\"Constructs a InputExample.\n",
        "        Args:\n",
        "            guid: Unique id for the example.\n",
        "            words: list. The words of the sequence.\n",
        "            labels: (Optional) list. The labels for each word of the sequence. This should be\n",
        "            specified for train and dev examples, but not for test examples.\n",
        "        \"\"\"\n",
        "        self.guid = guid\n",
        "        self.words = words\n",
        "        self.labels = labels\n",
        "\n",
        "class InputFeatures(object):\n",
        "    \"\"\"A single set of features of data.\"\"\"\n",
        "\n",
        "    def __init__(self, input_ids, input_mask, segment_ids, label_ids):\n",
        "        self.input_ids = input_ids\n",
        "        self.input_mask = input_mask\n",
        "        self.segment_ids = segment_ids\n",
        "        self.label_ids = label_ids\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "go5VEwxR5iMW"
      },
      "source": [
        "Next, we define the train and evaluation functions"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "GWS0CY0L5iMW"
      },
      "outputs": [],
      "source": [
        "def train(args, train_dataset, model, tokenizer, labels, pad_token_label_id):\n",
        "    \"\"\" Train the model \"\"\"\n",
        "    if args.local_rank in [-1, 0]:\n",
        "        tb_writer = SummaryWriter()\n",
        "\n",
        "    args.train_batch_size = args.per_gpu_train_batch_size * max(1, args.n_gpu)\n",
        "    train_sampler = RandomSampler(train_dataset) if args.local_rank == -1 else DistributedSampler(train_dataset)\n",
        "    train_dataloader = DataLoader(train_dataset, sampler=train_sampler, batch_size=args.train_batch_size)\n",
        "\n",
        "    if args.max_steps > 0:\n",
        "        t_total = args.max_steps\n",
        "        args.num_train_epochs = args.max_steps // (len(train_dataloader) // args.gradient_accumulation_steps) + 1\n",
        "    else:\n",
        "        t_total = len(train_dataloader) // args.gradient_accumulation_steps * args.num_train_epochs\n",
        "\n",
        "    # Prepare optimizer and schedule (linear warmup and decay)\n",
        "    no_decay = [\"bias\", \"LayerNorm.weight\"]\n",
        "    optimizer_grouped_parameters = [\n",
        "        {\n",
        "            \"params\": [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)],\n",
        "            \"weight_decay\": args.weight_decay,\n",
        "        },\n",
        "        {\"params\": [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], \"weight_decay\": 0.0},\n",
        "    ]\n",
        "    optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon)\n",
        "    scheduler = get_linear_schedule_with_warmup(\n",
        "        optimizer, num_warmup_steps=args.warmup_steps, num_training_steps=t_total\n",
        "    )\n",
        "\n",
        "    # Check if saved optimizer or scheduler states exist\n",
        "    if os.path.isfile(os.path.join(args.model_name_or_path, \"optimizer.pt\")) and os.path.isfile(\n",
        "            os.path.join(args.model_name_or_path, \"scheduler.pt\")\n",
        "    ):\n",
        "        # Load in optimizer and scheduler states\n",
        "        optimizer.load_state_dict(torch.load(os.path.join(args.model_name_or_path, \"optimizer.pt\")))\n",
        "        scheduler.load_state_dict(torch.load(os.path.join(args.model_name_or_path, \"scheduler.pt\")))\n",
        "\n",
        "    # multi-gpu training (should be after apex fp16 initialization)\n",
        "    if args.n_gpu > 1:\n",
        "        model = torch.nn.DataParallel(model)\n",
        "\n",
        "    # Distributed training (should be after apex fp16 initialization)\n",
        "    if args.local_rank != -1:\n",
        "        model = torch.nn.parallel.DistributedDataParallel(\n",
        "            model, device_ids=[args.local_rank], output_device=args.local_rank, find_unused_parameters=True\n",
        "        )\n",
        "\n",
        "    # Train!\n",
        "    logger.info(\"***** Running training *****\")\n",
        "    logger.info(\"  Num examples = %d\", len(train_dataset))\n",
        "    logger.info(\"  Num Epochs = %d\", args.num_train_epochs)\n",
        "    logger.info(\"  Instantaneous batch size per GPU = %d\", args.per_gpu_train_batch_size)\n",
        "    logger.info(\n",
        "        \"  Total train batch size (w. parallel, distributed & accumulation) = %d\",\n",
        "        args.train_batch_size\n",
        "        * args.gradient_accumulation_steps\n",
        "        * (torch.distributed.get_world_size() if args.local_rank != -1 else 1),\n",
        "    )\n",
        "    logger.info(\"  Gradient Accumulation steps = %d\", args.gradient_accumulation_steps)\n",
        "    logger.info(\"  Total optimization steps = %d\", t_total)\n",
        "\n",
        "    global_step = 0\n",
        "    epochs_trained = 0\n",
        "    steps_trained_in_current_epoch = 0\n",
        "    # Check if continuing training from a checkpoint\n",
        "    if os.path.exists(args.model_name_or_path):\n",
        "        # set global_step to gobal_step of last saved checkpoint from model path\n",
        "        try:\n",
        "            global_step = int(args.model_name_or_path.split(\"-\")[-1].split(\"/\")[0])\n",
        "        except ValueError:\n",
        "            global_step = 0\n",
        "        epochs_trained = global_step // (len(train_dataloader) // args.gradient_accumulation_steps)\n",
        "        steps_trained_in_current_epoch = global_step % (len(train_dataloader) // args.gradient_accumulation_steps)\n",
        "\n",
        "        logger.info(\"  Continuing training from checkpoint, will skip to saved global_step\")\n",
        "        logger.info(\"  Continuing training from epoch %d\", epochs_trained)\n",
        "        logger.info(\"  Continuing training from global step %d\", global_step)\n",
        "        logger.info(\"  Will skip the first %d steps in the first epoch\", steps_trained_in_current_epoch)\n",
        "\n",
        "    tr_loss, logging_loss = 0.0, 0.0\n",
        "    model.zero_grad()\n",
        "    train_iterator = trange(\n",
        "        epochs_trained, int(args.num_train_epochs), desc=\"Epoch\", disable=args.local_rank not in [-1, 0]\n",
        "    )\n",
        "    set_seed(args)  # Added here for reproductibility\n",
        "    for _ in train_iterator:\n",
        "        epoch_iterator = tqdm(train_dataloader, desc=\"Iteration\", disable=args.local_rank not in [-1, 0])\n",
        "        for step, batch in enumerate(epoch_iterator):\n",
        "\n",
        "            # Skip past any already trained steps if resuming training\n",
        "            if steps_trained_in_current_epoch > 0:\n",
        "                steps_trained_in_current_epoch -= 1\n",
        "                continue\n",
        "\n",
        "            model.train()\n",
        "            batch = tuple(t.to(args.device) for t in batch)\n",
        "            inputs = {\"input_ids\": batch[0], \"attention_mask\": batch[1], \"labels\": batch[3]}\n",
        "            if args.model_type != \"distilbert\":\n",
        "                inputs[\"token_type_ids\"] = (\n",
        "                    batch[2] if args.model_type in [\"bert\", \"xlnet\"] else None\n",
        "                )  # XLM and RoBERTa don\"t use segment_ids\n",
        "\n",
        "            outputs = model(**inputs)\n",
        "            loss = outputs[0]  # model outputs are always tuple in pytorch-transformers (see doc)\n",
        "\n",
        "            if args.n_gpu > 1:\n",
        "                loss = loss.mean()  # mean() to average on multi-gpu parallel training\n",
        "            if args.gradient_accumulation_steps > 1:\n",
        "                loss = loss / args.gradient_accumulation_steps\n",
        "\n",
        "            loss.backward()\n",
        "\n",
        "            tr_loss += loss.item()\n",
        "            if (step + 1) % args.gradient_accumulation_steps == 0:\n",
        "                torch.nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm)\n",
        "\n",
        "                scheduler.step()  # Update learning rate schedule\n",
        "                optimizer.step()\n",
        "                model.zero_grad()\n",
        "                global_step += 1\n",
        "\n",
        "                if args.local_rank in [-1, 0] and args.logging_steps > 0 and global_step % args.logging_steps == 0:\n",
        "                    # Log metrics\n",
        "                    if (\n",
        "                            args.local_rank == -1 and args.evaluate_during_training\n",
        "                    ):  # Only evaluate when single GPU otherwise metrics may not average well\n",
        "                        results, _ = evaluate(args, model, tokenizer, labels, pad_token_label_id, mode=\"dev\")\n",
        "                        for key, value in results.items():\n",
        "                            tb_writer.add_scalar(\"eval_{}\".format(key), value, global_step)\n",
        "                    tb_writer.add_scalar(\"lr\", scheduler.get_lr()[0], global_step)\n",
        "                    tb_writer.add_scalar(\"loss\", (tr_loss - logging_loss) / args.logging_steps, global_step)\n",
        "                    logging_loss = tr_loss\n",
        "\n",
        "                if args.local_rank in [-1, 0] and args.save_steps > 0 and global_step % args.save_steps == 0:\n",
        "                    # Save model checkpoint\n",
        "                    output_dir = os.path.join(args.output_dir, \"checkpoint-{}\".format(global_step))\n",
        "                    if not os.path.exists(output_dir):\n",
        "                        os.makedirs(output_dir)\n",
        "                    model_to_save = (\n",
        "                        model.module if hasattr(model, \"module\") else model\n",
        "                    )  # Take care of distributed/parallel training\n",
        "                    model_to_save.save_pretrained(output_dir)\n",
        "                    tokenizer.save_pretrained(output_dir)\n",
        "\n",
        "                    torch.save(args, os.path.join(output_dir, \"training_args.bin\"))\n",
        "                    logger.info(\"Saving model checkpoint to %s\", output_dir)\n",
        "\n",
        "                    torch.save(optimizer.state_dict(), os.path.join(output_dir, \"optimizer.pt\"))\n",
        "                    torch.save(scheduler.state_dict(), os.path.join(output_dir, \"scheduler.pt\"))\n",
        "                    logger.info(\"Saving optimizer and scheduler states to %s\", output_dir)\n",
        "\n",
        "            if args.max_steps > 0 and global_step > args.max_steps:\n",
        "                epoch_iterator.close()\n",
        "                break\n",
        "        if args.max_steps > 0 and global_step > args.max_steps:\n",
        "            train_iterator.close()\n",
        "            break\n",
        "\n",
        "    if args.local_rank in [-1, 0]:\n",
        "        tb_writer.close()\n",
        "\n",
        "    return global_step, tr_loss / global_step"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "H-aiOYmO5iMX"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "OH1w2auf5iMX"
      },
      "outputs": [],
      "source": [
        "def evaluate(args, model, tokenizer, labels, pad_token_label_id, mode, prefix=\"\"):\n",
        "    eval_dataset = load_and_cache_examples(args, tokenizer, labels, pad_token_label_id, mode=mode)\n",
        "\n",
        "    args.eval_batch_size = args.per_gpu_eval_batch_size * max(1, args.n_gpu)\n",
        "    # Note that DistributedSampler samples randomly\n",
        "    eval_sampler = SequentialSampler(eval_dataset) if args.local_rank == -1 else DistributedSampler(eval_dataset)\n",
        "    eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size)\n",
        "\n",
        "    # multi-gpu evaluate\n",
        "    if args.n_gpu > 1:\n",
        "        model = torch.nn.DataParallel(model)\n",
        "\n",
        "    # Eval!\n",
        "    logger.info(\"***** Running evaluation %s *****\", prefix)\n",
        "    logger.info(\"  Num examples = %d\", len(eval_dataset))\n",
        "    logger.info(\"  Batch size = %d\", args.eval_batch_size)\n",
        "    eval_loss = 0.0\n",
        "    nb_eval_steps = 0\n",
        "    preds = None\n",
        "    out_label_ids = None\n",
        "    model.eval()\n",
        "    for batch in tqdm(eval_dataloader, desc=\"Evaluating\"):\n",
        "        batch = tuple(t.to(args.device) for t in batch)\n",
        "\n",
        "        with torch.no_grad():\n",
        "            inputs = {\"input_ids\": batch[0], \"attention_mask\": batch[1], \"labels\": batch[3]}\n",
        "            if args.model_type != \"distilbert\":\n",
        "                inputs[\"token_type_ids\"] = (\n",
        "                    batch[2] if args.model_type in [\"bert\", \"xlnet\"] else None\n",
        "                )  # XLM and RoBERTa don\"t use segment_ids\n",
        "            outputs = model(**inputs)\n",
        "            tmp_eval_loss, logits = outputs[:2]\n",
        "\n",
        "            if args.n_gpu > 1:\n",
        "                tmp_eval_loss = tmp_eval_loss.mean()  # mean() to average on multi-gpu parallel evaluating\n",
        "\n",
        "            eval_loss += tmp_eval_loss.item()\n",
        "        nb_eval_steps += 1\n",
        "        if preds is None:\n",
        "            preds = logits.detach().cpu().numpy()\n",
        "            out_label_ids = inputs[\"labels\"].detach().cpu().numpy()\n",
        "        else:\n",
        "            preds = np.append(preds, logits.detach().cpu().numpy(), axis=0)\n",
        "            out_label_ids = np.append(out_label_ids, inputs[\"labels\"].detach().cpu().numpy(), axis=0)\n",
        "\n",
        "    eval_loss = eval_loss / nb_eval_steps\n",
        "    preds = np.argmax(preds, axis=2)\n",
        "\n",
        "    label_map = {i: label for i, label in enumerate(labels)}\n",
        "\n",
        "    out_label_list = [[] for _ in range(out_label_ids.shape[0])]\n",
        "    preds_list = [[] for _ in range(out_label_ids.shape[0])]\n",
        "\n",
        "    for i in range(out_label_ids.shape[0]):\n",
        "        for j in range(out_label_ids.shape[1]):\n",
        "            if out_label_ids[i, j] != pad_token_label_id:\n",
        "                out_label_list[i].append(label_map[out_label_ids[i][j]])\n",
        "                preds_list[i].append(label_map[preds[i][j]])\n",
        "\n",
        "    results = {\n",
        "        \"loss\": eval_loss,\n",
        "        \"precision\": precision_score(out_label_list, preds_list),\n",
        "        \"recall\": recall_score(out_label_list, preds_list),\n",
        "        \"f1\": f1_score(out_label_list, preds_list),\n",
        "    }\n",
        "\n",
        "    logger.info(\"***** Eval results %s *****\", prefix)\n",
        "    for key in sorted(results.keys()):\n",
        "        logger.info(\"  %s = %s\", key, str(results[key]))\n",
        "\n",
        "    return results, preds_list\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1Gv0nswE5iMX"
      },
      "source": [
        "Next, we define functions that will help us load and preprocess the examples."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "IivioOhU5iMX"
      },
      "outputs": [],
      "source": [
        "def load_and_cache_examples(args, tokenizer, labels, pad_token_label_id, mode):\n",
        "    if args.local_rank not in [-1, 0] and not evaluate:\n",
        "        torch.distributed.barrier()  # Make sure only the first process in distributed training process the dataset, and the others will use the cache\n",
        "\n",
        "    # Load data features from cache or dataset file\n",
        "    cached_features_file = os.path.join(\n",
        "        args.data_dir,\n",
        "        \"cached_{}_{}_{}\".format(\n",
        "            mode, list(filter(None, args.model_name_or_path.split(\"/\"))).pop(), str(args.max_seq_length)\n",
        "        ),\n",
        "    )\n",
        "    if os.path.exists(cached_features_file) and not args.overwrite_cache:\n",
        "        logger.info(\"Loading features from cached file %s\", cached_features_file)\n",
        "        features = torch.load(cached_features_file)\n",
        "    else:\n",
        "        logger.info(\"Creating features from dataset file at %s\", args.data_dir)\n",
        "        examples = read_examples_from_file(args.data_dir, mode)\n",
        "        features = convert_examples_to_features(\n",
        "            examples,\n",
        "            labels,\n",
        "            args.max_seq_length,\n",
        "            tokenizer,\n",
        "            cls_token_at_end=bool(args.model_type in [\"xlnet\"]),\n",
        "            # xlnet has a cls token at the end\n",
        "            cls_token=tokenizer.cls_token,\n",
        "            cls_token_segment_id=2 if args.model_type in [\"xlnet\"] else 0,\n",
        "            sep_token=tokenizer.sep_token,\n",
        "            sep_token_extra=bool(args.model_type in [\"roberta\"]),\n",
        "            # roberta uses an extra separator b/w pairs of sentences, cf. github.com/pytorch/fairseq/commit/1684e166e3da03f5b600dbb7855cb98ddfcd0805\n",
        "            pad_on_left=bool(args.model_type in [\"xlnet\"]),\n",
        "            # pad on the left for xlnet\n",
        "            pad_token=tokenizer.convert_tokens_to_ids([tokenizer.pad_token])[0],\n",
        "            pad_token_segment_id=4 if args.model_type in [\"xlnet\"] else 0,\n",
        "            pad_token_label_id=pad_token_label_id,\n",
        "        )\n",
        "        if args.local_rank in [-1, 0]:\n",
        "            logger.info(\"Saving features into cached file %s\", cached_features_file)\n",
        "            torch.save(features, cached_features_file)\n",
        "\n",
        "    if args.local_rank == 0 and not evaluate:\n",
        "        torch.distributed.barrier()  # Make sure only the first process in distributed training process the dataset, and the others will use the cache\n",
        "\n",
        "    # Convert to Tensors and build dataset\n",
        "    all_input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long)\n",
        "    all_input_mask = torch.tensor([f.input_mask for f in features], dtype=torch.long)\n",
        "    all_segment_ids = torch.tensor([f.segment_ids for f in features], dtype=torch.long)\n",
        "    all_label_ids = torch.tensor([f.label_ids for f in features], dtype=torch.long)\n",
        "\n",
        "    dataset = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids)\n",
        "    return dataset\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ofASQjhW5iMX"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "NoyPzFyt5iMX"
      },
      "outputs": [],
      "source": [
        "def read_examples_from_file(data_dir, mode):\n",
        "    file_path = os.path.join(data_dir, \"{}.txt\".format(mode))\n",
        "    guid_index = 1\n",
        "    examples = []\n",
        "    with open(file_path, encoding=\"utf-8\") as f:\n",
        "        words = []\n",
        "        labels = []\n",
        "        for line in f:\n",
        "            line = line.strip()\n",
        "            if len(line) < 2  or line == \"\\n\":\n",
        "                #print(line, words)\n",
        "                if words:\n",
        "                    examples.append(InputExample(guid=\"{}-{}\".format(mode, guid_index), words=words, labels=labels))\n",
        "                    guid_index += 1\n",
        "                    words = []\n",
        "                    labels = []\n",
        "            else:\n",
        "                splits = line.split(\" \")\n",
        "                words.append(splits[0])\n",
        "                if len(splits) > 1:\n",
        "                    labels.append(splits[-1].replace(\"\\n\", \"\"))\n",
        "                else:\n",
        "                    # Examples could have no label for mode = \"test\"\n",
        "                    labels.append(\"O\")\n",
        "        if words:\n",
        "            examples.append(InputExample(guid=\"{}-{}\".format(mode, guid_index), words=words, labels=labels))\n",
        "    return examples\n",
        "\n",
        "\n",
        "def convert_examples_to_features(\n",
        "    examples,\n",
        "    label_list,\n",
        "    max_seq_length,\n",
        "    tokenizer,\n",
        "    cls_token_at_end=False,\n",
        "    cls_token=\"[CLS]\",\n",
        "    cls_token_segment_id=1,\n",
        "    sep_token=\"[SEP]\",\n",
        "    sep_token_extra=False,\n",
        "    pad_on_left=False,\n",
        "    pad_token=0,\n",
        "    pad_token_segment_id=0,\n",
        "    pad_token_label_id=-100,\n",
        "    sequence_a_segment_id=0,\n",
        "    mask_padding_with_zero=True,\n",
        "):\n",
        "    \"\"\" Loads a data file into a list of `InputBatch`s\n",
        "        `cls_token_at_end` define the location of the CLS token:\n",
        "            - False (Default, BERT/XLM pattern): [CLS] + A + [SEP] + B + [SEP]\n",
        "            - True (XLNet/GPT pattern): A + [SEP] + B + [SEP] + [CLS]\n",
        "        `cls_token_segment_id` define the segment id associated to the CLS token (0 for BERT, 2 for XLNet)\n",
        "    \"\"\"\n",
        "\n",
        "    label_map = {label: i for i, label in enumerate(label_list)}\n",
        "\n",
        "    features = []\n",
        "    for (ex_index, example) in enumerate(examples):\n",
        "        #print(ex_index, len(example.words))\n",
        "        if ex_index % 10000 == 0:\n",
        "            logger.info(\"Writing example %d of %d\", ex_index, len(examples))\n",
        "\n",
        "        tokens = []\n",
        "        label_ids = []\n",
        "        for word, label in zip(example.words, example.labels):\n",
        "            word_tokens = tokenizer.tokenize(word)\n",
        "            tokens.extend(word_tokens)\n",
        "            # Use the real label id for the first token of the word, and padding ids for the remaining tokens\n",
        "            label_ids.extend([label_map[label]] + [pad_token_label_id] * (len(word_tokens) - 1))\n",
        "\n",
        "        # Account for [CLS] and [SEP] with \"- 2\" and with \"- 3\" for RoBERTa.\n",
        "        special_tokens_count = 3 if sep_token_extra else 2\n",
        "        if len(tokens) > max_seq_length - special_tokens_count:\n",
        "            tokens = tokens[: (max_seq_length - special_tokens_count)]\n",
        "            label_ids = label_ids[: (max_seq_length - special_tokens_count)]\n",
        "\n",
        "        # The convention in BERT is:\n",
        "        # (a) For sequence pairs:\n",
        "        #  tokens:   [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]\n",
        "        #  type_ids:   0   0  0    0    0     0       0   0   1  1  1  1   1   1\n",
        "        # (b) For single sequences:\n",
        "        #  tokens:   [CLS] the dog is hairy . [SEP]\n",
        "        #  type_ids:   0   0   0   0  0     0   0\n",
        "        #\n",
        "        # Where \"type_ids\" are used to indicate whether this is the first\n",
        "        # sequence or the second sequence. The embedding vectors for `type=0` and\n",
        "        # `type=1` were learned during pre-training and are added to the wordpiece\n",
        "        # embedding vector (and position vector). This is not *strictly* necessary\n",
        "        # since the [SEP] token unambiguously separates the sequences, but it makes\n",
        "        # it easier for the model to learn the concept of sequences.\n",
        "        #\n",
        "        # For classification tasks, the first vector (corresponding to [CLS]) is\n",
        "        # used as as the \"sentence vector\". Note that this only makes sense because\n",
        "        # the entire model is fine-tuned.\n",
        "        tokens += [sep_token]\n",
        "        label_ids += [pad_token_label_id]\n",
        "        if sep_token_extra:\n",
        "            # roberta uses an extra separator b/w pairs of sentences\n",
        "            tokens += [sep_token]\n",
        "            label_ids += [pad_token_label_id]\n",
        "        segment_ids = [sequence_a_segment_id] * len(tokens)\n",
        "\n",
        "        if cls_token_at_end:\n",
        "            tokens += [cls_token]\n",
        "            label_ids += [pad_token_label_id]\n",
        "            segment_ids += [cls_token_segment_id]\n",
        "        else:\n",
        "            tokens = [cls_token] + tokens\n",
        "            label_ids = [pad_token_label_id] + label_ids\n",
        "            segment_ids = [cls_token_segment_id] + segment_ids\n",
        "\n",
        "        input_ids = tokenizer.convert_tokens_to_ids(tokens)\n",
        "\n",
        "        # The mask has 1 for real tokens and 0 for padding tokens. Only real\n",
        "        # tokens are attended to.\n",
        "        input_mask = [1 if mask_padding_with_zero else 0] * len(input_ids)\n",
        "\n",
        "        # Zero-pad up to the sequence length.\n",
        "        padding_length = max_seq_length - len(input_ids)\n",
        "        if pad_on_left:\n",
        "            input_ids = ([pad_token] * padding_length) + input_ids\n",
        "            input_mask = ([0 if mask_padding_with_zero else 1] * padding_length) + input_mask\n",
        "            segment_ids = ([pad_token_segment_id] * padding_length) + segment_ids\n",
        "            label_ids = ([pad_token_label_id] * padding_length) + label_ids\n",
        "        else:\n",
        "            input_ids += [pad_token] * padding_length\n",
        "            input_mask += [0 if mask_padding_with_zero else 1] * padding_length\n",
        "            segment_ids += [pad_token_segment_id] * padding_length\n",
        "            label_ids += [pad_token_label_id] * padding_length\n",
        "\n",
        "        assert len(input_ids) == max_seq_length\n",
        "        assert len(input_mask) == max_seq_length\n",
        "        assert len(segment_ids) == max_seq_length\n",
        "        try:\n",
        "            assert len(label_ids) == max_seq_length\n",
        "        except:\n",
        "            continue\n",
        "\n",
        "        if ex_index < 5:\n",
        "            logger.info(\"*** Example ***\")\n",
        "            logger.info(\"guid: %s\", example.guid)\n",
        "            logger.info(\"tokens: %s\", \" \".join([str(x) for x in tokens]))\n",
        "            logger.info(\"input_ids: %s\", \" \".join([str(x) for x in input_ids]))\n",
        "            logger.info(\"input_mask: %s\", \" \".join([str(x) for x in input_mask]))\n",
        "            logger.info(\"segment_ids: %s\", \" \".join([str(x) for x in segment_ids]))\n",
        "            logger.info(\"label_ids: %s\", \" \".join([str(x) for x in label_ids]))\n",
        "\n",
        "        features.append(\n",
        "            InputFeatures(input_ids=input_ids, input_mask=input_mask, segment_ids=segment_ids, label_ids=label_ids)\n",
        "        )\n",
        "    return features\n",
        "\n",
        "def get_labels(path):\n",
        "    if path:\n",
        "        with open(path, \"r\") as f:\n",
        "            labels = f.read().splitlines()\n",
        "        if \"O\" not in labels:\n",
        "            labels = [\"O\"] + labels\n",
        "        return labels\n",
        "    else:\n",
        "        return [\"O\", \"B-DATE\", \"I-DATE\", \"B-PER\", \"I-PER\", \"B-ORG\", \"I-ORG\", \"B-LOC\", \"I-LOC\"]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dXfue3xi5iMX"
      },
      "source": [
        "Next, we define a function to set the seed and the function to start the actual training"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "id": "Appdswam5iMX"
      },
      "outputs": [],
      "source": [
        "def set_seed(args):\n",
        "    \"\"\"Set seed for training\"\"\"\n",
        "    random.seed(args.seed)\n",
        "    np.random.seed(args.seed)\n",
        "    torch.manual_seed(args.seed)\n",
        "    if args.n_gpu > 0:\n",
        "        torch.cuda.manual_seed_all(args.seed)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "u7XFcJQl5iMX"
      },
      "outputs": [],
      "source": [
        "def start_training(args):\n",
        "    \"\"\"\n",
        "    Start the actual training process\n",
        "    \"\"\"\n",
        "    if (\n",
        "        os.path.exists(args.output_dir)\n",
        "        and os.listdir(args.output_dir)\n",
        "        and args.do_train\n",
        "        and not args.overwrite_output_dir\n",
        "    ):\n",
        "        raise ValueError(\n",
        "            \"Output directory ({}) already exists and is not empty. Use --overwrite_output_dir to overcome.\".format(\n",
        "                args.output_dir\n",
        "            )\n",
        "        )\n",
        "\n",
        "    # Setup distant debugging if needed\n",
        "    if args.server_ip and args.server_port:\n",
        "        # Distant debugging - see https://code.visualstudio.com/docs/python/debugging#_attach-to-a-local-script\n",
        "        import ptvsd\n",
        "\n",
        "        print(\"Waiting for debugger attach\")\n",
        "        ptvsd.enable_attach(address=(args.server_ip, args.server_port), redirect_output=True)\n",
        "        ptvsd.wait_for_attach()\n",
        "\n",
        "    # Setup CUDA, GPU & distributed training\n",
        "    if args.local_rank == -1 or args.no_cuda:\n",
        "        device = torch.device(\"cuda\" if torch.cuda.is_available() and not args.no_cuda else \"cpu\")\n",
        "        args.n_gpu = torch.cuda.device_count()\n",
        "    else:  # Initializes the distributed backend which will take care of sychronizing nodes/GPUs\n",
        "        torch.cuda.set_device(args.local_rank)\n",
        "        device = torch.device(\"cuda\", args.local_rank)\n",
        "        torch.distributed.init_process_group(backend=\"nccl\")\n",
        "        args.n_gpu = 1\n",
        "    args.device = device\n",
        "\n",
        "    # Setup logging\n",
        "    logging.basicConfig(\n",
        "        format=\"%(asctime)s - %(levelname)s - %(name)s -   %(message)s\",\n",
        "        datefmt=\"%m/%d/%Y %H:%M:%S\",\n",
        "        level=logging.INFO if args.local_rank in [-1, 0] else logging.WARN,\n",
        "    )\n",
        "\n",
        "    # Set seed\n",
        "    set_seed(args)\n",
        "\n",
        "    # Prepare CONLL-2003 task\n",
        "    labels = get_labels(args.labels)\n",
        "    num_labels = len(labels)\n",
        "    # Use cross entropy ignore index as padding label id so that only real label ids contribute to the loss later\n",
        "    pad_token_label_id = CrossEntropyLoss().ignore_index\n",
        "\n",
        "    # Load pretrained model and tokenizer\n",
        "    if args.local_rank not in [-1, 0]:\n",
        "        torch.distributed.barrier()  # Make sure only the first process in distributed training will download model & vocab\n",
        "\n",
        "    args.model_type = args.model_type.lower()\n",
        "    config_class, model_class, tokenizer_class = AutoConfig, AutoModelForTokenClassification, AutoTokenizer #MODEL_CLASSES[args.model_type]\n",
        "\n",
        "    config = config_class.from_pretrained(\n",
        "    args.config_name if args.config_name else args.model_name_or_path,\n",
        "    num_labels=num_labels,\n",
        "    id2label={str(i): label for i, label in enumerate(labels)},\n",
        "    label2id={label: i for i, label in enumerate(labels)},\n",
        "    cache_dir=args.cache_dir if args.cache_dir else None,\n",
        "    )\n",
        "    tokenizer = tokenizer_class.from_pretrained(\n",
        "        args.tokenizer_name if args.tokenizer_name else args.model_name_or_path,\n",
        "        #do_lower_case=args.do_lower_case,\n",
        "        cache_dir=args.cache_dir if args.cache_dir else None,\n",
        "        #use_fast=args.use_fast,\n",
        "    )\n",
        "    model = model_class.from_pretrained(\n",
        "        args.model_name_or_path,\n",
        "        from_tf=bool(\".ckpt\" in args.model_name_or_path),\n",
        "        config=config,\n",
        "        cache_dir=args.cache_dir if args.cache_dir else None,\n",
        "    )\n",
        "\n",
        "    if args.local_rank == 0:\n",
        "        torch.distributed.barrier()  # Make sure only the first process in distributed training will download model & vocab\n",
        "\n",
        "    model.to(args.device)\n",
        "\n",
        "    logger.info(\"Training/evaluation parameters %s\", args)\n",
        "\n",
        "    # Training\n",
        "    if args.do_train:\n",
        "        train_dataset = load_and_cache_examples(args, tokenizer, labels, pad_token_label_id, mode=\"train\")\n",
        "        #train_dataset = load_examples(args, mode=\"train\")\n",
        "        global_step, tr_loss = train(args, train_dataset, model, tokenizer, labels, pad_token_label_id)\n",
        "        #global_step, tr_loss = train_ner(args, train_dataset, model, tokenizer, labels, pad_token_label_id)\n",
        "        logger.info(\" global_step = %s, average loss = %s\", global_step, tr_loss)\n",
        "\n",
        "    # Fine-tuning\n",
        "    if args.do_finetune:\n",
        "        tokenizer = tokenizer_class.from_pretrained(args.input_dir, do_lower_case=args.do_lower_case)\n",
        "        model = model_class.from_pretrained(args.input_dir)\n",
        "        model.to(args.device)\n",
        "        result, predictions = evaluate(args, model, tokenizer, labels, pad_token_label_id, mode=\"test\")\n",
        "        train_dataset = load_and_cache_examples(args, tokenizer, labels, pad_token_label_id, mode=\"train\")\n",
        "\n",
        "        # train_dataset = load_examples(args, mode=\"train\")\n",
        "        global_step, tr_loss = train(args, train_dataset, model, tokenizer, labels, pad_token_label_id)\n",
        "        # global_step, tr_loss = train_ner(args, train_dataset, model, tokenizer, labels, pad_token_label_id)\n",
        "        logger.info(\" global_step = %s, average loss = %s\", global_step, tr_loss)\n",
        "\n",
        "    # Saving best-practices: if you use defaults names for the model, you can reload it using from_pretrained()\n",
        "    if (args.do_train or args.do_finetune) and (args.local_rank == -1 or torch.distributed.get_rank() == 0):\n",
        "        # Create output directory if needed\n",
        "        if not os.path.exists(args.output_dir) and args.local_rank in [-1, 0]:\n",
        "            os.makedirs(args.output_dir)\n",
        "\n",
        "        logger.info(\"Saving model checkpoint to %s\", args.output_dir)\n",
        "        # Save a trained model, configuration and tokenizer using `save_pretrained()`.\n",
        "        # They can then be reloaded using `from_pretrained()`\n",
        "        model_to_save = (\n",
        "            model.module if hasattr(model, \"module\") else model\n",
        "        )  # Take care of distributed/parallel training\n",
        "        model_to_save.save_pretrained(args.output_dir)\n",
        "        tokenizer.save_pretrained(args.output_dir)\n",
        "\n",
        "        # Good practice: save your training arguments together with the trained model\n",
        "        torch.save(args, os.path.join(args.output_dir, \"training_args.bin\"))\n",
        "\n",
        "    results = {}\n",
        "    if args.do_eval and args.local_rank in [-1, 0]:\n",
        "        tokenizer = tokenizer_class.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case)\n",
        "        checkpoints = [args.output_dir]\n",
        "        if args.eval_all_checkpoints:\n",
        "            checkpoints = list(\n",
        "                os.path.dirname(c) for c in sorted(glob.glob(args.output_dir + \"/**/\" + WEIGHTS_NAME, recursive=True))\n",
        "            )\n",
        "            logging.getLogger(\"pytorch_transformers.modeling_utils\").setLevel(logging.WARN)  # Reduce logging\n",
        "        logger.info(\"Evaluate the following checkpoints: %s\", checkpoints)\n",
        "        for checkpoint in checkpoints:\n",
        "            global_step = checkpoint.split(\"-\")[-1] if len(checkpoints) > 1 else \"\"\n",
        "            model = model_class.from_pretrained(checkpoint)\n",
        "            model.to(args.device)\n",
        "            result, _ = evaluate(args, model, tokenizer, labels, pad_token_label_id, mode=\"dev\", prefix=global_step)\n",
        "            if global_step:\n",
        "                result = {\"{}_{}\".format(global_step, k): v for k, v in result.items()}\n",
        "            results.update(result)\n",
        "        output_eval_file = os.path.join(args.output_dir, \"eval_results.txt\")\n",
        "        with open(output_eval_file, \"w\") as writer:\n",
        "            for key in sorted(results.keys()):\n",
        "                writer.write(\"{} = {}\\n\".format(key, str(results[key])))\n",
        "\n",
        "    if args.do_predict and args.local_rank in [-1, 0]:\n",
        "        tokenizer = tokenizer_class.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case)\n",
        "        model = model_class.from_pretrained(args.output_dir)\n",
        "        model.to(args.device)\n",
        "        result, predictions = evaluate(args, model, tokenizer, labels, pad_token_label_id, mode=\"test\")\n",
        "        # Save results\n",
        "        output_test_results_file = os.path.join(args.output_dir, \"test_results.txt\")\n",
        "        with open(output_test_results_file, \"w\") as writer:\n",
        "            for key in sorted(result.keys()):\n",
        "                writer.write(\"{} = {}\\n\".format(key, str(result[key])))\n",
        "        # Save predictions\n",
        "        output_test_predictions_file = os.path.join(args.output_dir, \"test_predictions.txt\")\n",
        "        with open(output_test_predictions_file, \"w\") as writer:\n",
        "            with open(os.path.join(args.data_dir, \"test.txt\"), \"r\") as f:\n",
        "                example_id = 0\n",
        "                for line in f:\n",
        "                    if line.startswith(\"-DOCSTART-\") or line == \"\" or line == \"\\n\":\n",
        "                        writer.write(line)\n",
        "                        if not predictions[example_id]:\n",
        "                            example_id += 1\n",
        "                    elif predictions[example_id]:\n",
        "                        output_line = line.split()[0] + \" \" + predictions[example_id].pop(0) + \"\\n\"\n",
        "                        writer.write(output_line)\n",
        "                    else:\n",
        "                        logger.warning(\"Maximum sequence length exceeded: No prediction for '%s'.\", line.split()[0])\n",
        "\n",
        "    logger.info(results)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cKgs8s7c5iMY"
      },
      "source": [
        "### 4. Training\n",
        "\n",
        "Here, we perform the actual training process after defining the training arguments."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lyB1u8wL5iMY"
      },
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "id": "i-n4FGHO5iMY"
      },
      "outputs": [],
      "source": [
        "import argparse"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "id": "3TEm1TzD5iMY"
      },
      "outputs": [],
      "source": [
        "def get_args():\n",
        "    \"\"\"\n",
        "    Get training arguments\n",
        "    \"\"\"\n",
        "    parser = argparse.ArgumentParser()\n",
        "    # Required parameters\n",
        "    parser.add_argument(\n",
        "        \"--data_dir\",\n",
        "        default=None,\n",
        "        type=str,\n",
        "        help=\"The input data dir. Should contain the training files for the CoNLL-2003 NER task.\",\n",
        "    )\n",
        "    parser.add_argument(\n",
        "        \"--model_type\",\n",
        "        default=None,\n",
        "        type=str,\n",
        "        #help=\"Model type selected in the list: \" + \", \".join(MODEL_CLASSES.keys()),\n",
        "    )\n",
        "    parser.add_argument(\n",
        "        \"--model_name_or_path\",\n",
        "        default=None,\n",
        "        type=str,\n",
        "        help=\"Path to pre-trained model or shortcut name selected in the list: \" + \", \",\n",
        "    )\n",
        "    parser.add_argument(\n",
        "        \"--input_dir\",\n",
        "        default=None,\n",
        "        type=str,\n",
        "        required=False,\n",
        "        help=\"The input model directory.\",\n",
        "    )\n",
        "    parser.add_argument(\n",
        "        \"--output_dir\",\n",
        "        default=None,\n",
        "        type=str,\n",
        "        help=\"The output directory where the model predictions and checkpoints will be written.\",\n",
        "    )\n",
        "\n",
        "    # Other parameters\n",
        "    parser.add_argument(\n",
        "        \"--labels\",\n",
        "        default=\"\",\n",
        "        type=str,\n",
        "        help=\"Path to a file containing all labels. If not specified, CoNLL-2003 labels are used.\",\n",
        "    )\n",
        "    parser.add_argument(\n",
        "        \"--config_name\", default=\"\", type=str, help=\"Pretrained config name or path if not the same as model_name\"\n",
        "    )\n",
        "    parser.add_argument(\n",
        "        \"--tokenizer_name\",\n",
        "        default=\"\",\n",
        "        type=str,\n",
        "        help=\"Pretrained tokenizer name or path if not the same as model_name\",\n",
        "    )\n",
        "    parser.add_argument(\n",
        "        \"--cache_dir\",\n",
        "        default=\"\",\n",
        "        type=str,\n",
        "        help=\"Where do you want to store the pre-trained models downloaded from s3\",\n",
        "    )\n",
        "    parser.add_argument(\n",
        "        \"--max_seq_length\",\n",
        "        default=128,\n",
        "        type=int,\n",
        "        help=\"The maximum total input sequence length after tokenization. Sequences longer \"\n",
        "        \"than this will be truncated, sequences shorter will be padded.\",\n",
        "    )\n",
        "    parser.add_argument(\"--do_train\", action=\"store_true\", help=\"Whether to run training.\")\n",
        "    parser.add_argument(\"--do_finetune\", action=\"store_true\", help=\"Whether to run training.\")\n",
        "    parser.add_argument(\"--do_eval\", action=\"store_true\", help=\"Whether to run eval on the dev set.\")\n",
        "    parser.add_argument(\"--do_predict\", action=\"store_true\", help=\"Whether to run predictions on the test set.\")\n",
        "    parser.add_argument(\n",
        "        \"--evaluate_during_training\",\n",
        "        action=\"store_true\",\n",
        "        help=\"Whether to run evaluation during training at each logging step.\",\n",
        "    )\n",
        "    parser.add_argument(\n",
        "        \"--do_lower_case\", action=\"store_true\", help=\"Set this flag if you are using an uncased model.\"\n",
        "    )\n",
        "\n",
        "    parser.add_argument(\"--per_gpu_train_batch_size\", default=8, type=int, help=\"Batch size per GPU/CPU for training.\")\n",
        "    parser.add_argument(\n",
        "        \"--per_gpu_eval_batch_size\", default=8, type=int, help=\"Batch size per GPU/CPU for evaluation.\"\n",
        "    )\n",
        "    parser.add_argument(\n",
        "        \"--gradient_accumulation_steps\",\n",
        "        type=int,\n",
        "        default=1,\n",
        "        help=\"Number of updates steps to accumulate before performing a backward/update pass.\",\n",
        "    )\n",
        "    parser.add_argument(\"--learning_rate\", default=5e-5, type=float, help=\"The initial learning rate for Adam.\")\n",
        "    parser.add_argument(\"--weight_decay\", default=0.0, type=float, help=\"Weight decay if we apply some.\")\n",
        "    parser.add_argument(\"--adam_epsilon\", default=1e-8, type=float, help=\"Epsilon for Adam optimizer.\")\n",
        "    parser.add_argument(\"--max_grad_norm\", default=1.0, type=float, help=\"Max gradient norm.\")\n",
        "    parser.add_argument(\n",
        "        \"--num_train_epochs\", default=3.0, type=float, help=\"Total number of training epochs to perform.\"\n",
        "    )\n",
        "    parser.add_argument(\n",
        "        \"--max_steps\",\n",
        "        default=-1,\n",
        "        type=int,\n",
        "        help=\"If > 0: set total number of training steps to perform. Override num_train_epochs.\",\n",
        "    )\n",
        "    parser.add_argument(\"--warmup_steps\", default=0, type=int, help=\"Linear warmup over warmup_steps.\")\n",
        "\n",
        "    parser.add_argument(\"--logging_steps\", type=int, default=500, help=\"Log every X updates steps.\")\n",
        "    parser.add_argument(\"--save_steps\", type=int, default=500, help=\"Save checkpoint every X updates steps.\")\n",
        "    parser.add_argument(\n",
        "        \"--eval_all_checkpoints\",\n",
        "        action=\"store_true\",\n",
        "        help=\"Evaluate all checkpoints starting with the same prefix as model_name ending and ending with step number\",\n",
        "    )\n",
        "    parser.add_argument(\"--no_cuda\", action=\"store_true\", help=\"Avoid using CUDA when available\")\n",
        "    parser.add_argument(\n",
        "        \"--overwrite_output_dir\", action=\"store_true\", help=\"Overwrite the content of the output directory\"\n",
        "    )\n",
        "    parser.add_argument(\n",
        "        \"--overwrite_cache\", action=\"store_true\", help=\"Overwrite the cached training and evaluation sets\"\n",
        "    )\n",
        "    parser.add_argument(\"--seed\", type=int, default=42, help=\"random seed for initialization\")\n",
        "\n",
        "    parser.add_argument(\"--local_rank\", type=int, default=-1, help=\"For distributed training: local_rank\")\n",
        "    parser.add_argument(\"--server_ip\", type=str, default=\"\", help=\"For distant debugging.\")\n",
        "    parser.add_argument(\"--server_port\", type=str, default=\"\", help=\"For distant debugging.\")\n",
        "\n",
        "    return parser.parse_known_args()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "id": "t7pHTfTb5iMY"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qIMo8YMk5iMY"
      },
      "source": [
        "**CHANGES CAN BE MADE HERE:**\n",
        "\n",
        "Note that the argments with comments may need to be modified. The remaining arguments can be left as they are, as these are good defaults.\n",
        "\n",
        "Hence, you should start by only supplying the data directory and output directory."
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### Specify the source language data path"
      ],
      "metadata": {
        "id": "FHJL-fXsAcbE"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!git clone https://github.com/masakhane-io/NLP-at-Indaba-Hands-on-Session-2023.git"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MC7eCfTZAkr3",
        "outputId": "14368aa6-f0aa-40b1-8ac8-d0f9eb58eb47"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'NLP-at-Indaba-Hands-on-Session-2023'...\n",
            "remote: Enumerating objects: 143, done.\u001b[K\n",
            "remote: Counting objects: 100% (143/143), done.\u001b[K\n",
            "remote: Compressing objects: 100% (115/115), done.\u001b[K\n",
            "remote: Total 143 (delta 31), reused 131 (delta 23), pack-reused 0\u001b[K\n",
            "Receiving objects: 100% (143/143), 2.68 MiB | 3.20 MiB/s, done.\n",
            "Resolving deltas: 100% (31/31), done.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "source_data_path = 'NLP-at-Indaba-Hands-on-Session-2023/data/labeled/yor'"
      ],
      "metadata": {
        "id": "N3hHscAFAz5L"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!git clone https://github.com/masakhane-io/masakhane-ner.git"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gmiwGqf08VAH",
        "outputId": "631fef97-8164-4d2e-d5c6-c5d94b2c5f2d"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'masakhane-ner'...\n",
            "remote: Enumerating objects: 4182, done.\u001b[K\n",
            "remote: Counting objects: 100% (448/448), done.\u001b[K\n",
            "remote: Compressing objects: 100% (282/282), done.\u001b[K\n",
            "remote: Total 4182 (delta 183), reused 406 (delta 163), pack-reused 3734\u001b[K\n",
            "Receiving objects: 100% (4182/4182), 75.01 MiB | 11.63 MiB/s, done.\n",
            "Resolving deltas: 100% (1771/1771), done.\n",
            "Updating files: 100% (2235/2235), done.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "target_data_path = 'masakhane-ner/transfer_corpus/news/yor'"
      ],
      "metadata": {
        "id": "HwL7XrwA8YE7"
      },
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gm8ku0c85iMY"
      },
      "source": [
        "XLMR-based model training:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "id": "_YrrmSXQ5iMY"
      },
      "outputs": [],
      "source": [
        "args, _ = get_args()\n",
        "args.data_dir = source_data_path # to-change: supply data directory\n",
        "args.output_dir = \"yor-xlmr\" # to-change: supply output directory\n",
        "args.model_type = \"xlmroberta\"\n",
        "args.model_name_or_path = \"Davlan/afro-xlmr-base\"\n",
        "args.max_seq_length = 164\n",
        "args.num_train_epochs = 10\n",
        "args.per_gpu_train_batch_size = 32\n",
        "args.save_steps = 10000\n",
        "args.seed = 1\n",
        "args.do_train = True\n",
        "args.do_eval = True\n",
        "args.do_predict = True\n",
        "args.overwrite_output_dir=True"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "d1a2f9233aa64efda69082cd1778f64b",
            "51ef4cc07b714d848809e49d885b981a",
            "b2d0a308432d4c07a958f4f537e278d1",
            "1715638b7e0443bfaa2228de5a643de1",
            "e6568687a9964bdb85504e0175bdd159",
            "71a2d4ba46674fff9339a47c8da681fb",
            "252324069e0142148b8c50f742e5445d",
            "b50ee598dd9e4981b5009549b7298404",
            "698cb7dc7d9d433f9fd2f598dd908635",
            "a8988d3783124873b86380e6697e69d5",
            "800c08d849e5426f87184fa7bfa63d9b",
            "c6ba843c40254cb6bf1fd9ba762daf34",
            "9cab661270e5413ebdccdb16c93dc73c",
            "6f9ea67aa37048cf93e9842b68dc40aa",
            "ec592c7b130f44bc88e9ee8a1c2f8538",
            "e04af340389f4a0eba3a8ee751f3eb91",
            "987f44f987c8470f8f7066d2a3f0c5dc",
            "77d559234480437ca87331951eb5e19f",
            "8f2eabf21d554c189d7cd72c5f9dd6d5",
            "18db276be34f447fb08e060ff411896f",
            "f9ecb99ec54a40d387ab1318463fa207",
            "7eb2358ceb41412bbc8f94677124eb7c",
            "6174422d39b2417e902726202bcc8722",
            "9302681cba854be0b46c866cb5278c27",
            "ba7d0f2e212b4b72ac7264219c3e80cc",
            "8721b85386e247208f51e7e777acca48",
            "cb4ee02b5b7644b48f482397bbce611e",
            "d5ba654a112c4f7487f7598392a96b19",
            "81522db5e5e648eab7ae01bccf0e8602",
            "28d746c9447340369b2a9f9ac2eea990",
            "cd778a1c39124a10b87ffeae022b1f0b",
            "3c30a3e74d6b4a1999a4a0aa597e4aaf",
            "bd31aebe5d8c4343a7d36fcc35d69dba",
            "f2f486494e104cd39c4a663384540d88",
            "4e40592b9a4045faa811a545f591b65e",
            "29852b1085ab410ba74c4b150e9e4301",
            "5d09067e2fbd415ca170acfb4cd55886",
            "433ae6799f01476598b8aa30f96eff74",
            "a81a56da562c4886ac2b71e9b45e749a",
            "72bc583633124d3a89020356bd6806b0",
            "98aacd2e353544d7ab2f1c7e9f8c3ec3",
            "807081de1245427c87a90efaf26cbbe1",
            "cc90fe8809e647b29888e9f9a4a6d81c",
            "8c40950b1ea446ed9f54269d00bdef51",
            "5117a30772f0401f94b9596d16bb8dcc",
            "dbaeb75ade1d479ab689177b66039ca4",
            "d1efb98b93a94f5f8889b762784df17c",
            "1d91cb442f264bde9aa09f47c613338b",
            "9db5a653bf8c4266b8a0ca4d41e5eaa7",
            "47be910c34a149cc845fb3636db1f876",
            "2c20259c1fbb4377ae216afa511bd27a",
            "780cc3f4cd9442f9ac780d325cbf938d",
            "a6231f5ab25346869dc9a83d9bf49a42",
            "562c728bad4e449e921fa636a09f1dc8",
            "89a37b2185bb4478b20c714b81f79877",
            "7dea244ad889403db9d88b945c7293c5",
            "82fd81e468814201877c9d2bacf1d90b",
            "a13fbe9353584f42b746ba92e627b07b",
            "44ef118008e0442dab8c8ed6ed4446b8",
            "45085d56993343d08283b9f53850be1d",
            "6a1a1a3014664060a11c45d539bd1e51",
            "9c6cdc47d1914967b44c4e124515848b",
            "3a9ef5d8c7c7468bbe72aa1ab0377b7e",
            "796b04dda6a4440bb52cab7fe55f1a55",
            "ffac0b833ac946e9beb52235b89c10a1",
            "d58e0dc4ad854cd8930a75ae5b2b6d09"
          ]
        },
        "id": "cWYPnRlB5iMY",
        "outputId": "d35d1525-228f-4e5b-c933-815a44f3b4db"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)lve/main/config.json:   0%|          | 0.00/707 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "d1a2f9233aa64efda69082cd1778f64b"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)okenizer_config.json:   0%|          | 0.00/398 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "c6ba843c40254cb6bf1fd9ba762daf34"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)tencepiece.bpe.model:   0%|          | 0.00/5.07M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "6174422d39b2417e902726202bcc8722"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)/main/tokenizer.json:   0%|          | 0.00/9.08M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "f2f486494e104cd39c4a663384540d88"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)cial_tokens_map.json:   0%|          | 0.00/239 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "5117a30772f0401f94b9596d16bb8dcc"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading pytorch_model.bin:   0%|          | 0.00/1.11G [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "7dea244ad889403db9d88b945c7293c5"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of XLMRobertaForTokenClassification were not initialized from the model checkpoint at Davlan/afro-xlmr-base and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "Epoch:   0%|          | 0/10 [00:00<?, ?it/s]\n",
            "Iteration:   0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "Iteration: 100%|██████████| 1/1 [00:00<00:00,  6.72it/s]\n",
            "Epoch:  10%|█         | 1/10 [00:00<00:01,  6.09it/s]\n",
            "Iteration:   0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "Iteration: 100%|██████████| 1/1 [00:00<00:00,  6.73it/s]\n",
            "Epoch:  20%|██        | 2/10 [00:00<00:01,  6.25it/s]\n",
            "Iteration:   0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "Iteration: 100%|██████████| 1/1 [00:00<00:00,  7.50it/s]\n",
            "Epoch:  30%|███       | 3/10 [00:00<00:01,  6.56it/s]\n",
            "Iteration:   0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "Iteration: 100%|██████████| 1/1 [00:00<00:00,  7.74it/s]\n",
            "Epoch:  40%|████      | 4/10 [00:00<00:00,  6.77it/s]\n",
            "Iteration:   0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "Iteration: 100%|██████████| 1/1 [00:00<00:00,  7.55it/s]\n",
            "Epoch:  50%|█████     | 5/10 [00:00<00:00,  6.78it/s]\n",
            "Iteration:   0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "Iteration: 100%|██████████| 1/1 [00:00<00:00,  7.04it/s]\n",
            "Epoch:  60%|██████    | 6/10 [00:00<00:00,  6.72it/s]\n",
            "Iteration:   0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "Iteration: 100%|██████████| 1/1 [00:00<00:00,  7.48it/s]\n",
            "Epoch:  70%|███████   | 7/10 [00:01<00:00,  6.71it/s]\n",
            "Iteration:   0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "Iteration: 100%|██████████| 1/1 [00:00<00:00,  7.30it/s]\n",
            "Epoch:  80%|████████  | 8/10 [00:01<00:00,  6.76it/s]\n",
            "Iteration:   0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "Iteration: 100%|██████████| 1/1 [00:00<00:00,  7.33it/s]\n",
            "Epoch:  90%|█████████ | 9/10 [00:01<00:00,  6.80it/s]\n",
            "Iteration:   0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "Iteration: 100%|██████████| 1/1 [00:00<00:00,  7.33it/s]\n",
            "Epoch: 100%|██████████| 10/10 [00:01<00:00,  6.72it/s]\n",
            "Evaluating: 100%|██████████| 2/2 [00:00<00:00, 43.31it/s]\n",
            "Evaluating: 100%|██████████| 2/2 [00:00<00:00, 36.97it/s]\n"
          ]
        }
      ],
      "source": [
        "# confirm your cuda devices before setting this command\n",
        "start_training(args)"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.3"
    },
    "colab": {
      "provenance": [],
      "gpuType": "V100"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "d1a2f9233aa64efda69082cd1778f64b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_51ef4cc07b714d848809e49d885b981a",
              "IPY_MODEL_b2d0a308432d4c07a958f4f537e278d1",
              "IPY_MODEL_1715638b7e0443bfaa2228de5a643de1"
            ],
            "layout": "IPY_MODEL_e6568687a9964bdb85504e0175bdd159"
          }
        },
        "51ef4cc07b714d848809e49d885b981a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_71a2d4ba46674fff9339a47c8da681fb",
            "placeholder": "​",
            "style": "IPY_MODEL_252324069e0142148b8c50f742e5445d",
            "value": "Downloading (…)lve/main/config.json: 100%"
          }
        },
        "b2d0a308432d4c07a958f4f537e278d1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b50ee598dd9e4981b5009549b7298404",
            "max": 707,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_698cb7dc7d9d433f9fd2f598dd908635",
            "value": 707
          }
        },
        "1715638b7e0443bfaa2228de5a643de1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a8988d3783124873b86380e6697e69d5",
            "placeholder": "​",
            "style": "IPY_MODEL_800c08d849e5426f87184fa7bfa63d9b",
            "value": " 707/707 [00:00&lt;00:00, 16.0kB/s]"
          }
        },
        "e6568687a9964bdb85504e0175bdd159": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "71a2d4ba46674fff9339a47c8da681fb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "252324069e0142148b8c50f742e5445d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b50ee598dd9e4981b5009549b7298404": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "698cb7dc7d9d433f9fd2f598dd908635": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "a8988d3783124873b86380e6697e69d5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "800c08d849e5426f87184fa7bfa63d9b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c6ba843c40254cb6bf1fd9ba762daf34": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_9cab661270e5413ebdccdb16c93dc73c",
              "IPY_MODEL_6f9ea67aa37048cf93e9842b68dc40aa",
              "IPY_MODEL_ec592c7b130f44bc88e9ee8a1c2f8538"
            ],
            "layout": "IPY_MODEL_e04af340389f4a0eba3a8ee751f3eb91"
          }
        },
        "9cab661270e5413ebdccdb16c93dc73c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_987f44f987c8470f8f7066d2a3f0c5dc",
            "placeholder": "​",
            "style": "IPY_MODEL_77d559234480437ca87331951eb5e19f",
            "value": "Downloading (…)okenizer_config.json: 100%"
          }
        },
        "6f9ea67aa37048cf93e9842b68dc40aa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8f2eabf21d554c189d7cd72c5f9dd6d5",
            "max": 398,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_18db276be34f447fb08e060ff411896f",
            "value": 398
          }
        },
        "ec592c7b130f44bc88e9ee8a1c2f8538": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f9ecb99ec54a40d387ab1318463fa207",
            "placeholder": "​",
            "style": "IPY_MODEL_7eb2358ceb41412bbc8f94677124eb7c",
            "value": " 398/398 [00:00&lt;00:00, 7.47kB/s]"
          }
        },
        "e04af340389f4a0eba3a8ee751f3eb91": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "987f44f987c8470f8f7066d2a3f0c5dc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "77d559234480437ca87331951eb5e19f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "8f2eabf21d554c189d7cd72c5f9dd6d5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "18db276be34f447fb08e060ff411896f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "f9ecb99ec54a40d387ab1318463fa207": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7eb2358ceb41412bbc8f94677124eb7c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "6174422d39b2417e902726202bcc8722": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_9302681cba854be0b46c866cb5278c27",
              "IPY_MODEL_ba7d0f2e212b4b72ac7264219c3e80cc",
              "IPY_MODEL_8721b85386e247208f51e7e777acca48"
            ],
            "layout": "IPY_MODEL_cb4ee02b5b7644b48f482397bbce611e"
          }
        },
        "9302681cba854be0b46c866cb5278c27": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d5ba654a112c4f7487f7598392a96b19",
            "placeholder": "​",
            "style": "IPY_MODEL_81522db5e5e648eab7ae01bccf0e8602",
            "value": "Downloading (…)tencepiece.bpe.model: 100%"
          }
        },
        "ba7d0f2e212b4b72ac7264219c3e80cc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_28d746c9447340369b2a9f9ac2eea990",
            "max": 5069051,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_cd778a1c39124a10b87ffeae022b1f0b",
            "value": 5069051
          }
        },
        "8721b85386e247208f51e7e777acca48": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3c30a3e74d6b4a1999a4a0aa597e4aaf",
            "placeholder": "​",
            "style": "IPY_MODEL_bd31aebe5d8c4343a7d36fcc35d69dba",
            "value": " 5.07M/5.07M [00:00&lt;00:00, 21.7MB/s]"
          }
        },
        "cb4ee02b5b7644b48f482397bbce611e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d5ba654a112c4f7487f7598392a96b19": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "81522db5e5e648eab7ae01bccf0e8602": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "28d746c9447340369b2a9f9ac2eea990": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cd778a1c39124a10b87ffeae022b1f0b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "3c30a3e74d6b4a1999a4a0aa597e4aaf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bd31aebe5d8c4343a7d36fcc35d69dba": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f2f486494e104cd39c4a663384540d88": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_4e40592b9a4045faa811a545f591b65e",
              "IPY_MODEL_29852b1085ab410ba74c4b150e9e4301",
              "IPY_MODEL_5d09067e2fbd415ca170acfb4cd55886"
            ],
            "layout": "IPY_MODEL_433ae6799f01476598b8aa30f96eff74"
          }
        },
        "4e40592b9a4045faa811a545f591b65e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a81a56da562c4886ac2b71e9b45e749a",
            "placeholder": "​",
            "style": "IPY_MODEL_72bc583633124d3a89020356bd6806b0",
            "value": "Downloading (…)/main/tokenizer.json: 100%"
          }
        },
        "29852b1085ab410ba74c4b150e9e4301": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_98aacd2e353544d7ab2f1c7e9f8c3ec3",
            "max": 9081351,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_807081de1245427c87a90efaf26cbbe1",
            "value": 9081351
          }
        },
        "5d09067e2fbd415ca170acfb4cd55886": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cc90fe8809e647b29888e9f9a4a6d81c",
            "placeholder": "​",
            "style": "IPY_MODEL_8c40950b1ea446ed9f54269d00bdef51",
            "value": " 9.08M/9.08M [00:00&lt;00:00, 18.3MB/s]"
          }
        },
        "433ae6799f01476598b8aa30f96eff74": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a81a56da562c4886ac2b71e9b45e749a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "72bc583633124d3a89020356bd6806b0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "98aacd2e353544d7ab2f1c7e9f8c3ec3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "807081de1245427c87a90efaf26cbbe1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "cc90fe8809e647b29888e9f9a4a6d81c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8c40950b1ea446ed9f54269d00bdef51": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5117a30772f0401f94b9596d16bb8dcc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_dbaeb75ade1d479ab689177b66039ca4",
              "IPY_MODEL_d1efb98b93a94f5f8889b762784df17c",
              "IPY_MODEL_1d91cb442f264bde9aa09f47c613338b"
            ],
            "layout": "IPY_MODEL_9db5a653bf8c4266b8a0ca4d41e5eaa7"
          }
        },
        "dbaeb75ade1d479ab689177b66039ca4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_47be910c34a149cc845fb3636db1f876",
            "placeholder": "​",
            "style": "IPY_MODEL_2c20259c1fbb4377ae216afa511bd27a",
            "value": "Downloading (…)cial_tokens_map.json: 100%"
          }
        },
        "d1efb98b93a94f5f8889b762784df17c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_780cc3f4cd9442f9ac780d325cbf938d",
            "max": 239,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_a6231f5ab25346869dc9a83d9bf49a42",
            "value": 239
          }
        },
        "1d91cb442f264bde9aa09f47c613338b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_562c728bad4e449e921fa636a09f1dc8",
            "placeholder": "​",
            "style": "IPY_MODEL_89a37b2185bb4478b20c714b81f79877",
            "value": " 239/239 [00:00&lt;00:00, 11.5kB/s]"
          }
        },
        "9db5a653bf8c4266b8a0ca4d41e5eaa7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "47be910c34a149cc845fb3636db1f876": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2c20259c1fbb4377ae216afa511bd27a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "780cc3f4cd9442f9ac780d325cbf938d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a6231f5ab25346869dc9a83d9bf49a42": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "562c728bad4e449e921fa636a09f1dc8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "89a37b2185bb4478b20c714b81f79877": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "7dea244ad889403db9d88b945c7293c5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_82fd81e468814201877c9d2bacf1d90b",
              "IPY_MODEL_a13fbe9353584f42b746ba92e627b07b",
              "IPY_MODEL_44ef118008e0442dab8c8ed6ed4446b8"
            ],
            "layout": "IPY_MODEL_45085d56993343d08283b9f53850be1d"
          }
        },
        "82fd81e468814201877c9d2bacf1d90b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6a1a1a3014664060a11c45d539bd1e51",
            "placeholder": "​",
            "style": "IPY_MODEL_9c6cdc47d1914967b44c4e124515848b",
            "value": "Downloading pytorch_model.bin: 100%"
          }
        },
        "a13fbe9353584f42b746ba92e627b07b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3a9ef5d8c7c7468bbe72aa1ab0377b7e",
            "max": 1113271634,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_796b04dda6a4440bb52cab7fe55f1a55",
            "value": 1113271634
          }
        },
        "44ef118008e0442dab8c8ed6ed4446b8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ffac0b833ac946e9beb52235b89c10a1",
            "placeholder": "​",
            "style": "IPY_MODEL_d58e0dc4ad854cd8930a75ae5b2b6d09",
            "value": " 1.11G/1.11G [00:05&lt;00:00, 255MB/s]"
          }
        },
        "45085d56993343d08283b9f53850be1d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6a1a1a3014664060a11c45d539bd1e51": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9c6cdc47d1914967b44c4e124515848b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3a9ef5d8c7c7468bbe72aa1ab0377b7e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "796b04dda6a4440bb52cab7fe55f1a55": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "ffac0b833ac946e9beb52235b89c10a1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d58e0dc4ad854cd8930a75ae5b2b6d09": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}