xlm-roberta-large.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "editorial-cinema",
   "metadata": {
    "id": "X4cRE8IbIrIV",
    "papermill": {
     "duration": 0.040788,
     "end_time": "2021-03-26T11:44:48.428517",
     "exception": false,
     "start_time": "2021-03-26T11:44:48.387729",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "If you're opening this Notebook on colab, you will probably need to install 🤗 Transformers and 🤗 Datasets. Uncomment the following cell and run it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "surprising-lafayette",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-03-26T11:44:48.520482Z",
     "iopub.status.busy": "2021-03-26T11:44:48.519533Z",
     "iopub.status.idle": "2021-03-26T11:44:58.571615Z",
     "shell.execute_reply": "2021-03-26T11:44:58.570612Z"
    },
    "id": "MOsHUjgdIrIW",
    "outputId": "73538ad3-69d0-41c2-b7fe-5a01ad830147",
    "papermill": {
     "duration": 10.105393,
     "end_time": "2021-03-26T11:44:58.571768",
     "exception": false,
     "start_time": "2021-03-26T11:44:48.466375",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: transformers in /opt/conda/lib/python3.7/site-packages (4.3.3)\r\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting datasets\r\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Downloading datasets-1.5.0-py3-none-any.whl (192 kB)\r\n",
      "\u001b[?25l\r",
      "\u001b[K     |█▊                              | 10 kB 26.1 MB/s eta 0:00:01"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      "\u001b[K     |███▍                            | 20 kB 9.9 MB/s eta 0:00:01\r",
      "\u001b[K     |█████                           | 30 kB 8.0 MB/s eta 0:00:01\r",
      "\u001b[K     |██████▉                         | 40 kB 7.3 MB/s eta 0:00:01\r",
      "\u001b[K     |████████▌                       | 51 kB 4.7 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████▏                     | 61 kB 4.8 MB/s eta 0:00:01\r",
      "\u001b[K     |████████████                    | 71 kB 5.5 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████▋                  | 81 kB 5.9 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████▎                | 92 kB 5.4 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████               | 102 kB 4.6 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████▊             | 112 kB 4.6 MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████▍           | 122 kB 4.6 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████▏         | 133 kB 4.6 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████▉        | 143 kB 4.6 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████████▌      | 153 kB 4.6 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████████▎    | 163 kB 4.6 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████████████   | 174 kB 4.6 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████████████▋ | 184 kB 4.6 MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████████████| 192 kB 4.6 MB/s \r\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[?25hCollecting nlp\r\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Downloading nlp-0.4.0-py3-none-any.whl (1.7 MB)\r\n",
      "\u001b[?25l\r",
      "\u001b[K     |▏                               | 10 kB 24.4 MB/s eta 0:00:01\r",
      "\u001b[K     |▍                               | 20 kB 17.7 MB/s eta 0:00:01\r",
      "\u001b[K     |▋                               | 30 kB 15.3 MB/s eta 0:00:01\r",
      "\u001b[K     |▉                               | 40 kB 14.2 MB/s eta 0:00:01\r",
      "\u001b[K     |█                               | 51 kB 11.1 MB/s eta 0:00:01\r",
      "\u001b[K     |█▏                              | 61 kB 11.2 MB/s eta 0:00:01\r",
      "\u001b[K     |█▍                              | 71 kB 10.0 MB/s eta 0:00:01\r",
      "\u001b[K     |█▋                              | 81 kB 10.2 MB/s eta 0:00:01\r",
      "\u001b[K     |█▉                              | 92 kB 8.3 MB/s eta 0:00:01\r",
      "\u001b[K     |██                              | 102 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |██▏                             | 112 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |██▍                             | 122 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |██▋                             | 133 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |██▊                             | 143 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |███                             | 153 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |███▏                            | 163 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |███▍                            | 174 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |███▋                            | 184 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |███▊                            | 194 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |████                            | 204 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |████▏                           | 215 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |████▍                           | 225 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |████▌                           | 235 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |████▊                           | 245 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |█████                           | 256 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |█████▏                          | 266 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |█████▍                          | 276 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |█████▌                          | 286 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |█████▊                          | 296 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |██████                          | 307 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |██████▏                         | 317 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |██████▎                         | 327 kB 8.5 MB/s eta 0:00:01"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      "\u001b[K     |██████▌                         | 337 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |██████▊                         | 348 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |███████                         | 358 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |███████▏                        | 368 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |███████▎                        | 378 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |███████▌                        | 389 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |███████▊                        | 399 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |████████                        | 409 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |████████                        | 419 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |████████▎                       | 430 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |████████▌                       | 440 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |████████▊                       | 450 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████                       | 460 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████                       | 471 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████▎                      | 481 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████▌                      | 491 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████▊                      | 501 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████▉                      | 512 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████                      | 522 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████▎                     | 532 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████▌                     | 542 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████▊                     | 552 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████▉                     | 563 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████                     | 573 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████▎                    | 583 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████▌                    | 593 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████▋                    | 604 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████▉                    | 614 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |████████████                    | 624 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |████████████▎                   | 634 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |████████████▌                   | 645 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |████████████▋                   | 655 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |████████████▉                   | 665 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████                   | 675 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████▎                  | 686 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████▍                  | 696 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████▋                  | 706 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████▉                  | 716 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████                  | 727 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████▎                 | 737 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████▍                 | 747 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████▋                 | 757 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████▉                 | 768 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████                 | 778 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████▏                | 788 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████▍                | 798 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████▋                | 808 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████▉                | 819 kB 8.5 MB/s eta 0:00:01"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      "\u001b[K     |████████████████                | 829 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████▏               | 839 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████▍               | 849 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████▋               | 860 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████▉               | 870 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████               | 880 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████▏              | 890 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████▍              | 901 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████▋              | 911 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████▉              | 921 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████              | 931 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████▏             | 942 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████▍             | 952 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████▋             | 962 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████▊             | 972 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████             | 983 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████▏            | 993 kB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████▍            | 1.0 MB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████▋            | 1.0 MB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████▊            | 1.0 MB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████            | 1.0 MB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████▏           | 1.0 MB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████▍           | 1.1 MB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████▌           | 1.1 MB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████▊           | 1.1 MB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████           | 1.1 MB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████▏          | 1.1 MB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████▍          | 1.1 MB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████▌          | 1.1 MB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████▊          | 1.1 MB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████          | 1.1 MB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████▏         | 1.1 MB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████▎         | 1.2 MB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████▌         | 1.2 MB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████▊         | 1.2 MB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████         | 1.2 MB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████▏        | 1.2 MB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████▎        | 1.2 MB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████▌        | 1.2 MB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████▊        | 1.2 MB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████        | 1.2 MB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████        | 1.2 MB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████▎       | 1.3 MB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████▌       | 1.3 MB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████▊       | 1.3 MB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████████       | 1.3 MB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████████       | 1.3 MB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████████▎      | 1.3 MB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████████▌      | 1.3 MB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████████▊      | 1.3 MB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████████▉      | 1.3 MB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████████      | 1.4 MB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████████▎     | 1.4 MB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████████▌     | 1.4 MB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████████▊     | 1.4 MB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████████▉     | 1.4 MB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████████     | 1.4 MB 8.5 MB/s eta 0:00:01"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      "\u001b[K     |███████████████████████████▎    | 1.4 MB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████████▌    | 1.4 MB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████████▋    | 1.4 MB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████████▉    | 1.4 MB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████████    | 1.5 MB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████████▎   | 1.5 MB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████████▌   | 1.5 MB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████████▋   | 1.5 MB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████████▉   | 1.5 MB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████████████   | 1.5 MB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████████████▎  | 1.5 MB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████████████▍  | 1.5 MB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████████████▋  | 1.5 MB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████████████▉  | 1.5 MB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████████████  | 1.6 MB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████████████▎ | 1.6 MB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████████████▍ | 1.6 MB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████████████▋ | 1.6 MB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████████████▉ | 1.6 MB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████████████ | 1.6 MB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████████████▏| 1.6 MB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████████████▍| 1.6 MB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████████████▋| 1.6 MB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████████████▉| 1.6 MB 8.5 MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████████████| 1.7 MB 8.5 MB/s \r\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[?25hCollecting tqdm<4.50.0,>=4.27\r\n",
      "  Downloading tqdm-4.49.0-py2.py3-none-any.whl (69 kB)\r\n",
      "\u001b[?25l\r",
      "\u001b[K     |████▊                           | 10 kB 37.4 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████▍                      | 20 kB 29.5 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████                  | 30 kB 36.3 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████▉             | 40 kB 24.4 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████▌        | 51 kB 20.3 MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████████▏   | 61 kB 23.0 MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████████████| 69 kB 5.2 MB/s \r\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[?25hCollecting xxhash\r\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Downloading xxhash-2.0.0-cp37-cp37m-manylinux2010_x86_64.whl (243 kB)\r\n",
      "\u001b[?25l\r",
      "\u001b[K     |█▍                              | 10 kB 36.3 MB/s eta 0:00:01\r",
      "\u001b[K     |██▊                             | 20 kB 32.3 MB/s eta 0:00:01\r",
      "\u001b[K     |████                            | 30 kB 38.2 MB/s eta 0:00:01\r",
      "\u001b[K     |█████▍                          | 40 kB 26.4 MB/s eta 0:00:01\r",
      "\u001b[K     |██████▊                         | 51 kB 20.0 MB/s eta 0:00:01\r",
      "\u001b[K     |████████                        | 61 kB 22.7 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████▍                      | 71 kB 20.8 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████▊                     | 81 kB 22.7 MB/s eta 0:00:01\r",
      "\u001b[K     |████████████                    | 92 kB 20.9 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████▍                  | 102 kB 17.6 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████▉                 | 112 kB 17.6 MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████▏               | 122 kB 17.6 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████▌              | 133 kB 17.6 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████▉             | 143 kB 17.6 MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████▏           | 153 kB 17.6 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████▌          | 163 kB 17.6 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████▉         | 174 kB 17.6 MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████▏       | 184 kB 17.6 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████████▌      | 194 kB 17.6 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████████▉     | 204 kB 17.6 MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████████▏   | 215 kB 17.6 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████████████▋  | 225 kB 17.6 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████████████ | 235 kB 17.6 MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████████████| 243 kB 17.6 MB/s \r\n",
      "\u001b[?25hRequirement already satisfied: pyarrow>=0.17.1 in /opt/conda/lib/python3.7/site-packages (from datasets) (1.0.1)\r\n",
      "Requirement already satisfied: importlib-metadata in /opt/conda/lib/python3.7/site-packages (from datasets) (3.4.0)\r\n",
      "Requirement already satisfied: fsspec in /opt/conda/lib/python3.7/site-packages (from datasets) (0.8.5)\r\n",
      "Requirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.7/site-packages (from datasets) (1.19.5)\r\n",
      "Requirement already satisfied: pandas in /opt/conda/lib/python3.7/site-packages (from datasets) (1.1.5)\r\n",
      "Requirement already satisfied: requests>=2.19.0 in /opt/conda/lib/python3.7/site-packages (from datasets) (2.25.1)\r\n",
      "Requirement already satisfied: multiprocess in /opt/conda/lib/python3.7/site-packages (from datasets) (0.70.11.1)\r\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting huggingface-hub<0.1.0\r\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Downloading huggingface_hub-0.0.7-py3-none-any.whl (33 kB)\r\n",
      "Requirement already satisfied: dill in /opt/conda/lib/python3.7/site-packages (from datasets) (0.3.3)\r\n",
      "Requirement already satisfied: filelock in /opt/conda/lib/python3.7/site-packages (from huggingface-hub<0.1.0->datasets) (3.0.12)\r\n",
      "Requirement already satisfied: chardet<5,>=3.0.2 in /opt/conda/lib/python3.7/site-packages (from requests>=2.19.0->datasets) (3.0.4)\r\n",
      "Requirement already satisfied: idna<3,>=2.5 in /opt/conda/lib/python3.7/site-packages (from requests>=2.19.0->datasets) (2.10)\r\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.7/site-packages (from requests>=2.19.0->datasets) (2020.12.5)\r\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.7/site-packages (from requests>=2.19.0->datasets) (1.26.3)\r\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: regex!=2019.12.17 in /opt/conda/lib/python3.7/site-packages (from transformers) (2020.11.13)\r\n",
      "Requirement already satisfied: packaging in /opt/conda/lib/python3.7/site-packages (from transformers) (20.9)\r\n",
      "Requirement already satisfied: tokenizers<0.11,>=0.10.1 in /opt/conda/lib/python3.7/site-packages (from transformers) (0.10.1)\r\n",
      "Requirement already satisfied: sacremoses in /opt/conda/lib/python3.7/site-packages (from transformers) (0.0.43)\r\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: typing-extensions>=3.6.4 in /opt/conda/lib/python3.7/site-packages (from importlib-metadata->datasets) (3.7.4.3)\r\n",
      "Requirement already satisfied: zipp>=0.5 in /opt/conda/lib/python3.7/site-packages (from importlib-metadata->datasets) (3.4.0)\r\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in /opt/conda/lib/python3.7/site-packages (from packaging->transformers) (2.4.7)\r\n",
      "Requirement already satisfied: python-dateutil>=2.7.3 in /opt/conda/lib/python3.7/site-packages (from pandas->datasets) (2.8.1)\r\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: pytz>=2017.2 in /opt/conda/lib/python3.7/site-packages (from pandas->datasets) (2021.1)\r\n",
      "Requirement already satisfied: six>=1.5 in /opt/conda/lib/python3.7/site-packages (from python-dateutil>=2.7.3->pandas->datasets) (1.15.0)\r\n",
      "Requirement already satisfied: joblib in /opt/conda/lib/python3.7/site-packages (from sacremoses->transformers) (1.0.1)\r\n",
      "Requirement already satisfied: click in /opt/conda/lib/python3.7/site-packages (from sacremoses->transformers) (7.1.2)\r\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Installing collected packages: tqdm, xxhash, huggingface-hub, nlp, datasets\r\n",
      "  Attempting uninstall: tqdm\r\n",
      "    Found existing installation: tqdm 4.56.2\r\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    Uninstalling tqdm-4.56.2:\r\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "      Successfully uninstalled tqdm-4.56.2\r\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Successfully installed datasets-1.5.0 huggingface-hub-0.0.7 nlp-0.4.0 tqdm-4.49.0 xxhash-2.0.0\r\n"
     ]
    }
   ],
   "source": [
    "!pip install transformers datasets nlp"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "distinguished-halifax",
   "metadata": {
    "id": "rEJBSTyZIrIb",
    "papermill": {
     "duration": 0.039533,
     "end_time": "2021-03-26T11:44:58.651815",
     "exception": false,
     "start_time": "2021-03-26T11:44:58.612282",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Fine-tuning a model on a text classification task"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "incoming-partner",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-03-26T11:44:58.733831Z",
     "iopub.status.busy": "2021-03-26T11:44:58.733269Z",
     "iopub.status.idle": "2021-03-26T11:44:58.737099Z",
     "shell.execute_reply": "2021-03-26T11:44:58.736513Z"
    },
    "id": "zVvslsfMIrIh",
    "papermill": {
     "duration": 0.046341,
     "end_time": "2021-03-26T11:44:58.737213",
     "exception": false,
     "start_time": "2021-03-26T11:44:58.690872",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "task = \"test\"\n",
    "model_checkpoint = \"xlm-roberta-large\"\n",
    "batch_size = 3"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "grave-guyana",
   "metadata": {
    "id": "whPRbBNbIrIl",
    "papermill": {
     "duration": 0.038884,
     "end_time": "2021-03-26T11:44:58.814989",
     "exception": false,
     "start_time": "2021-03-26T11:44:58.776105",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Loading the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "instructional-spring",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-03-26T11:44:58.897684Z",
     "iopub.status.busy": "2021-03-26T11:44:58.896521Z",
     "iopub.status.idle": "2021-03-26T11:44:59.834530Z",
     "shell.execute_reply": "2021-03-26T11:44:59.833430Z"
    },
    "id": "IreSlFmlIrIm",
    "papermill": {
     "duration": 0.980806,
     "end_time": "2021-03-26T11:44:59.834716",
     "exception": false,
     "start_time": "2021-03-26T11:44:58.853910",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import string\n",
    "import re\n",
    "import os"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "generic-virginia",
   "metadata": {
    "id": "WHUmphG3IrI3",
    "papermill": {
     "duration": 0.039115,
     "end_time": "2021-03-26T11:44:59.913324",
     "exception": false,
     "start_time": "2021-03-26T11:44:59.874209",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "To get a sense of what the data looks like, the following function will show some examples picked randomly in the dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "alive-colors",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-03-26T11:44:59.998696Z",
     "iopub.status.busy": "2021-03-26T11:44:59.998132Z",
     "iopub.status.idle": "2021-03-26T11:45:01.741987Z",
     "shell.execute_reply": "2021-03-26T11:45:01.741182Z"
    },
    "id": "WjiMHv0fgFkl",
    "outputId": "3acd51ea-1ba2-4056-d543-48aae68e7053",
    "papermill": {
     "duration": 1.789585,
     "end_time": "2021-03-26T11:45:01.742116",
     "exception": false,
     "start_time": "2021-03-26T11:44:59.952531",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence1</th>\n",
       "      <th>sentence2</th>\n",
       "      <th>images</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>idx</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Міністри Зеленського лишають мільйони пацієнті...</td>\n",
       "      <td>Понад півтора мільйони пацієнтів із тяжкими ді...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>В АМКУ звертаються щодо незаконної державної д...</td>\n",
       "      <td>За 15 років державна підтримка виробництва ене...</td>\n",
       "      <td>fd92a3dd-1109-49d8-8f5f-eeed72da22ef.png</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>У Польщі виявили майже 16 тисяч нових хворих н...</td>\n",
       "      <td>У Польщі виявили 15 698 нових хворих на корона...</td>\n",
       "      <td>f4a284d0-9bb7-4910-8a62-a7bfd0ec29b6.png</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Закриття міжбанку: гривня трохи ослабла</td>\n",
       "      <td>2 жовтня котирування гривні до долара на закри...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>У Москві помер український шахіст із бази \"Мир...</td>\n",
       "      <td>У п'ятницю низка російських ЗМІ повідомили про...</td>\n",
       "      <td>2cbb2339-6dd2-4000-a53c-225ec7aad892.png,d5233...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32366</th>\n",
       "      <td>\"Укрзалізниця\" просить Раду у бюджеті-2021 зак...</td>\n",
       "      <td>\"Укрзалізниця\" закликає парламент закласти у д...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32367</th>\n",
       "      <td>Сніг та 20-градусні морози протримаються до ви...</td>\n",
       "      <td>У вівторок, 16 лютого, в Україні буде без істо...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32368</th>\n",
       "      <td>Україна хоче оподаткувати Google, Facebook та ...</td>\n",
       "      <td>Гетманцев розповів, як Україна оподатковуватим...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32369</th>\n",
       "      <td>Лижі: як підготувати дитину до відпустки та че...</td>\n",
       "      <td>Якщо батькам подобається кататися на лижах – ц...</td>\n",
       "      <td>7770c665-825b-433a-b63e-f6e0ebe7f4cb.png,66866...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32371</th>\n",
       "      <td>Щеплення від COVID-19 позачергово: як українці...</td>\n",
       "      <td>Соціологи дізналися думку українців про вакцин...</td>\n",
       "      <td>272d0229-c589-4aca-aa0a-163e1b652461.png,17c8b...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>16177 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               sentence1  \\\n",
       "idx                                                        \n",
       "1      Міністри Зеленського лишають мільйони пацієнті...   \n",
       "2      В АМКУ звертаються щодо незаконної державної д...   \n",
       "5      У Польщі виявили майже 16 тисяч нових хворих н...   \n",
       "9                Закриття міжбанку: гривня трохи ослабла   \n",
       "10     У Москві помер український шахіст із бази \"Мир...   \n",
       "...                                                  ...   \n",
       "32366  \"Укрзалізниця\" просить Раду у бюджеті-2021 зак...   \n",
       "32367  Сніг та 20-градусні морози протримаються до ви...   \n",
       "32368  Україна хоче оподаткувати Google, Facebook та ...   \n",
       "32369  Лижі: як підготувати дитину до відпустки та че...   \n",
       "32371  Щеплення від COVID-19 позачергово: як українці...   \n",
       "\n",
       "                                               sentence2  \\\n",
       "idx                                                        \n",
       "1      Понад півтора мільйони пацієнтів із тяжкими ді...   \n",
       "2      За 15 років державна підтримка виробництва ене...   \n",
       "5      У Польщі виявили 15 698 нових хворих на корона...   \n",
       "9      2 жовтня котирування гривні до долара на закри...   \n",
       "10     У п'ятницю низка російських ЗМІ повідомили про...   \n",
       "...                                                  ...   \n",
       "32366  \"Укрзалізниця\" закликає парламент закласти у д...   \n",
       "32367  У вівторок, 16 лютого, в Україні буде без істо...   \n",
       "32368  Гетманцев розповів, як Україна оподатковуватим...   \n",
       "32369  Якщо батькам подобається кататися на лижах – ц...   \n",
       "32371  Соціологи дізналися думку українців про вакцин...   \n",
       "\n",
       "                                                  images  label  \n",
       "idx                                                              \n",
       "1                                                    NaN      5  \n",
       "2               fd92a3dd-1109-49d8-8f5f-eeed72da22ef.png      6  \n",
       "5               f4a284d0-9bb7-4910-8a62-a7bfd0ec29b6.png      2  \n",
       "9                                                    NaN      3  \n",
       "10     2cbb2339-6dd2-4000-a53c-225ec7aad892.png,d5233...      0  \n",
       "...                                                  ...    ...  \n",
       "32366                                                NaN      3  \n",
       "32367                                                NaN      2  \n",
       "32368                                                NaN      6  \n",
       "32369  7770c665-825b-433a-b63e-f6e0ebe7f4cb.png,66866...      5  \n",
       "32371  272d0229-c589-4aca-aa0a-163e1b652461.png,17c8b...      1  \n",
       "\n",
       "[16177 rows x 4 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv('../input/newsclass01/train.csv', index_col=0)\n",
    "data = data.rename({'title': 'sentence1', 'text': 'sentence2', 'source': 'label'}, axis=1)\n",
    "data.index = data.index.rename('idx')\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "boxed-republic",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-03-26T11:45:01.830580Z",
     "iopub.status.busy": "2021-03-26T11:45:01.829789Z",
     "iopub.status.idle": "2021-03-26T11:45:01.838415Z",
     "shell.execute_reply": "2021-03-26T11:45:01.838000Z"
    },
    "id": "m4fsJF4_5566",
    "outputId": "a6518a8b-b2b1-4b4d-baa3-cd3662f9ebdd",
    "papermill": {
     "duration": 0.056806,
     "end_time": "2021-03-26T11:45:01.838528",
     "exception": false,
     "start_time": "2021-03-26T11:45:01.781722",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 16177 entries, 1 to 32371\n",
      "Data columns (total 4 columns):\n",
      " #   Column     Non-Null Count  Dtype \n",
      "---  ------     --------------  ----- \n",
      " 0   sentence1  16177 non-null  object\n",
      " 1   sentence2  16177 non-null  object\n",
      " 2   images     7223 non-null   object\n",
      " 3   label      16177 non-null  int64 \n",
      "dtypes: int64(1), object(3)\n",
      "memory usage: 631.9+ KB\n"
     ]
    }
   ],
   "source": [
    "data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "boolean-signal",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-03-26T11:45:01.928091Z",
     "iopub.status.busy": "2021-03-26T11:45:01.927556Z",
     "iopub.status.idle": "2021-03-26T11:45:03.619834Z",
     "shell.execute_reply": "2021-03-26T11:45:03.620409Z"
    },
    "id": "6R1n-2DMTD8J",
    "outputId": "03f7830b-bd0f-4961-fa6a-f3214910a26e",
    "papermill": {
     "duration": 1.741578,
     "end_time": "2021-03-26T11:45:03.620601",
     "exception": false,
     "start_time": "2021-03-26T11:45:01.879023",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence1</th>\n",
       "      <th>sentence2</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>idx</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Падіння економіки в Україні не є критичним - ЄБРР</td>\n",
       "      <td>Падіння економічного рівня в Україні не є крит...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Геймерам віддають безкоштовно шутер Rage 2 та ...</td>\n",
       "      <td>Шутер Rage 2 віддають безкоштовно в EGS / фото...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Записки из Луганска: привычка быть виноватыми ...</td>\n",
       "      <td>\"Ты печешь?\" – спрашивает у меня подруга, пере...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Індекс Dow Jones вперше перевищив 30 тисяч пун...</td>\n",
       "      <td>На тлі новин щодо передачі влади у США та вакц...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Трамп і Меркель обговорили підтримку економічн...</td>\n",
       "      <td>Президент США Дональд Трамп і канцлерка Німечч...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32362</th>\n",
       "      <td>США хочуть закупити ще 100 млн доз вакцини про...</td>\n",
       "      <td>Президент США Джо Байден доручив Міністерству ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32363</th>\n",
       "      <td>Вчителька вдарила школярку по голові і відпові...</td>\n",
       "      <td>У Первомайському ліцеї №7 Харківської області ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32364</th>\n",
       "      <td>\"Не розслабляємося\": українців просять святкув...</td>\n",
       "      <td>Президент Володимир Зеленський попросив україн...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32370</th>\n",
       "      <td>Флот, торгівля та крок до безвізу: про що домо...</td>\n",
       "      <td>Експорт з України до Великої Британії не постр...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32372</th>\n",
       "      <td>Євродепутати про Росію: досить ілюзій, треба ж...</td>\n",
       "      <td>Жозепу Боррелю сильно дісталося за його візит ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>16183 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               sentence1  \\\n",
       "idx                                                        \n",
       "0      Падіння економіки в Україні не є критичним - ЄБРР   \n",
       "3      Геймерам віддають безкоштовно шутер Rage 2 та ...   \n",
       "4      Записки из Луганска: привычка быть виноватыми ...   \n",
       "6      Індекс Dow Jones вперше перевищив 30 тисяч пун...   \n",
       "7      Трамп і Меркель обговорили підтримку економічн...   \n",
       "...                                                  ...   \n",
       "32362  США хочуть закупити ще 100 млн доз вакцини про...   \n",
       "32363  Вчителька вдарила школярку по голові і відпові...   \n",
       "32364  \"Не розслабляємося\": українців просять святкув...   \n",
       "32370  Флот, торгівля та крок до безвізу: про що домо...   \n",
       "32372  Євродепутати про Росію: досить ілюзій, треба ж...   \n",
       "\n",
       "                                               sentence2  \n",
       "idx                                                       \n",
       "0      Падіння економічного рівня в Україні не є крит...  \n",
       "3      Шутер Rage 2 віддають безкоштовно в EGS / фото...  \n",
       "4      \"Ты печешь?\" – спрашивает у меня подруга, пере...  \n",
       "6      На тлі новин щодо передачі влади у США та вакц...  \n",
       "7      Президент США Дональд Трамп і канцлерка Німечч...  \n",
       "...                                                  ...  \n",
       "32362  Президент США Джо Байден доручив Міністерству ...  \n",
       "32363  У Первомайському ліцеї №7 Харківської області ...  \n",
       "32364  Президент Володимир Зеленський попросив україн...  \n",
       "32370  Експорт з України до Великої Британії не постр...  \n",
       "32372  Жозепу Боррелю сильно дісталося за його візит ...  \n",
       "\n",
       "[16183 rows x 2 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_data = pd.read_csv('../input/newsclass01/test_without_target.csv', index_col=0)\n",
    "test_data = test_data.drop(columns=['images'], axis=1).rename({'title': 'sentence1', 'text': 'sentence2', 'source': 'label'}, axis=1)\n",
    "test_data.index = test_data.index.rename('idx')\n",
    "test_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "three-bleeding",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-03-26T11:45:03.709460Z",
     "iopub.status.busy": "2021-03-26T11:45:03.708954Z",
     "iopub.status.idle": "2021-03-26T11:45:03.712674Z",
     "shell.execute_reply": "2021-03-26T11:45:03.712204Z"
    },
    "id": "rXzFw8DI1HJO",
    "papermill": {
     "duration": 0.050984,
     "end_time": "2021-03-26T11:45:03.712840",
     "exception": false,
     "start_time": "2021-03-26T11:45:03.661856",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "X, y = data.drop(columns=['label', 'images']), data['label']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "spread-omega",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-03-26T11:45:03.800535Z",
     "iopub.status.busy": "2021-03-26T11:45:03.799861Z",
     "iopub.status.idle": "2021-03-26T11:45:03.805084Z",
     "shell.execute_reply": "2021-03-26T11:45:03.804640Z"
    },
    "id": "n4ID127s1F7g",
    "papermill": {
     "duration": 0.051223,
     "end_time": "2021-03-26T11:45:03.805189",
     "exception": false,
     "start_time": "2021-03-26T11:45:03.753966",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "parliamentary-allergy",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-03-26T11:45:03.892459Z",
     "iopub.status.busy": "2021-03-26T11:45:03.891643Z",
     "iopub.status.idle": "2021-03-26T11:45:05.330828Z",
     "shell.execute_reply": "2021-03-26T11:45:05.331465Z"
    },
    "id": "efM6a9dTjZJQ",
    "outputId": "278309a6-8a07-4703-d5c8-58a04b74a48b",
    "papermill": {
     "duration": 1.485919,
     "end_time": "2021-03-26T11:45:05.331632",
     "exception": false,
     "start_time": "2021-03-26T11:45:03.845713",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['sentence1', 'sentence2', 'label', 'idx'],\n",
       "        num_rows: 16177\n",
       "    })\n",
       "    validation: Dataset({\n",
       "        features: ['sentence1', 'sentence2', 'label', 'idx'],\n",
       "        num_rows: 5339\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['sentence1', 'sentence2', 'idx'],\n",
       "        num_rows: 16183\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import datasets\n",
    "from datasets import Dataset, DatasetDict, load_metric\n",
    "\n",
    "dataset = datasets.dataset_dict.DatasetDict({'train': Dataset.from_pandas(pd.concat([X, y], axis=1), split='train'), 'validation': Dataset.from_pandas(pd.concat([X_test, y_test], axis=1), split='validation'), 'test': Dataset.from_pandas(test_data, split='test')})\n",
    "dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "after-tennis",
   "metadata": {
    "id": "jAWdqcUBIrJC",
    "papermill": {
     "duration": 0.042155,
     "end_time": "2021-03-26T11:45:05.416986",
     "exception": false,
     "start_time": "2021-03-26T11:45:05.374831",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "You can call its `compute` method with your predictions and labels directly and it will return a dictionary with the metric(s) value:"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "experimental-interaction",
   "metadata": {
    "id": "n9qywopnIrJH",
    "papermill": {
     "duration": 0.041132,
     "end_time": "2021-03-26T11:45:05.500938",
     "exception": false,
     "start_time": "2021-03-26T11:45:05.459806",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Preprocessing the data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "coated-music",
   "metadata": {
    "id": "YVx71GdAIrJH",
    "papermill": {
     "duration": 0.041506,
     "end_time": "2021-03-26T11:45:05.583544",
     "exception": false,
     "start_time": "2021-03-26T11:45:05.542038",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "Before we can feed those texts to our model, we need to preprocess them. This is done by a 🤗 Transformers `Tokenizer` which will (as the name indicates) tokenize the inputs (including converting the tokens to their corresponding IDs in the pretrained vocabulary) and put it in a format the model expects, as well as generate the other inputs that model requires.\n",
    "\n",
    "To do all of this, we instantiate our tokenizer with the `AutoTokenizer.from_pretrained` method, which will ensure:\n",
    "\n",
    "- we get a tokenizer that corresponds to the model architecture we want to use,\n",
    "- we download the vocabulary used when pretraining this specific checkpoint.\n",
    "\n",
    "That vocabulary will be cached, so it's not downloaded again the next time we run the cell."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "preliminary-drain",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-03-26T11:45:05.670526Z",
     "iopub.status.busy": "2021-03-26T11:45:05.669915Z",
     "iopub.status.idle": "2021-03-26T11:45:09.051958Z",
     "shell.execute_reply": "2021-03-26T11:45:09.051492Z"
    },
    "id": "eXNLu_-nIrJI",
    "outputId": "3a610d0d-3f34-48ce-a789-56c599693b53",
    "papermill": {
     "duration": 3.427568,
     "end_time": "2021-03-26T11:45:09.052088",
     "exception": false,
     "start_time": "2021-03-26T11:45:05.624520",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "584cd3513ab84b59975780acf22c2ae7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=513.0, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "80808b29f56f4025828199c4b7e1cfd8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=5069051.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b991d6a2f949402e9cbe3879fb57cbd1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=9096718.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoTokenizer\n",
    "    \n",
    "tokenizer = AutoTokenizer.from_pretrained(model_checkpoint, use_fast=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "under-rebound",
   "metadata": {
    "id": "Vl6IidfdIrJK",
    "papermill": {
     "duration": 0.044644,
     "end_time": "2021-03-26T11:45:09.141341",
     "exception": false,
     "start_time": "2021-03-26T11:45:09.096697",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "We pass along `use_fast=True` to the call above to use one of the fast tokenizers (backed by Rust) from the 🤗 Tokenizers library. Those fast tokenizers are available for almost all models, but if you got an error with the previous call, remove that argument."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "written-editing",
   "metadata": {
    "id": "rowT4iCLIrJK",
    "papermill": {
     "duration": 0.043165,
     "end_time": "2021-03-26T11:45:09.227926",
     "exception": false,
     "start_time": "2021-03-26T11:45:09.184761",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "You can directly call this tokenizer on one sentence or a pair of sentences:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "peripheral-prairie",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-03-26T11:45:09.318429Z",
     "iopub.status.busy": "2021-03-26T11:45:09.317807Z",
     "iopub.status.idle": "2021-03-26T11:45:09.324776Z",
     "shell.execute_reply": "2021-03-26T11:45:09.324229Z"
    },
    "id": "a5hBlsrHIrJL",
    "outputId": "873371ae-cd33-4231-ad16-99a0bd1c3dde",
    "papermill": {
     "duration": 0.053947,
     "end_time": "2021-03-26T11:45:09.324898",
     "exception": false,
     "start_time": "2021-03-26T11:45:09.270951",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'input_ids': [0, 901, 42554, 49546, 33891, 147308, 89, 179387, 4459, 11849, 48878, 827, 2, 2, 589, 894, 476, 489, 13484, 12315, 105, 47279, 177640, 89, 591, 58070, 25863, 5, 2], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer(\"Понад півтора мільйони пацієнтів із тяжкими\", \"Сніг та 20-градусні морози протримаються.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "employed-breath",
   "metadata": {
    "id": "qo_0B1M2IrJM",
    "papermill": {
     "duration": 0.043192,
     "end_time": "2021-03-26T11:45:09.411404",
     "exception": false,
     "start_time": "2021-03-26T11:45:09.368212",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "Depending on the model you selected, you will see different keys in the dictionary returned by the cell above. They don't matter much for what we're doing here (just know they are required by the model we will instantiate later), you can learn more about them in [this tutorial](https://huggingface.co/transformers/preprocessing.html) if you're interested.\n",
    "\n",
    "To preprocess our dataset, we will thus need the names of the columns containing the sentence(s). The following dictionary keeps track of the correspondence task to column names:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "canadian-detective",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-03-26T11:45:09.502366Z",
     "iopub.status.busy": "2021-03-26T11:45:09.501733Z",
     "iopub.status.idle": "2021-03-26T11:45:09.504580Z",
     "shell.execute_reply": "2021-03-26T11:45:09.504111Z"
    },
    "id": "fyGdtK9oIrJM",
    "papermill": {
     "duration": 0.049657,
     "end_time": "2021-03-26T11:45:09.504684",
     "exception": false,
     "start_time": "2021-03-26T11:45:09.455027",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "task_to_keys = {\n",
    "    \"test\": (\"sentence1\", \"sentence2\"),\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "overhead-purple",
   "metadata": {
    "id": "xbqtC4MrIrJO",
    "papermill": {
     "duration": 0.043509,
     "end_time": "2021-03-26T11:45:09.591725",
     "exception": false,
     "start_time": "2021-03-26T11:45:09.548216",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "We can double check it does work on our current dataset:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "parliamentary-pilot",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-03-26T11:45:09.712816Z",
     "iopub.status.busy": "2021-03-26T11:45:09.683238Z",
     "iopub.status.idle": "2021-03-26T11:45:09.976924Z",
     "shell.execute_reply": "2021-03-26T11:45:09.976311Z"
    },
    "id": "19GG646uIrJO",
    "outputId": "87539946-b1ad-4c24-b4d0-593c8444bc8b",
    "papermill": {
     "duration": 0.341777,
     "end_time": "2021-03-26T11:45:09.977090",
     "exception": false,
     "start_time": "2021-03-26T11:45:09.635313",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Text: Міністри Зеленського лишають мільйони пацієнтів без ліків\n",
      "Title: Понад півтора мільйони пацієнтів із тяжкими діагнозами ризикують залишитись без ліків у 2020 році, бо МОЗ блокує закупівлі.\n",
      "Заблоковані рекордні майже 10 мільярдів гривень у бюджеті на закупівлю ліків та медвиробів.\n",
      "Ключові документи не підписувалися 4 місяці.\n",
      "З голими руками і неадекватними заявами міністра Ємця: як Україна зустрічає коронавірус\n",
      "І це в час, коли у світі пандемія коронавірусу, дефіцит багатьох ліків, країни стають в чергу на постачання і рахунок іде не на дні, а на години.\n",
      "Натомість Україна в цю чергу ще навіть не стала.\n",
      "З листопада 2019 року був готовий документ – перелік ліків і медвиробів для закупівель – і він весь цей час валявся для підпису на столі то в одного, то в іншого міністра.\n",
      "Зараз він досі не ухвалений урядом.\n",
      "Більше того, в міністерстві досі не призначений заступник міністра, який опікуватиметься закупівлями, не працює спеціальна робоча група, без якої закупівлі стоять, – в МОЗ не зроблено нічого для того, щоб пацієнти цього року отримали ліки. Це означає, що ми не лише не готові лікувати хворих на коронавірус.\n",
      "Зовсім скоро на додачу до цього в нас не буде чим лікувати мільйони пацієнтів по всій країні: дітей і дорослих з онкологією, пацієнтів із інфарктами, інсультами, туберкульозом, гепатитами, ВІЛ і рідкісними захворюваннями. Пацієнти навіть розпочали флешмоб \"Лікування важких хвороб не можна поставити на STOP\".\n",
      "Вимагають негайно розблокувати закупівлі, інакше тяжкохворі пацієнти ризикують перервати лікування і померти.\n",
      "Чому ситуація критична?\n",
      "Закупівельний процес досить тривалий.\n",
      "МОЗ мав розпочати його ще минулого року, щоб українські лікарні вже найближчим часом могли отримати необхідні препарати і медичні вироби. В медзакладах і регіонах певний запас ліків є завжди, але хто і скільки зможе протягнути – велике питання.\n",
      "Наприклад, в Охматдиті можуть закінчитися критично важливі препарати для онкохворих дітей.\n",
      "120 тисяч ВІЛ-позитивних пацієнтів можуть опинитися без ліків вже за кілька місяців.\n",
      "Зараз в країні уже закінчуються ліки для них, є регіони, які змушені переводити пацієнтів на більш дороговартісне лікування вже сьогодні.\n",
      "Всі чекають на державні поставки, яких нема через бездіяльність МОЗ.\n",
      "Під загрозою опиняються і всі інші тяжкохворі пацієнти, і вся реформа охорони здоров’я загалом.\n",
      "Адже щоб лікарі могли могли безоплатно лікувати пацієнтів за програмою медичних гарантій, у них мають бути препарати, закуплені за державний кошт.\n",
      "До чого тут офіс президента?\n",
      "Відповідальність за провал несе насамперед офіс президента. Саме там весь цей час обирають неспроможних міністрів.\n",
      "Ці кадрові помилки коштують українському народові надто дорого – людських життів. Змінні міністри ніяк не могли поділити сфери впливу і встановити свій контроль над закупівлями спочатку через \"радників\", а потім смотрящих.\n",
      "Екс-міністерка Зоряна Скалецька просто не підписувала потрібних документів місяцями і оточувала себе радниками із фармбізнесу. А Ілля Ємець хотів поставити своїх \"смотрящих\" в незалежну закупівельну агенцію.\n",
      "Через це блокував будь-яку роботу Державного підприємства \"Медичні закупівлі України\", погрожував керівнику та вимагав прилаштувати свою \"довірену особу\".\n",
      "Новий міністр Максим Степанов на своїй посаді уже майже два тижні, і він досі не зробив того, що повинен був зробити в перші дні – розблокувати закупівлі.\n",
      "Якщо цього не зробити негайно, нас може чекати гуманітарна катастрофа і тисячі смертей на додачу до втрат від коронавірусу та війни.\n",
      "Що треба робити негайно\n",
      "💊 Міністр має негайно призначити відповідального заступника міністра, який буде щоденно керувати процесом закупівель, а не намагатися на них підзаробити.\n",
      "Без такої людини вся операційна робота в міністерстві стоїть. 💊 В МОЗ має негайно запрацювати спеціальна постійна робоча група з питань закупівель, покликана вирішувати всі поточні питання, затверджувати поставки і документи.\n",
      "Вона не збиралася вже більше місяця, закуплені ліки заблоковано і вони не можуть потрапити в українські лікарні. 💊 Кабмін має в найкоротші терміни ухвалити переліки ліків та медвиробів на закупівлю в 2020 році, а МОЗ – обрати міжнародні організації для закупівель та залучити ДП \"Медичні закупівлі України\".\n",
      "На вулиці квітень, закупівлі 20-го року ще навіть не почалися, а вони займуть багато місяців! 💊 МОЗ повинен терміново сформувати та подати в Мінфін бюджетні паспорти на закупівельні програми, а Мінфін – їх затвердити.\n",
      "💊 Документи, що стосуються закупівель життєво важливих препаратів, мають надаватися на підпис відповідальному заступнику та міністру якомога скоріше, а необхідні постанови Кабмін має ухвалювати першочергово.\n",
      "Ольга Стефанишина, народна депутатка від партії \"Голос\", ексзаступниця міністра охорони здоров’я, \n"
     ]
    }
   ],
   "source": [
    "sentence1_key, sentence2_key = task_to_keys[task]\n",
    "if sentence2_key is None:\n",
    "    print(f\"Text: {dataset['train'][sentence1_key][0]}\")\n",
    "else:\n",
    "    print(f\"Text: {dataset['train'][sentence1_key][0]}\")\n",
    "    print(f\"Title: {dataset['train'][sentence2_key][0]}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "vocal-northeast",
   "metadata": {
    "id": "2C0hcmp9IrJQ",
    "papermill": {
     "duration": 0.044663,
     "end_time": "2021-03-26T11:45:10.066839",
     "exception": false,
     "start_time": "2021-03-26T11:45:10.022176",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "We can them write the function that will preprocess our samples. We just feed them to the `tokenizer` with the argument `truncation=True`. This will ensure that an input longer that what the model selected can handle will be truncated to the maximum length accepted by the model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "forty-leeds",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-03-26T11:45:10.161092Z",
     "iopub.status.busy": "2021-03-26T11:45:10.160436Z",
     "iopub.status.idle": "2021-03-26T11:45:10.163435Z",
     "shell.execute_reply": "2021-03-26T11:45:10.163039Z"
    },
    "id": "vc0BSBLIIrJQ",
    "papermill": {
     "duration": 0.051493,
     "end_time": "2021-03-26T11:45:10.163543",
     "exception": false,
     "start_time": "2021-03-26T11:45:10.112050",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def preprocess_function(examples):\n",
    "    if sentence2_key is None:\n",
    "        return tokenizer(examples[sentence1_key], truncation=True)\n",
    "    return tokenizer(examples[sentence1_key], examples[sentence2_key], truncation=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "abandoned-syndrome",
   "metadata": {
    "id": "0lm8ozrJIrJR",
    "papermill": {
     "duration": 0.070744,
     "end_time": "2021-03-26T11:45:10.299816",
     "exception": false,
     "start_time": "2021-03-26T11:45:10.229072",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "This function works with one or several examples. In the case of several examples, the tokenizer will return a list of lists for each key:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "failing-footage",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-03-26T11:45:10.401286Z",
     "iopub.status.busy": "2021-03-26T11:45:10.400526Z",
     "iopub.status.idle": "2021-03-26T11:45:10.415350Z",
     "shell.execute_reply": "2021-03-26T11:45:10.415781Z"
    },
    "id": "-b70jh26IrJS",
    "outputId": "b758fcad-c42d-42be-8e63-1766c2c5c6a4",
    "papermill": {
     "duration": 0.071586,
     "end_time": "2021-03-26T11:45:10.415915",
     "exception": false,
     "start_time": "2021-03-26T11:45:10.344329",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'input_ids': [[0, 135867, 89, 111675, 11188, 53737, 14632, 147308, 89, 179387, 1093, 11080, 10654, 2, 2, 901, 42554, 49546, 33891, 147308, 89, 179387, 4459, 11849, 48878, 827, 96417, 476, 122387, 1281, 50641, 14815, 177019, 4401, 1093, 11080, 10654, 84, 11075, 18122, 4, 3577, 1435, 77333, 21059, 10403, 240978, 5, 829, 76591, 44008, 70428, 894, 52503, 209, 166065, 790, 57321, 84, 12358, 260, 29, 40378, 790, 5235, 11080, 10654, 489, 17792, 2827, 8492, 790, 5, 208856, 260, 30560, 77, 99338, 122183, 201, 156440, 5, 1522, 17510, 5593, 94520, 189, 234238, 10606, 22263, 1281, 6, 61037, 27733, 130, 6218, 12, 1443, 28105, 71866, 13205, 211247, 59, 4335, 20017, 2144, 2157, 49, 2707, 4, 8790, 84, 60755, 36571, 30462, 3660, 211247, 6207, 97428, 2871, 4, 1757, 21305, 86463, 90165, 11080, 10654, 4, 51550, 6716, 14632, 49, 110824, 29, 224965, 189, 77579, 66280, 77, 29, 58089, 4, 252, 29, 3563, 5, 672, 328, 130122, 28105, 49, 64517, 110824, 1091, 24027, 77, 18637, 5, 1522, 56749, 3640, 4509, 14571, 67759, 312, 13053, 46, 146895, 11080, 10654, 189, 17792, 2827, 8492, 790, 518, 40378, 790, 19108, 46, 189, 7315, 37474, 25789, 2707, 9887, 2033, 30858, 518, 99338, 105, 29, 17641, 260, 690, 49, 14806, 4, 690, 49, 96363, 6, 61037, 5, 87771, 7315, 168799, 77, 118840, 41690, 109187, 419, 5, 180230, 2574, 4, 49, 234237, 260, 168799, 77, 440, 168681, 62911, 6, 61037, 4, 8098, 8534, 260, 131369, 888, 2454, 40378, 790, 2033, 827, 4, 77, 77482, 135, 118197, 59, 69410, 59, 15610, 4, 1093, 61912, 240978, 6, 135977, 4, 46, 49, 1435, 77333, 77, 242779, 87695, 518, 2574, 4, 8734, 146411, 89, 9950, 4509, 121565, 11080, 751, 5, 7633, 129555, 4, 899, 1450, 77, 12934, 77, 172925, 47809, 18281, 180507, 29, 211247, 59, 4335, 20017, 5, 1522, 559, 50666, 18777, 29, 255, 131549, 255, 9950, 49, 1611, 77, 6055, 50342, 47809, 18281, 147308, 89, 179387, 129, 6, 224707, 116968, 12, 18047, 189, 173382, 210, 203677, 33230, 4, 179387, 4459, 77623, 3298, 33421, 1281, 4, 189, 42678, 1281, 4, 222257, 1041, 8899, 419, 4, 221865, 1281, 4, 417, 145371, 189, 6, 80073, 2087, 68160, 89, 87540, 827, 5, 3512, 2475, 217812, 89, 24027, 2724, 57870, 7310, 185929, 130, 6737, 44, 124749, 77695, 6, 12149, 23122, 119750, 77, 4891, 129, 71063, 29, 159, 70675, 740, 2857, 28306, 14632, 225118, 2724, 76591, 6326, 240978, 4, 162959, 103, 208511, 244, 19159, 260, 146411, 89, 50641, 14815, 1741, 800, 18281, 43506, 189, 112027, 640, 5, 127608, 132662, 20144, 59293, 32, 829, 11355, 790, 103, 49022, 15258, 61503, 3146, 7533, 312, 5, 1435, 77333, 111642, 2724, 2042, 66818, 4350, 1091, 126516, 4509, 4, 8734, 110732, 168298, 11069, 134758, 1595, 76871, 26196, 74307, 167254, 171271, 189, 128362, 260, 72810, 89, 5, 417, 17792, 1323, 13119, 1214, 189, 123742, 1214, 68271, 983, 54883, 11080, 10654, 3273, 41553, 4, 3573, 11501, 189, 131612, 168293, 591, 240836, 46, 93626, 19921, 5, 110248, 4, 49, 127707, 10235, 1752, 3386, 18489, 61, 22762, 1891, 17470, 235157, 188657, 171271, 518, 1266, 1417, 244, 19159, 1009, 18047, 5, 5390, 46333, 417, 145371, 9, 21741, 115871, 244, 179387, 18489, 407, 54925, 32801, 2], [0, 417, 6, 33968, 32653, 37681, 205, 25863, 12621, 119880, 2401, 46931, 62831, 44, 126636, 1595, 58, 208328, 130, 20, 114510, 2, 2, 829, 423, 12320, 29209, 212, 213589, 65439, 154913, 4, 72810, 1200, 4048, 210, 102074, 1315, 142727, 4, 1741, 8830, 95176, 29, 77, 64165, 312, 419, 1394, 11849, 11054, 518, 2513, 5, 2443, 2157, 64058, 25169, 44, 160677, 260, 69245, 10447, 740, 829, 154756, 4, 899, 518, 170080, 77, 201619, 135997, 20160, 163431, 5401, 94, 126636, 1867, 167, 61980, 17604, 4, 94, 124749, 680, 14448, 6574, 26197, 167, 2336, 27324, 43914, 134522, 255, 73098, 21408, 85193, 44494, 157869, 591, 77, 47650, 957, 213817, 112761, 489, 222807, 43104, 22857, 32801, 255, 4350, 209959, 1951, 26275, 4, 4443, 2401, 2868, 129, 19890, 34073, 5, 93545, 1125, 2454, 222253, 591, 1145, 57680, 1394, 94, 126636, 983, 167, 45551, 46, 135, 118197, 983, 61, 61944, 41690, 45551, 4, 61, 74393, 154297, 24397, 42710, 64088, 559, 10403, 84, 37978, 73960, 105, 83718, 10506, 4, 72810, 5969, 105, 210, 102074, 1315, 142727, 154913, 15, 103, 14307, 97418, 36715, 6218, 4, 26117, 33825, 66423, 247, 77692, 86536, 44, 160677, 260, 69245, 10447, 5, 829, 6139, 117750, 4, 100437, 22126, 2511, 78474, 1343, 141298, 43196, 518, 94, 126636, 1867, 167, 61980, 17604, 4, 154297, 117018, 551, 36788, 61, 8220, 59915, 419, 5, 44, 8741, 1757, 17931, 5593, 209414, 827, 4, 49, 157159, 48833, 260, 3640, 4509, 417, 49251, 27931, 546, 67079, 106, 4, 12678, 49, 77958, 1719, 6, 220408, 260, 23211, 14321, 36175, 3645, 4, 3573, 121565, 382, 4, 12187, 1182, 118977, 1082, 72553, 105, 29, 58270, 5, 417, 3516, 5785, 23495, 4, 84, 51099, 894, 11075, 4509, 4, 79091, 94, 126636, 1867, 167, 61980, 17604, 210, 130640, 255, 95531, 4, 489, 191839, 11069, 255, 116, 12187, 26275, 139162, 29, 58270, 5, 829, 45404, 1281, 99497, 171437, 3823, 4, 84, 64371, 18122, 94, 126636, 260, 167, 39245, 72810, 75807, 169886, 23211, 14321, 36175, 3645, 489, 183, 24918, 14632, 61, 2157, 190, 14427, 139162, 29, 58270, 830, 20, 125093, 84, 214101, 5, 901, 11009, 419, 139139, 4, 899, 84, 11075, 18122, 154297, 77, 210, 1455, 15487, 232373, 36273, 66155, 26, 122584, 6962, 225216, 827, 154913, 210, 6, 114277, 5235, 1120, 1315, 142727, 489, 2724, 33681, 32801, 61, 440, 59693, 957, 49, 5308, 83718, 10506, 61, 94, 126636, 1595, 167, 45551, 419, 5, 5188, 989, 476, 141275, 61, 10698, 62653, 12088, 84, 11075, 18122, 135, 26680, 49361, 714, 27275, 19754, 5, 44, 2426, 7917, 130, 77, 201619, 135997, 132621, 5934, 751, 154913, 4, 72810, 1200, 4048, 210, 6, 114277, 5235, 652, 52241, 142727, 4, 49601, 128587, 89, 29, 78913, 11231, 1093, 201619, 1270, 20160, 163431, 5401, 29209, 2375, 94, 126636, 1867, 167, 61980, 17604, 28390, 210, 153415, 97750, 23211, 14321, 36175, 3645, 518, 55979, 489, 18777, 40144, 91655, 790, 65439, 154913, 210, 94, 222, 33611, 147214, 167, 142727, 830, 20, 125093, 84, 214101, 5, 1522, 80812, 2375, 4, 1443, 6, 29933, 18633, 154594, 44, 160677, 260, 69245, 10447, 830, 116649, 152950, 23211, 14321, 36175, 3645, 1182, 165551, 189, 71957, 168164, 1068, 4, 8098, 14711, 10403, 6, 140983, 24032, 957, 2], [0, 447, 120457, 176357, 52503, 611, 46333, 43715, 180507, 29, 8244, 74116, 8363, 46, 49, 526, 44479, 260, 16256, 4, 24677, 28411, 23407, 2, 2, 447, 120457, 176357, 423, 6, 174543, 43715, 180507, 29, 211247, 59, 4335, 20017, 46, 49, 526, 44479, 260, 16256, 4, 252, 97769, 28411, 23407, 5, 241758, 12, 62911, 50846, 113596, 57971, 14217, 25, 245, 120457, 417, 23300, 1125, 17181, 61421, 415, 44, 10042, 2716, 35785, 97964, 830, 601, 28814, 5, 2424, 215083, 59, 28686, 12, 44, 1560, 1041, 1082, 59125, 89707, 105928, 6, 57441, 62561, 4, 899, 109016, 245, 67414, 89, 2033, 20, 2157, 77, 11551, 12206, 4, 3573, 1450, 6761, 66136, 12375, 127908, 7574, 4, 1443, 20177, 137104, 4108, 1086, 62849, 790, 5, 77870, 84, 1611, 232194, 197, 423, 6, 174543, 43715, 150426, 5, 7633, 52503, 49, 526, 44479, 260, 16256, 4, 24677, 28411, 23407, 740, 496, 1126, 55641, 10654, 112027, 546, 133549, 61, 103005, 211247, 6207, 97428, 59342, 5, 829, 117750, 113596, 57971, 14217, 26, 245, 4, 87968, 1867, 19760, 89, 8378, 232194, 197, 427, 72726, 13674, 31543, 107433, 4, 252, 6, 108265, 29778, 112027, 546, 5, 55602, 127810, 5835, 8378, 232194, 197, 382, 305, 14889, 61, 103005, 4, 252, 6, 116518, 6826, 112027, 1219, 5, 829, 33280, 62911, 59, 6, 61037, 57971, 14217, 25, 245, 4, 221271, 61, 336, 2042, 1417, 4108, 7125, 1960, 4, 899, 1203, 16256, 6826, 4, 180507, 29, 8244, 74116, 8363, 4, 948, 81353, 205, 36051, 44008, 5, 10604, 894, 49, 120457, 47815, 106, 125592, 39050, 23059, 1009, 11080, 861, 1417, 9, 2372, 17902, 4, 47815, 10285, 160245, 6826, 2336, 13713, 34397, 255, 58281, 790, 3681, 2354, 5089, 5, 829, 87968, 105, 128596, 84, 60755, 96417, 476, 4698, 16931, 47815, 6, 120301, 46333, 43715, 150426, 87540, 29, 211247, 59, 4335, 20017, 5, 140865, 4, 84, 92389, 3645, 61, 31543, 650, 16931, 16256, 9318, 46333, 43715, 13674, 31543, 107433, 4, 84, 14560, 46, 9103, 6, 136442, 4, 84, 185081, 46, 47815, 1039, 160245, 5, 2], [0, 829, 149953, 15941, 52137, 105, 12, 51149, 652, 1951, 117675, 407, 45846, 551, 2, 2, 116, 59822, 2791, 38811, 5401, 51149, 652, 894, 255, 54835, 29, 74051, 80572, 15941, 52137, 105, 239065, 4401, 29, 68228, 29835, 8659, 20, 29835, 10945, 51149, 652, 894, 5, 2443, 2157, 64058, 81163, 5, 3116, 5, 672, 611, 26451, 19760, 894, 13428, 89, 36753, 29, 15941, 52137, 105, 17910, 21033, 10606, 12, 2718, 3411, 14560, 20, 29835, 8659, 20, 29835, 10945, 51149, 652, 894, 74, 83016, 46, 3912, 4, 4015, 20, 3912, 4, 3742, 51149, 652, 894, 5, 672, 948, 26310, 2697, 4045, 58270, 2791, 38811, 5401, 51149, 652, 894, 255, 54835, 6, 12758, 39049, 29835, 4046, 46, 29835, 11548, 51149, 652, 894, 5, 83016, 14711, 14815, 129, 2789, 4, 11591, 51149, 652, 894, 4, 591, 60178, 129, 3912, 4, 10289, 51149, 652, 894, 5, 672, 44, 45131, 6194, 58, 58270, 255, 3411, 126312, 20233, 129, 29835, 9185, 20, 29835, 8894, 51149, 652, 894, 4, 105849, 20, 3912, 4, 8318, 20, 3912, 4, 1549, 51149, 652, 894, 5, 103999, 12, 196642, 8544, 27153, 50799, 98524, 983, 13428, 51149, 652, 894, 255, 54835, 29, 116, 15229, 2800, 751, 4, 49, 12758, 57758, 4350, 29, 116, 59822, 29, 68228, 29835, 9185, 51149, 652, 894, 5, 187166, 983, 13428, 51149, 652, 894, 255, 105849, 198546, 312, 29, 68228, 3912, 4, 6460, 19754, 15, 212, 106, 59822, 20, 3912, 4, 2485, 51149, 652, 894, 194, 116, 59822, 2791, 38811, 5401, 51149, 652, 894, 255, 54835, 29, 1182, 55929, 3386, 15941, 52137, 105, 239065, 4401, 29, 68228, 29835, 10289, 20, 29835, 5843, 51149, 652, 894, 5, 2], [0, 447, 242691, 112027, 145865, 53001, 26769, 4459, 85580, 44, 207302, 197, 8830, 6218, 58, 20, 114510, 2, 2, 447, 3707, 25, 1343, 111751, 21214, 415, 193100, 114510, 152322, 591, 102954, 78903, 44494, 73272, 1070, 19477, 31514, 12544, 59, 234261, 59, 99686, 127879, 4, 37715, 44747, 4459, 36198, 127049, 2375, 160950, 328, 171218, 169961, 1595, 84, 67201, 260, 49, 242691, 5, 178588, 30694, 179638, 9403, 84, 109673, 214101, 77, 78150, 4108, 6, 28257, 112027, 52926, 20, 1145, 1400, 4, 1443, 189, 28037, 6139, 65715, 161226, 5, 102576, 56172, 13205, 4, 899, 111010, 29, 106261, 358, 105389, 161226, 28232, 10690, 77, 176357, 5, 1813, 16285, 220944, 108970, 210, 145804, 29, 166679, 84, 240432, 244, 161767, 489, 67527, 1041, 23122, 214459, 91367, 1117, 4, 899, 125093, 591, 39282, 156581, 73272, 1070, 19477, 31514, 12544, 59, 234261, 59, 99686, 127879, 489, 22175, 7917, 32394, 206600, 53001, 26769, 928, 45771, 105, 19493, 358, 10088, 105, 5, 829, 143047, 743, 9309, 211098, 4, 77692, 14424, 171285, 158193, 36690, 45961, 4, 12556, 44, 1323, 3281, 48093, 1182, 6, 142418, 105, 6812, 512, 105, 4, 29, 92472, 244, 58400, 244, 17910, 31871, 89, 740, 2443, 102954, 99686, 127879, 84, 109673, 6, 215222, 5013, 2475, 108483, 189, 58400, 6, 154000, 1867, 119039, 10612, 53001, 790, 15, 5840, 7309, 49251, 16, 2578, 130, 34452, 9436, 222, 167066, 5, 44, 30865, 77, 17910, 229109, 5, 5758, 16256, 20, 61, 26877, 1117, 723, 19921, 827, 49, 1611, 17910, 129, 32892, 894, 162925, 20, 3573, 784, 4350, 129, 12149, 6207, 1449, 1087, 29325, 68605, 45993, 6055, 13572, 80498, 830, 20, 170001, 9436, 222, 167066, 5, 13578, 130, 49, 161264, 99686, 17999, 189, 56054, 7315, 129, 6574, 151322, 255, 85580, 44, 207302, 197, 8830, 6218, 58, 99686, 17999, 3273, 106760, 419, 2513, 9234, 117898, 1009, 53001, 790, 72153, 489, 1741, 19408, 38167, 152217, 17041, 114265, 790, 4, 64764, 94441, 130, 76871, 49, 115892, 116237, 6, 82513, 1951, 32444, 77, 4350, 6, 81041, 260, 114931, 89, 5, 672, 48101, 59596, 99686, 127879, 49, 146568, 255, 85580, 44, 207302, 197, 8830, 6218, 830, 10944, 18036, 10403, 33762, 21305, 131537, 135, 158958, 26291, 4, 11501, 4, 29, 56678, 42773, 37062, 4, 241285, 29, 1976, 35359, 7061, 2475, 2513, 5, 179590, 110401, 210, 17831, 6326, 546, 49, 44, 27543, 12270, 84, 230844, 2225, 18183, 14448, 187392, 18183, 252, 55208, 244, 830, 44, 723, 894, 51100, 743, 54919, 181852, 28618, 105, 13196, 743, 178157, 58, 489, 44, 1323, 23434, 14998, 894, 214038, 133821, 49921, 740, 2842, 1794, 16826, 1041, 4, 28037, 2375, 518, 9950, 11827, 14271, 15016, 7811, 78903, 4, 8790, 99686, 17999, 49, 9648, 9, 2042, 4108, 4853, 928, 9234, 476, 13510, 21891, 2513, 61, 86856, 84294, 105, 132782, 5, 39092, 67414, 89, 1536, 1086, 14394, 13191, 52876, 6, 127049, 4281, 99686, 17999, 212694, 52207, 62981, 36273, 28925, 89, 5, 44, 128664, 33438, 677, 743, 10719, 4, 11501, 77, 196198, 4, 56054, 784, 9234, 476, 13510, 61, 42229, 10506, 5, 90917, 103, 20, 42099, 61, 2157, 61, 15459, 1047, 3107, 5, 1509, 40363, 5235, 11551, 1960, 4, 899, 66298, 518, 94095, 5, 75258, 103, 20, 784, 226856, 1741, 2]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preprocess_function(dataset['train'][:5])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "early-census",
   "metadata": {
    "id": "zS-6iXTkIrJT",
    "papermill": {
     "duration": 0.04462,
     "end_time": "2021-03-26T11:45:10.505575",
     "exception": false,
     "start_time": "2021-03-26T11:45:10.460955",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "To apply this function on all the sentences (or pairs of sentences) in our dataset, we just use the `map` method of our `dataset` object we created earlier. This will apply the function on all the elements of all the splits in `dataset`, so our training, validation and testing data will be preprocessed in one single command."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "daily-contamination",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-03-26T11:45:10.602952Z",
     "iopub.status.busy": "2021-03-26T11:45:10.602195Z",
     "iopub.status.idle": "2021-03-26T11:46:53.289441Z",
     "shell.execute_reply": "2021-03-26T11:46:53.288799Z"
    },
    "id": "DDtsaJeVIrJT",
    "outputId": "a4f3ceb3-3f45-45cd-ef1f-26ff4753fcc1",
    "papermill": {
     "duration": 102.738981,
     "end_time": "2021-03-26T11:46:53.289614",
     "exception": false,
     "start_time": "2021-03-26T11:45:10.550633",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4609ca26fcea49d19a9dc0121a40a5ba",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=17.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "162fb67d2c19492cb144ab98857e45cd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=6.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0ebe447c93ac49728be238138e0448bb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=17.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['attention_mask', 'idx', 'input_ids', 'label', 'sentence1', 'sentence2'],\n",
       "        num_rows: 16177\n",
       "    })\n",
       "    validation: Dataset({\n",
       "        features: ['attention_mask', 'idx', 'input_ids', 'label', 'sentence1', 'sentence2'],\n",
       "        num_rows: 5339\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['attention_mask', 'idx', 'input_ids', 'sentence1', 'sentence2'],\n",
       "        num_rows: 16183\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoded_dataset = dataset.map(preprocess_function, batched=True)\n",
    "encoded_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "absolute-dublin",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-03-26T11:46:53.406646Z",
     "iopub.status.busy": "2021-03-26T11:46:53.405926Z",
     "iopub.status.idle": "2021-03-26T11:46:53.410384Z",
     "shell.execute_reply": "2021-03-26T11:46:53.410833Z"
    },
    "id": "Z632oJaEqxO5",
    "outputId": "301e798f-114a-4700-b18c-3b3d1b9a03b6",
    "papermill": {
     "duration": 0.073163,
     "end_time": "2021-03-26T11:46:53.410957",
     "exception": false,
     "start_time": "2021-03-26T11:46:53.337794",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'attention_mask': [1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1],\n",
       " 'idx': 2118,\n",
       " 'input_ids': [0,\n",
       "  142740,\n",
       "  70615,\n",
       "  12,\n",
       "  75150,\n",
       "  2513,\n",
       "  1252,\n",
       "  217129,\n",
       "  242705,\n",
       "  30037,\n",
       "  26940,\n",
       "  1611,\n",
       "  2372,\n",
       "  82912,\n",
       "  2,\n",
       "  2,\n",
       "  44,\n",
       "  2426,\n",
       "  380,\n",
       "  1078,\n",
       "  1417,\n",
       "  15016,\n",
       "  3280,\n",
       "  58,\n",
       "  11069,\n",
       "  31760,\n",
       "  23633,\n",
       "  28037,\n",
       "  2375,\n",
       "  518,\n",
       "  1611,\n",
       "  2372,\n",
       "  82912,\n",
       "  2511,\n",
       "  65346,\n",
       "  2375,\n",
       "  76570,\n",
       "  743,\n",
       "  248,\n",
       "  14419,\n",
       "  378,\n",
       "  294,\n",
       "  76234,\n",
       "  10854,\n",
       "  268,\n",
       "  4,\n",
       "  45771,\n",
       "  236233,\n",
       "  3191,\n",
       "  28105,\n",
       "  210,\n",
       "  14193,\n",
       "  105,\n",
       "  551,\n",
       "  9543,\n",
       "  10573,\n",
       "  928,\n",
       "  4,\n",
       "  3573,\n",
       "  6,\n",
       "  12149,\n",
       "  14709,\n",
       "  105,\n",
       "  93565,\n",
       "  105,\n",
       "  49,\n",
       "  231008,\n",
       "  260,\n",
       "  21891,\n",
       "  214462,\n",
       "  210506,\n",
       "  4,\n",
       "  229717,\n",
       "  57758,\n",
       "  105128,\n",
       "  39890,\n",
       "  205042,\n",
       "  84,\n",
       "  65346,\n",
       "  2800,\n",
       "  23384,\n",
       "  260,\n",
       "  5,\n",
       "  60163,\n",
       "  8041,\n",
       "  91331,\n",
       "  129,\n",
       "  22762,\n",
       "  63542,\n",
       "  210,\n",
       "  129,\n",
       "  44680,\n",
       "  2375,\n",
       "  161256,\n",
       "  2375,\n",
       "  205042,\n",
       "  150543,\n",
       "  1182,\n",
       "  67262,\n",
       "  43527,\n",
       "  222463,\n",
       "  244,\n",
       "  114187,\n",
       "  5,\n",
       "  2443,\n",
       "  2157,\n",
       "  49,\n",
       "  106844,\n",
       "  45059,\n",
       "  25110,\n",
       "  58219,\n",
       "  142740,\n",
       "  70615,\n",
       "  50848,\n",
       "  2757,\n",
       "  49398,\n",
       "  22762,\n",
       "  6546,\n",
       "  5,\n",
       "  6,\n",
       "  61684,\n",
       "  260,\n",
       "  141968,\n",
       "  32098,\n",
       "  47124,\n",
       "  546,\n",
       "  199543,\n",
       "  72681,\n",
       "  38335,\n",
       "  3660,\n",
       "  827,\n",
       "  3470,\n",
       "  49148,\n",
       "  13433,\n",
       "  76229,\n",
       "  10612,\n",
       "  173187,\n",
       "  16283,\n",
       "  4,\n",
       "  10944,\n",
       "  70284,\n",
       "  181441,\n",
       "  59,\n",
       "  76452,\n",
       "  1595,\n",
       "  142865,\n",
       "  114618,\n",
       "  5,\n",
       "  84736,\n",
       "  1045,\n",
       "  41839,\n",
       "  32801,\n",
       "  12621,\n",
       "  86162,\n",
       "  105,\n",
       "  4,\n",
       "  33567,\n",
       "  78508,\n",
       "  190113,\n",
       "  110832,\n",
       "  743,\n",
       "  29,\n",
       "  33834,\n",
       "  37221,\n",
       "  142865,\n",
       "  3712,\n",
       "  4,\n",
       "  252,\n",
       "  63296,\n",
       "  2375,\n",
       "  20,\n",
       "  40190,\n",
       "  210,\n",
       "  105644,\n",
       "  2297,\n",
       "  518,\n",
       "  205042,\n",
       "  5,\n",
       "  180230,\n",
       "  2574,\n",
       "  4,\n",
       "  2336,\n",
       "  175598,\n",
       "  40097,\n",
       "  142865,\n",
       "  23407,\n",
       "  35307,\n",
       "  49,\n",
       "  72212,\n",
       "  67079,\n",
       "  20300,\n",
       "  205042,\n",
       "  5,\n",
       "  34600,\n",
       "  49880,\n",
       "  4,\n",
       "  141968,\n",
       "  49,\n",
       "  6574,\n",
       "  1068,\n",
       "  14632,\n",
       "  41282,\n",
       "  591,\n",
       "  67492,\n",
       "  6326,\n",
       "  61,\n",
       "  153631,\n",
       "  790,\n",
       "  5,\n",
       "  32936,\n",
       "  137900,\n",
       "  10403,\n",
       "  4,\n",
       "  899,\n",
       "  145865,\n",
       "  20070,\n",
       "  183,\n",
       "  800,\n",
       "  10403,\n",
       "  1394,\n",
       "  17210,\n",
       "  4968,\n",
       "  70971,\n",
       "  205042,\n",
       "  130,\n",
       "  11069,\n",
       "  13753,\n",
       "  31760,\n",
       "  5,\n",
       "  417,\n",
       "  11580,\n",
       "  2157,\n",
       "  28239,\n",
       "  35172,\n",
       "  168806,\n",
       "  44,\n",
       "  698,\n",
       "  380,\n",
       "  1078,\n",
       "  1417,\n",
       "  15016,\n",
       "  60058,\n",
       "  740,\n",
       "  127532,\n",
       "  89,\n",
       "  168822,\n",
       "  20070,\n",
       "  23343,\n",
       "  29658,\n",
       "  751,\n",
       "  222463,\n",
       "  244,\n",
       "  114187,\n",
       "  4,\n",
       "  8734,\n",
       "  61,\n",
       "  76981,\n",
       "  6326,\n",
       "  92472,\n",
       "  28513,\n",
       "  89,\n",
       "  5,\n",
       "  60163,\n",
       "  34820,\n",
       "  16399,\n",
       "  187740,\n",
       "  743,\n",
       "  518,\n",
       "  204627,\n",
       "  67079,\n",
       "  27033,\n",
       "  128997,\n",
       "  120119,\n",
       "  105,\n",
       "  4,\n",
       "  255,\n",
       "  36421,\n",
       "  78004,\n",
       "  49692,\n",
       "  59730,\n",
       "  210,\n",
       "  156618,\n",
       "  10943,\n",
       "  54676,\n",
       "  957,\n",
       "  41101,\n",
       "  28513,\n",
       "  790,\n",
       "  3470,\n",
       "  18036,\n",
       "  6326,\n",
       "  150146,\n",
       "  29,\n",
       "  2744,\n",
       "  1536,\n",
       "  74818,\n",
       "  36561,\n",
       "  244,\n",
       "  189737,\n",
       "  55692,\n",
       "  790,\n",
       "  5,\n",
       "  26469,\n",
       "  7454,\n",
       "  13585,\n",
       "  13405,\n",
       "  186408,\n",
       "  3410,\n",
       "  20070,\n",
       "  57094,\n",
       "  20484,\n",
       "  130,\n",
       "  5,\n",
       "  6061,\n",
       "  29045,\n",
       "  1757,\n",
       "  109577,\n",
       "  43914,\n",
       "  1093,\n",
       "  32104,\n",
       "  2033,\n",
       "  49,\n",
       "  217926,\n",
       "  210,\n",
       "  121987,\n",
       "  5,\n",
       "  829,\n",
       "  17210,\n",
       "  12320,\n",
       "  155677,\n",
       "  205042,\n",
       "  150543,\n",
       "  61,\n",
       "  130122,\n",
       "  43527,\n",
       "  222463,\n",
       "  244,\n",
       "  114187,\n",
       "  28390,\n",
       "  26675,\n",
       "  419,\n",
       "  146857,\n",
       "  1867,\n",
       "  4,\n",
       "  3573,\n",
       "  14741,\n",
       "  77,\n",
       "  178196,\n",
       "  1867,\n",
       "  37976,\n",
       "  3645,\n",
       "  2513,\n",
       "  5,\n",
       "  44,\n",
       "  118657,\n",
       "  13196,\n",
       "  141968,\n",
       "  49,\n",
       "  116968,\n",
       "  113820,\n",
       "  129,\n",
       "  19890,\n",
       "  14815,\n",
       "  8044,\n",
       "  4,\n",
       "  8790,\n",
       "  118840,\n",
       "  83812,\n",
       "  60819,\n",
       "  80737,\n",
       "  4,\n",
       "  56157,\n",
       "  96328,\n",
       "  77,\n",
       "  30799,\n",
       "  380,\n",
       "  4,\n",
       "  899,\n",
       "  6,\n",
       "  141512,\n",
       "  15100,\n",
       "  195555,\n",
       "  591,\n",
       "  1976,\n",
       "  24046,\n",
       "  7574,\n",
       "  49,\n",
       "  17041,\n",
       "  172046,\n",
       "  13914,\n",
       "  104001,\n",
       "  830,\n",
       "  20,\n",
       "  125093,\n",
       "  49,\n",
       "  45059,\n",
       "  5,\n",
       "  829,\n",
       "  105571,\n",
       "  142865,\n",
       "  3712,\n",
       "  790,\n",
       "  20,\n",
       "  2157,\n",
       "  13746,\n",
       "  7125,\n",
       "  210,\n",
       "  6317,\n",
       "  547,\n",
       "  1867,\n",
       "  152217,\n",
       "  69359,\n",
       "  85440,\n",
       "  3494,\n",
       "  44,\n",
       "  698,\n",
       "  380,\n",
       "  1078,\n",
       "  1417,\n",
       "  15016,\n",
       "  3280,\n",
       "  740,\n",
       "  90917,\n",
       "  260,\n",
       "  233010,\n",
       "  17910,\n",
       "  210,\n",
       "  8492,\n",
       "  68133,\n",
       "  1091,\n",
       "  49,\n",
       "  2021,\n",
       "  18122,\n",
       "  4,\n",
       "  8790,\n",
       "  69828,\n",
       "  8458,\n",
       "  123482,\n",
       "  36631,\n",
       "  150317,\n",
       "  2800,\n",
       "  230425,\n",
       "  61,\n",
       "  7044,\n",
       "  27673,\n",
       "  19681,\n",
       "  205278,\n",
       "  142865,\n",
       "  3712,\n",
       "  957,\n",
       "  39890,\n",
       "  5,\n",
       "  6061,\n",
       "  56157,\n",
       "  57838,\n",
       "  1045,\n",
       "  170163,\n",
       "  11284,\n",
       "  894,\n",
       "  11878,\n",
       "  17993,\n",
       "  3670,\n",
       "  47177,\n",
       "  130,\n",
       "  3470,\n",
       "  84,\n",
       "  4968,\n",
       "  36561,\n",
       "  130,\n",
       "  5,\n",
       "  47239,\n",
       "  15243,\n",
       "  84,\n",
       "  2871,\n",
       "  48093,\n",
       "  4,\n",
       "  113370,\n",
       "  59,\n",
       "  18637,\n",
       "  52218,\n",
       "  2375,\n",
       "  152217,\n",
       "  1145,\n",
       "  10394,\n",
       "  21059,\n",
       "  5401,\n",
       "  4,\n",
       "  187115,\n",
       "  86855,\n",
       "  790,\n",
       "  3470,\n",
       "  17041,\n",
       "  29,\n",
       "  42134,\n",
       "  1315,\n",
       "  176210,\n",
       "  5,\n",
       "  44,\n",
       "  2426,\n",
       "  380,\n",
       "  1078,\n",
       "  1417,\n",
       "  15016,\n",
       "  3280,\n",
       "  58,\n",
       "  82278,\n",
       "  210,\n",
       "  25,\n",
       "  206333,\n",
       "  29,\n",
       "  61966,\n",
       "  2],\n",
       " 'label': 6,\n",
       " 'sentence1': 'Atlantic Council: Парламент України може нарешті позбутися давніх насмішок',\n",
       " 'sentence2': '\"Кнопкодавство\" вже давно стало причиною для насмішок над Верховною Радою / фото [SOURCE], Олександр Кузьмін\\nУкраїна здобула маленьку, але важливу перемогу в битві проти політичної корупції, запустивши нову систему голосування у Верховній Раді. Ця система повинна покінчити з поширеною практикою голосування депутатів від імені своїх відсутніх колег.\\nПро це в своїй статті пише експерт Atlantic Council Пітер Дікінсон. Українські депутати заповнили соціальні мережі фотографіями й відео після презентації нової системи, яка тепер оснащена додатковим сенсором. Щоб висловитися щодо законопроекту, депутат повинен однією рукою натиснути сенсор, а іншою - одну з кнопок для голосування. Більше того, підтримувати роботу сенсора необхідно впродовж всього часу голосування. Таким чином, депутати втрачають можливість проголосувати за сусідів.\\nАвтор зауважує, що український парламент отруєний багаторазовим голосуванням вже дуже давно. В Україні це явище називають \"кнопкодавством\". Депутати використовують парламентські картки відсутніх колег, щоб зареєструвати їхні голоси. Ця практика була зручною для партій всього політичного спектру, дозволяючи їм збирати потрібну кількість голосів й формувати більшість на тлі мінливих політичних союзів.\\nТака поведінка суперечить парламентським правилам. Але влада демонструвала безсилля в боротьбі з нею. За багато років сцени голосування депутатів замість своїх відсутніх колег стали символом міцної, але часто нефункціональної демократії України.\\n\"Якщо депутати в країні регулярно порушують закон, коли ухвалюють нові закони, тоді зовсім не дивно, що корупція продовжує процвітати в інших сферах життя суспільства\", - йдеться в статті.\\nЗапровадження сенсорів - це остання з цілої низки спроб заборонити \"кнопкодавство\". Перші зусилля були зроблені ще в 2008 році, коли спікер Верховної Ради Арсеній Яценюк запропонував встановити сенсорну систему. Але тоді формат виявився технічно недосконалим й уразливим. Коли проблеми усунули, ініціатива стала жертвою низки тактик блокування, юридичних викликів й інших надуманих перешкод.\\n\"Кнопкодавство\" знову з\\'явилося на порядку денному одразу після Революції Гідності в 2014 році. На післяреволюційному політичному ландшафті України домінували вимоги реформ. Тож звучали заклики рішуче покінчити з багаторазовим голосуванням. Був проведений тендер на встановлення нової системи голосування, але фінансування так і не було виділене.\\nПісля перемоги Володимира Зеленського на президентських і парламентських виборах у 2019 році \"кнопкодавство\" знову опинилося в центрі уваги. В грудні того року партія президента \"Слуга Народу\" ухвалила закон про кримінальну відповідальність за голосування від імені інших депутатів. Однак, шквал судових переслідувань за порушення не почався. І фактична ситуація не змінилася. Про першу спробу покарати депутата за \"кнопкодавство\" оголосила Генпрокурор Ірина Венедіктова лише кілька тижнів тому, 12 лютого.\\nПісля років затримок, виправдань і фальш-стартів чому парламент нарешті запровадив, здавалося б, ефективне рішення проблеми \"кнопкодавства\"? На думку автора, одним з каталізаторів міг стати нещодавній скандал, який почався після голосування за призначення Сергія Шкарлета на посаду міністра освіти України.\\n\"Сенсорна система голосування, запущена 2 березня, не нова. Навпаки, вона використовує ту ж технологію, яка вперше була встановлена в українському парламенті в 2008 році, але так лишилася не активованою. Змінилася лише політична воля нарешті розібратися з \"кнопкодавством\". Це може бути ознакою зміни балансу сил у парламенті\", - йдеться в статті.\\nІснують припущення, що рішення запустити сенсори у Верховній Раді відображає дедалі більші політичні амбіції спікера парламенту Дмитра Разумкова. Він найбільш прямо відповідальний за ініціативу. Автор зауважує, що повідомлень про президентські амбіції Разумкова стає дедалі більше. Імідж того, хто покінчив з \"кнопкодавством\" може посилити його популярність серед виборців як реформатора і політика, здатного добиватися результатів.\\n\"Не залежно від політичних мотивів, які стоять за кроком, зробленим цього тижня, це чітка перемога для української демократії. Роками \"кнопкодавство\" виставляло на посміховисько українську демократію й підривало авторитет парламенту\", - пише автор.\\nВін додає, що захищене сенсорами голосування - не панацея. Але це крок в правильному напрямку, який відображає ширше застосування парламентських традицій в Україні.\\nПереклад: Лесь Димань'}"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoded_dataset[\"validation\"][0]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "generic-sperm",
   "metadata": {
    "id": "voWiw8C7IrJV",
    "papermill": {
     "duration": 0.04822,
     "end_time": "2021-03-26T11:46:53.507614",
     "exception": false,
     "start_time": "2021-03-26T11:46:53.459394",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "Even better, the results are automatically cached by the 🤗 Datasets library to avoid spending time on this step the next time you run your notebook. The 🤗 Datasets library is normally smart enough to detect when the function you pass to map has changed (and thus requires to not use the cache data). For instance, it will properly detect if you change the task in the first cell and rerun the notebook. 🤗 Datasets warns you when it uses cached files, you can pass `load_from_cache_file=False` in the call to `map` to not use the cached files and force the preprocessing to be applied again.\n",
    "\n",
    "Note that we passed `batched=True` to encode the texts by batches together. This is to leverage the full benefit of the fast tokenizer we loaded earlier, which will use multi-threading to treat the texts in a batch concurrently."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "included-lying",
   "metadata": {
    "id": "545PP3o8IrJV",
    "papermill": {
     "duration": 0.04854,
     "end_time": "2021-03-26T11:46:53.604832",
     "exception": false,
     "start_time": "2021-03-26T11:46:53.556292",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Fine-tuning the model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "proprietary-biology",
   "metadata": {
    "id": "FBiW8UpKIrJW",
    "papermill": {
     "duration": 0.048527,
     "end_time": "2021-03-26T11:46:53.701721",
     "exception": false,
     "start_time": "2021-03-26T11:46:53.653194",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "Now that our data is ready, we can download the pretrained model and fine-tune it. Since all our tasks are about sentence classification, we use the `AutoModelForSequenceClassification` class. Like with the tokenizer, the `from_pretrained` method will download and cache the model for us. The only thing we have to specify is the number of labels for our problem (which is always 2, except for STS-B which is a regression problem and MNLI where we have 3 labels):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "theoretical-leave",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-03-26T11:46:53.803466Z",
     "iopub.status.busy": "2021-03-26T11:46:53.802965Z",
     "iopub.status.idle": "2021-03-26T11:48:06.723147Z",
     "shell.execute_reply": "2021-03-26T11:48:06.720019Z"
    },
    "id": "TlqNaB8jIrJW",
    "outputId": "b6242768-9e04-409b-ed62-5bbb0484e40e",
    "papermill": {
     "duration": 72.973356,
     "end_time": "2021-03-26T11:48:06.723315",
     "exception": false,
     "start_time": "2021-03-26T11:46:53.749959",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b04b5965fb0445c8890d6565e9642a20",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=2244861551.0, style=ProgressStyle(descr…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at xlm-roberta-large were not used when initializing XLMRobertaForSequenceClassification: ['lm_head.bias', 'lm_head.dense.weight', 'lm_head.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias']\n",
      "- This IS expected if you are initializing XLMRobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing XLMRobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of XLMRobertaForSequenceClassification were not initialized from the model checkpoint at xlm-roberta-large and are newly initialized: ['classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.out_proj.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "XLMRobertaForSequenceClassification(\n",
       "  (roberta): RobertaModel(\n",
       "    (embeddings): RobertaEmbeddings(\n",
       "      (word_embeddings): Embedding(250002, 1024, padding_idx=1)\n",
       "      (position_embeddings): Embedding(514, 1024, padding_idx=1)\n",
       "      (token_type_embeddings): Embedding(1, 1024)\n",
       "      (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "      (dropout): Dropout(p=0.1, inplace=False)\n",
       "    )\n",
       "    (encoder): RobertaEncoder(\n",
       "      (layer): ModuleList(\n",
       "        (0): RobertaLayer(\n",
       "          (attention): RobertaAttention(\n",
       "            (self): RobertaSelfAttention(\n",
       "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): RobertaSelfOutput(\n",
       "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): RobertaIntermediate(\n",
       "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "          )\n",
       "          (output): RobertaOutput(\n",
       "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (1): RobertaLayer(\n",
       "          (attention): RobertaAttention(\n",
       "            (self): RobertaSelfAttention(\n",
       "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): RobertaSelfOutput(\n",
       "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): RobertaIntermediate(\n",
       "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "          )\n",
       "          (output): RobertaOutput(\n",
       "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (2): RobertaLayer(\n",
       "          (attention): RobertaAttention(\n",
       "            (self): RobertaSelfAttention(\n",
       "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): RobertaSelfOutput(\n",
       "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): RobertaIntermediate(\n",
       "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "          )\n",
       "          (output): RobertaOutput(\n",
       "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (3): RobertaLayer(\n",
       "          (attention): RobertaAttention(\n",
       "            (self): RobertaSelfAttention(\n",
       "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): RobertaSelfOutput(\n",
       "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): RobertaIntermediate(\n",
       "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "          )\n",
       "          (output): RobertaOutput(\n",
       "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (4): RobertaLayer(\n",
       "          (attention): RobertaAttention(\n",
       "            (self): RobertaSelfAttention(\n",
       "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): RobertaSelfOutput(\n",
       "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): RobertaIntermediate(\n",
       "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "          )\n",
       "          (output): RobertaOutput(\n",
       "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (5): RobertaLayer(\n",
       "          (attention): RobertaAttention(\n",
       "            (self): RobertaSelfAttention(\n",
       "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): RobertaSelfOutput(\n",
       "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): RobertaIntermediate(\n",
       "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "          )\n",
       "          (output): RobertaOutput(\n",
       "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (6): RobertaLayer(\n",
       "          (attention): RobertaAttention(\n",
       "            (self): RobertaSelfAttention(\n",
       "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): RobertaSelfOutput(\n",
       "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): RobertaIntermediate(\n",
       "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "          )\n",
       "          (output): RobertaOutput(\n",
       "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (7): RobertaLayer(\n",
       "          (attention): RobertaAttention(\n",
       "            (self): RobertaSelfAttention(\n",
       "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): RobertaSelfOutput(\n",
       "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): RobertaIntermediate(\n",
       "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "          )\n",
       "          (output): RobertaOutput(\n",
       "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (8): RobertaLayer(\n",
       "          (attention): RobertaAttention(\n",
       "            (self): RobertaSelfAttention(\n",
       "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): RobertaSelfOutput(\n",
       "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): RobertaIntermediate(\n",
       "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "          )\n",
       "          (output): RobertaOutput(\n",
       "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (9): RobertaLayer(\n",
       "          (attention): RobertaAttention(\n",
       "            (self): RobertaSelfAttention(\n",
       "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): RobertaSelfOutput(\n",
       "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): RobertaIntermediate(\n",
       "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "          )\n",
       "          (output): RobertaOutput(\n",
       "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (10): RobertaLayer(\n",
       "          (attention): RobertaAttention(\n",
       "            (self): RobertaSelfAttention(\n",
       "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): RobertaSelfOutput(\n",
       "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): RobertaIntermediate(\n",
       "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "          )\n",
       "          (output): RobertaOutput(\n",
       "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (11): RobertaLayer(\n",
       "          (attention): RobertaAttention(\n",
       "            (self): RobertaSelfAttention(\n",
       "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): RobertaSelfOutput(\n",
       "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): RobertaIntermediate(\n",
       "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "          )\n",
       "          (output): RobertaOutput(\n",
       "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (12): RobertaLayer(\n",
       "          (attention): RobertaAttention(\n",
       "            (self): RobertaSelfAttention(\n",
       "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): RobertaSelfOutput(\n",
       "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): RobertaIntermediate(\n",
       "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "          )\n",
       "          (output): RobertaOutput(\n",
       "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (13): RobertaLayer(\n",
       "          (attention): RobertaAttention(\n",
       "            (self): RobertaSelfAttention(\n",
       "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): RobertaSelfOutput(\n",
       "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): RobertaIntermediate(\n",
       "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "          )\n",
       "          (output): RobertaOutput(\n",
       "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (14): RobertaLayer(\n",
       "          (attention): RobertaAttention(\n",
       "            (self): RobertaSelfAttention(\n",
       "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): RobertaSelfOutput(\n",
       "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): RobertaIntermediate(\n",
       "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "          )\n",
       "          (output): RobertaOutput(\n",
       "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (15): RobertaLayer(\n",
       "          (attention): RobertaAttention(\n",
       "            (self): RobertaSelfAttention(\n",
       "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): RobertaSelfOutput(\n",
       "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): RobertaIntermediate(\n",
       "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "          )\n",
       "          (output): RobertaOutput(\n",
       "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (16): RobertaLayer(\n",
       "          (attention): RobertaAttention(\n",
       "            (self): RobertaSelfAttention(\n",
       "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): RobertaSelfOutput(\n",
       "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): RobertaIntermediate(\n",
       "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "          )\n",
       "          (output): RobertaOutput(\n",
       "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (17): RobertaLayer(\n",
       "          (attention): RobertaAttention(\n",
       "            (self): RobertaSelfAttention(\n",
       "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): RobertaSelfOutput(\n",
       "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): RobertaIntermediate(\n",
       "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "          )\n",
       "          (output): RobertaOutput(\n",
       "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (18): RobertaLayer(\n",
       "          (attention): RobertaAttention(\n",
       "            (self): RobertaSelfAttention(\n",
       "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): RobertaSelfOutput(\n",
       "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): RobertaIntermediate(\n",
       "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "          )\n",
       "          (output): RobertaOutput(\n",
       "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (19): RobertaLayer(\n",
       "          (attention): RobertaAttention(\n",
       "            (self): RobertaSelfAttention(\n",
       "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): RobertaSelfOutput(\n",
       "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): RobertaIntermediate(\n",
       "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "          )\n",
       "          (output): RobertaOutput(\n",
       "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (20): RobertaLayer(\n",
       "          (attention): RobertaAttention(\n",
       "            (self): RobertaSelfAttention(\n",
       "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): RobertaSelfOutput(\n",
       "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): RobertaIntermediate(\n",
       "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "          )\n",
       "          (output): RobertaOutput(\n",
       "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (21): RobertaLayer(\n",
       "          (attention): RobertaAttention(\n",
       "            (self): RobertaSelfAttention(\n",
       "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): RobertaSelfOutput(\n",
       "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): RobertaIntermediate(\n",
       "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "          )\n",
       "          (output): RobertaOutput(\n",
       "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (22): RobertaLayer(\n",
       "          (attention): RobertaAttention(\n",
       "            (self): RobertaSelfAttention(\n",
       "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): RobertaSelfOutput(\n",
       "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): RobertaIntermediate(\n",
       "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "          )\n",
       "          (output): RobertaOutput(\n",
       "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (23): RobertaLayer(\n",
       "          (attention): RobertaAttention(\n",
       "            (self): RobertaSelfAttention(\n",
       "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): RobertaSelfOutput(\n",
       "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): RobertaIntermediate(\n",
       "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "          )\n",
       "          (output): RobertaOutput(\n",
       "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (classifier): RobertaClassificationHead(\n",
       "    (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "    (dropout): Dropout(p=0.1, inplace=False)\n",
       "    (out_proj): Linear(in_features=1024, out_features=7, bias=True)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from transformers import AutoModelForSequenceClassification, TrainingArguments, Trainer\n",
    "\n",
    "num_labels = len(data['label'].unique())\n",
    "model = AutoModelForSequenceClassification.from_pretrained(model_checkpoint, num_labels=num_labels)\n",
    "model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cutting-writer",
   "metadata": {
    "id": "CczA5lJlIrJX",
    "papermill": {
     "duration": 0.050893,
     "end_time": "2021-03-26T11:48:06.825614",
     "exception": false,
     "start_time": "2021-03-26T11:48:06.774721",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "The warning is telling us we are throwing away some weights (the `vocab_transform` and `vocab_layer_norm` layers) and randomly initializing some other (the `pre_classifier` and `classifier` layers). This is absolutely normal in this case, because we are removing the head used to pretrain the model on a masked language modeling objective and replacing it with a new head for which we don't have pretrained weights, so the library warns us we should fine-tune this model before using it for inference, which is exactly what we are going to do."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "exotic-permission",
   "metadata": {
    "id": "_N8urzhyIrJY",
    "papermill": {
     "duration": 0.058938,
     "end_time": "2021-03-26T11:48:06.945573",
     "exception": false,
     "start_time": "2021-03-26T11:48:06.886635",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "To instantiate a `Trainer`, we will need to define two more things. The most important is the [`TrainingArguments`](https://huggingface.co/transformers/main_classes/trainer.html#transformers.TrainingArguments), which is a class that contains all the attributes to customize the training. It requires one folder name, which will be used to save the checkpoints of the model, and all other arguments are optional:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "curious-wayne",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-03-26T11:48:07.053243Z",
     "iopub.status.busy": "2021-03-26T11:48:07.052735Z",
     "iopub.status.idle": "2021-03-26T11:48:07.417625Z",
     "shell.execute_reply": "2021-03-26T11:48:07.418103Z"
    },
    "id": "Bliy8zgjIrJY",
    "papermill": {
     "duration": 0.421392,
     "end_time": "2021-03-26T11:48:07.418267",
     "exception": false,
     "start_time": "2021-03-26T11:48:06.996875",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "metric_name = \"f1\"\n",
    "\n",
    "args = TrainingArguments(\n",
    "    \"test-RoBERTa\",\n",
    "    evaluation_strategy = \"epoch\",\n",
    "    learning_rate=2e-5,\n",
    "    per_device_train_batch_size=batch_size,\n",
    "    per_device_eval_batch_size=batch_size,\n",
    "    num_train_epochs=3,\n",
    "    weight_decay=0.01,\n",
    "    load_best_model_at_end=True,\n",
    "    metric_for_best_model=metric_name,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "auburn-commissioner",
   "metadata": {
    "id": "km3pGVdTIrJc",
    "papermill": {
     "duration": 0.050893,
     "end_time": "2021-03-26T11:48:07.520785",
     "exception": false,
     "start_time": "2021-03-26T11:48:07.469892",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "Here we set the evaluation to be done at the end of each epoch, tweak the learning rate, use the `batch_size` defined at the top of the notebook and customize the number of epochs for training, as well as the weight decay. Since the best model might not be the one at the end of training, we ask the `Trainer` to load the best model it saved (according to `metric_name`) at the end of training."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "advisory-reason",
   "metadata": {
    "id": "7sZOdRlRIrJd",
    "papermill": {
     "duration": 0.050775,
     "end_time": "2021-03-26T11:48:07.622541",
     "exception": false,
     "start_time": "2021-03-26T11:48:07.571766",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "The last thing to define for our `Trainer` is how to compute the metrics from the predictions. We need to define a function for this, which will just use the `metric` we loaded earlier, the only preprocessing we have to do is to take the argmax of our predicted logits (our just squeeze the last axis in the case of STS-B):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "adjustable-plasma",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-03-26T11:48:07.733614Z",
     "iopub.status.busy": "2021-03-26T11:48:07.732948Z",
     "iopub.status.idle": "2021-03-26T11:48:07.735814Z",
     "shell.execute_reply": "2021-03-26T11:48:07.735284Z"
    },
    "id": "UmvbnJ9JIrJd",
    "papermill": {
     "duration": 0.062401,
     "end_time": "2021-03-26T11:48:07.735921",
     "exception": false,
     "start_time": "2021-03-26T11:48:07.673520",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import f1_score\n",
    "\n",
    "def compute_metrics(pred):\n",
    "    predictions, labels = pred\n",
    "    predictions = np.argmax(predictions, axis=1)\n",
    "\n",
    "    f1 = f1_score(labels, predictions, average='macro')\n",
    "    return {\n",
    "        'f1': f1,\n",
    "    }"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "grand-shame",
   "metadata": {
    "id": "rXuFTAzDIrJe",
    "papermill": {
     "duration": 0.056344,
     "end_time": "2021-03-26T11:48:07.847970",
     "exception": false,
     "start_time": "2021-03-26T11:48:07.791626",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "Then we just need to pass all of this along with our datasets to the `Trainer`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "crude-immigration",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-03-26T11:48:07.964487Z",
     "iopub.status.busy": "2021-03-26T11:48:07.964003Z",
     "iopub.status.idle": "2021-03-26T11:48:15.215824Z",
     "shell.execute_reply": "2021-03-26T11:48:15.216219Z"
    },
    "id": "imY1oC3SIrJf",
    "papermill": {
     "duration": 7.312828,
     "end_time": "2021-03-26T11:48:15.216374",
     "exception": false,
     "start_time": "2021-03-26T11:48:07.903546",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W&B installed but not logged in. Run `wandb login` or set the WANDB_API_KEY env variable.\n"
     ]
    }
   ],
   "source": [
    "validation_key = \"validation\"\n",
    "trainer = Trainer(\n",
    "    model,\n",
    "    args,\n",
    "    train_dataset=encoded_dataset[\"train\"],\n",
    "    eval_dataset=encoded_dataset[validation_key],\n",
    "    tokenizer=tokenizer,\n",
    "    compute_metrics=compute_metrics\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "structured-federal",
   "metadata": {
    "id": "ibWGmvxbIrJg",
    "papermill": {
     "duration": 0.083397,
     "end_time": "2021-03-26T11:48:15.390241",
     "exception": false,
     "start_time": "2021-03-26T11:48:15.306844",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "You might wonder why we pass along the `tokenizer` when we already preprocessed our data. This is because we will use it once last time to make all the samples we gather the same length by applying padding, which requires knowing the model's preferences regarding padding (to the left or right? with which token?). The `tokenizer` has a pad method that will do all of this right for us, and the `Trainer` will use it. You can customize this part by defining and passing your own `data_collator` which will receive the samples like the dictionaries seen above and will need to return a dictionary of tensors."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "angry-aggregate",
   "metadata": {
    "id": "CdzABDVcIrJg",
    "papermill": {
     "duration": 0.084198,
     "end_time": "2021-03-26T11:48:15.556642",
     "exception": false,
     "start_time": "2021-03-26T11:48:15.472444",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "We can now finetune our model by just calling the `train` method:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "frank-sweet",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-03-26T11:48:15.745834Z",
     "iopub.status.busy": "2021-03-26T11:48:15.745088Z"
    },
    "id": "uNx5pyRlIrJh",
    "outputId": "7bb7559d-d165-4db2-b5ea-75640cf6e5de",
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": false,
     "start_time": "2021-03-26T11:48:15.652368",
     "status": "running"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "        </style>\n",
       "      \n",
       "      <progress value='1303' max='16179' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [ 1303/16179 14:12 < 2:42:24, 1.53 it/s, Epoch 0.24/3]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "incorporate-highland",
   "metadata": {
    "id": "CKASz-2vIrJi",
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "source": [
    "We can check with the `evaluate` method that our `Trainer` did reload the best model properly (if it was not the last one):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "concerned-sector",
   "metadata": {
    "id": "UOUcBkX8IrJi",
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "trainer.evaluate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "preliminary-concentration",
   "metadata": {
    "id": "IKPDobAitsTH",
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "res = trainer.predict(encoded_dataset['test'])\n",
    "res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "basic-orchestra",
   "metadata": {
    "id": "lIQre9p2uib3",
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df_res = pd.DataFrame(np.argmax(res.predictions, axis=1), index=test_data.index)\n",
    "df_res.index = df_res.index.rename('Id')\n",
    "df_res.columns = ['Predicted']\n",
    "df_res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "young-intensity",
   "metadata": {
    "id": "AhgwJbFtyAcB",
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df_res.to_csv('./submission-xlm-roberta-large-v1.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dried-religious",
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "args = TrainingArguments(\n",
    "    \"test-RoBERTa\",\n",
    "    evaluation_strategy = \"epoch\",\n",
    "    learning_rate=2e-5,\n",
    "    per_device_train_batch_size=batch_size,\n",
    "    per_device_eval_batch_size=batch_size,\n",
    "    num_train_epochs=3,\n",
    "    weight_decay=0.01,\n",
    "    load_best_model_at_end=True,\n",
    "    metric_for_best_model=metric_name,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "emerging-circular",
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from transformers import pipeline\n",
    "\n",
    "pipeline('feature-extraction', model=model, tokenizer=tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "integral-beast",
   "metadata": {
    "id": "ag-xMqxnybn9",
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import shutil\n",
    "\n",
    "shutil.copytree('./test-RoBERTa', '/content/drive/MyDrive/contest/models/RoBERTa-large')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "younger-gospel",
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "source": [
    "<a href=\"./test-RoBERTa\"> Download File </a>"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "billion-board",
   "metadata": {
    "id": "7k8ge1L1IrJk",
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "source": [
    "## Hyperparameter search"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "southern-lodge",
   "metadata": {
    "id": "RNfajuw_IrJl",
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "source": [
    "The `Trainer` supports hyperparameter search using [optuna](https://optuna.org/) or [Ray Tune](https://docs.ray.io/en/latest/tune/). For this last section you will need either of those libraries installed, just uncomment the line you want on the next cell and run it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "electric-green",
   "metadata": {
    "id": "YUdakNBhIrJl",
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# ! pip install optuna\n",
    "# ! pip install ray[tune]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "neural-justice",
   "metadata": {
    "id": "ttfT0CqaIrJm",
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "source": [
    "During hyperparameter search, the `Trainer` will run several trainings, so it needs to have the model defined via a function (so it can be reinitialized at each new run) instead of just having it passed. We jsut use the same function as before:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "prostate-living",
   "metadata": {
    "id": "8sgjdLKcIrJm",
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def model_init():\n",
    "    return AutoModelForSequenceClassification.from_pretrained(model_checkpoint, num_labels=num_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "attended-scott",
   "metadata": {
    "id": "mMXfVJO4IrJo",
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "source": [
    "And we can instantiate our `Trainer` like before:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "progressive-chile",
   "metadata": {
    "id": "71pt6N0eIrJo",
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "trainer = Trainer(\n",
    "    model_init=model_init,\n",
    "    args=args,\n",
    "    train_dataset=encoded_dataset[\"train\"],\n",
    "    eval_dataset=encoded_dataset[validation_key],\n",
    "    tokenizer=tokenizer,\n",
    "    compute_metrics=compute_metrics\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "quiet-throw",
   "metadata": {
    "id": "yQxrzFP4IrJq",
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "source": [
    "The method we call this time is `hyperparameter_search`. Note that it can take a long time to run on the full dataset for some of the tasks. You can try to find some good hyperparameter on a portion of the training dataset by replacing the `train_dataset` line above by:\n",
    "```python\n",
    "train_dataset = encoded_dataset[\"train\"].shard(index=1, num_shards=10) \n",
    "```\n",
    "for 1/10th of the dataset. Then you can run a full training on the best hyperparameters picked by the search."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "macro-pavilion",
   "metadata": {
    "id": "NboJ7kDOIrJq",
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "best_run = trainer.hyperparameter_search(n_trials=10, direction=\"maximize\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "sealed-plain",
   "metadata": {
    "id": "gUTD72qCIrJs",
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "source": [
    "The `hyperparameter_search` method returns a `BestRun` objects, which contains the value of the objective maximized (by default the sum of all metrics) and the hyperparameters it used for that run."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "athletic-doctrine",
   "metadata": {
    "id": "Psi4JymeIrJs",
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "best_run"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "unlike-drawing",
   "metadata": {
    "id": "dFdjWbRIIrJu",
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "source": [
    "You can customize the objective to maximize by passing along a `compute_objective` function to the `hyperparameter_search` method, and you can customize the search space by passing a `hp_space` argument to `hyperparameter_search`. See this [forum post](https://discuss.huggingface.co/t/using-hyperparameter-search-in-trainer/785/10) for some examples.\n",
    "\n",
    "To reproduce the best training, just set the hyperparameters in your `TrainingArgument` before creating a `Trainer`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "radio-carry",
   "metadata": {
    "id": "EsJ6sqdGIrJu",
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "for n, v in best_run.hyperparameters.items():\n",
    "    setattr(trainer.args, n, v)\n",
    "\n",
    "trainer.train()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "internal-strand",
   "metadata": {
    "id": "m09tZgZRIrJv",
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "source": [
    "Don't forget to [update your model](https://huggingface.co/transformers/model_sharing.html) on the [🤗 Model Hub](https://huggingface.co/models). You can then use it only to generate results like the one shown in the first picture of this notebook!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "prescription-conservation",
   "metadata": {
    "id": "I3_YZCzNchYf",
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  },
  "papermill": {
   "default_parameters": {},
   "duration": null,
   "end_time": null,
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2021-03-26T11:44:43.388670",
   "version": "2.3.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}