notebooks/Transformers.ipynb

{
 "cells": [
  {
   "cell_type": "raw",
   "id": "a9642f4e-add8-4ee9-9ba9-79f5cf150e9a",
   "metadata": {},
   "source": [
    "https://huggingface.co/learn/nlp-course/chapter0/1?fw=pt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c839f711-09c1-4566-a64f-5c24e7b04a23",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Looking in indexes: https://pypi.org/simple, https://pypi.ngc.nvidia.com\n",
      "Collecting transformers\n",
      "  Downloading transformers-4.30.2-py3-none-any.whl (7.2 MB)\n",
      "     ---------------------------------------- 0.0/7.2 MB ? eta -:--:--\n",
      "     -- ------------------------------------- 0.4/7.2 MB 8.3 MB/s eta 0:00:01\n",
      "     ----- ---------------------------------- 1.0/7.2 MB 11.0 MB/s eta 0:00:01\n",
      "     ---------- ----------------------------- 1.8/7.2 MB 13.1 MB/s eta 0:00:01\n",
      "     --------------- ------------------------ 2.9/7.2 MB 15.2 MB/s eta 0:00:01\n",
      "     ----------------------- ---------------- 4.2/7.2 MB 18.0 MB/s eta 0:00:01\n",
      "     --------------------------------- ------ 6.0/7.2 MB 21.4 MB/s eta 0:00:01\n",
      "     ---------------------------------------- 7.2/7.2 MB 24.1 MB/s eta 0:00:00\n",
      "Collecting huggingface-hub<1.0,>=0.14.1\n",
      "  Downloading huggingface_hub-0.15.1-py3-none-any.whl (236 kB)\n",
      "     ---------------------------------------- 0.0/236.8 kB ? eta -:--:--\n",
      "     ---------------------------------------- 236.8/236.8 kB ? eta 0:00:00\n",
      "Requirement already satisfied: requests in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from transformers) (2.28.1)\n",
      "Collecting regex!=2019.12.17\n",
      "  Downloading regex-2023.6.3-cp39-cp39-win_amd64.whl (268 kB)\n",
      "     ---------------------------------------- 0.0/268.1 kB ? eta -:--:--\n",
      "     ---------------------------------------- 268.1/268.1 kB ? eta 0:00:00\n",
      "Collecting safetensors>=0.3.1\n",
      "  Downloading safetensors-0.3.1-cp39-cp39-win_amd64.whl (263 kB)\n",
      "     ---------------------------------------- 0.0/263.9 kB ? eta -:--:--\n",
      "     ---------------------------------------- 263.9/263.9 kB ? eta 0:00:00\n",
      "Requirement already satisfied: numpy>=1.17 in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from transformers) (1.23.5)\n",
      "Requirement already satisfied: filelock in c:\\users\\lkk68\\appdata\\roaming\\python\\python39\\site-packages (from transformers) (3.4.2)\n",
      "Requirement already satisfied: tqdm>=4.27 in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from transformers) (4.65.0)\n",
      "Collecting tokenizers!=0.11.3,<0.14,>=0.11.1\n",
      "  Downloading tokenizers-0.13.3-cp39-cp39-win_amd64.whl (3.5 MB)\n",
      "     ---------------------------------------- 0.0/3.5 MB ? eta -:--:--\n",
      "     ------------------------------- -------- 2.7/3.5 MB 85.4 MB/s eta 0:00:01\n",
      "     ---------------------------------------- 3.5/3.5 MB 73.6 MB/s eta 0:00:00\n",
      "Requirement already satisfied: pyyaml>=5.1 in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from transformers) (6.0)\n",
      "Requirement already satisfied: packaging>=20.0 in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from transformers) (23.0)\n",
      "Collecting fsspec\n",
      "  Downloading fsspec-2023.6.0-py3-none-any.whl (163 kB)\n",
      "     ---------------------------------------- 0.0/163.8 kB ? eta -:--:--\n",
      "     ---------------------------------------- 163.8/163.8 kB ? eta 0:00:00\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from huggingface-hub<1.0,>=0.14.1->transformers) (4.4.0)\n",
      "Requirement already satisfied: colorama in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from tqdm>=4.27->transformers) (0.4.6)\n",
      "Requirement already satisfied: idna<4,>=2.5 in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from requests->transformers) (3.4)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from requests->transformers) (2022.12.7)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from requests->transformers) (1.26.14)\n",
      "Requirement already satisfied: charset-normalizer<3,>=2 in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from requests->transformers) (2.0.4)\n",
      "Installing collected packages: tokenizers, safetensors, regex, fsspec, huggingface-hub, transformers\n",
      "  Attempting uninstall: huggingface-hub\n",
      "    Found existing installation: huggingface-hub 0.13.3\n",
      "    Uninstalling huggingface-hub-0.13.3:\n",
      "      Successfully uninstalled huggingface-hub-0.13.3\n",
      "Successfully installed fsspec-2023.6.0 huggingface-hub-0.15.1 regex-2023.6.3 safetensors-0.3.1 tokenizers-0.13.3 transformers-4.30.2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
      "opencood 0.1.0 requires matplotlib~=3.3.3, but you have matplotlib 3.7.1 which is incompatible.\n",
      "opencood 0.1.0 requires opencv-python~=4.5.1.48, but you have opencv-python 4.7.0.72 which is incompatible.\n",
      "opencood 0.1.0 requires scipy~=1.5.4, but you have scipy 1.10.1 which is incompatible.\n"
     ]
    }
   ],
   "source": [
    "!pip install transformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "0b4a71c5-83b2-46ae-9d71-af1a6268bc59",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import transformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "4c5a0788-6a66-4c38-9d4a-6e0d37908d46",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No model was supplied, defaulted to distilbert-base-uncased-finetuned-sst-2-english and revision af0f99b (https://huggingface.co/distilbert-base-uncased-finetuned-sst-2-english).\n",
      "Using a pipeline without specifying a model name and revision in production is not recommended.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6a4116a6137b4ec393eeb1677a04c14b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)lve/main/config.json:   0%|          | 0.00/629 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "77a93e2dd54945d08d5ea503b7004562",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading model.safetensors:   0%|          | 0.00/268M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5782ad46f5a743c5be423362d7225731",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)okenizer_config.json:   0%|          | 0.00/48.0 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "de278db2d9084f43836592d9c690d347",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)solve/main/vocab.txt:   0%|          | 0.00/232k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Xformers is not installed correctly. If you want to use memory_efficient_attention to accelerate training use the following command to install Xformers\n",
      "pip install xformers.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[{'label': 'POSITIVE', 'score': 0.9598049521446228}]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from transformers import pipeline\n",
    "\n",
    "classifier = pipeline(\"sentiment-analysis\")\n",
    "classifier(\"I've been waiting for a HuggingFace course my whole life.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "b59ceaec-392f-4013-821d-31c856aa9550",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'label': 'POSITIVE', 'score': 0.9598049521446228},\n",
       " {'label': 'NEGATIVE', 'score': 0.9994558691978455}]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "classifier(\n",
    "    [\"I've been waiting for a HuggingFace course my whole life.\", \"I hate this so much!\"]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "517f65b4-db14-4ef0-8564-7e8b5026979e",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No model was supplied, defaulted to facebook/bart-large-mnli and revision c626438 (https://huggingface.co/facebook/bart-large-mnli).\n",
      "Using a pipeline without specifying a model name and revision in production is not recommended.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ce604d2f3ceb405d92d073945b911a6a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)lve/main/config.json:   0%|          | 0.00/1.15k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c48996958bdf48d4b54e7618f516f578",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading model.safetensors:   0%|          | 0.00/1.63G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2c8d065bcf7645aea83c0d54d09301de",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)okenizer_config.json:   0%|          | 0.00/26.0 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d320ec02ad8f42148df34258cd5c2129",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)olve/main/vocab.json:   0%|          | 0.00/899k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fd3a59c8daee4d1ea60cb40c80e6375f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)olve/main/merges.txt:   0%|          | 0.00/456k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "598c86d2c1664d0c9a69072050cf995f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)/main/tokenizer.json:   0%|          | 0.00/1.36M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'sequence': 'This is a course about the Transformers library',\n",
       " 'labels': ['education', 'business', 'politics'],\n",
       " 'scores': [0.8445993065834045, 0.11197393387556076, 0.043426718562841415]}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from transformers import pipeline\n",
    "\n",
    "classifier = pipeline(\"zero-shot-classification\")\n",
    "classifier(\n",
    "    \"This is a course about the Transformers library\",\n",
    "    candidate_labels=[\"education\", \"politics\", \"business\"],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "5ee2d439-c249-49ed-95d1-364b5e749ca1",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No model was supplied, defaulted to gpt2 and revision 6c0e608 (https://huggingface.co/gpt2).\n",
      "Using a pipeline without specifying a model name and revision in production is not recommended.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "beabb9904ff24fb3acc1c801a01f2874",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)lve/main/config.json:   0%|          | 0.00/665 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2be7c4871a0d407e917864a5d2109a28",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading model.safetensors:   0%|          | 0.00/548M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f2f2bae47b494650b75583b6112aa504",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)neration_config.json:   0%|          | 0.00/124 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "68438cfd11284b5fa757746304e155a0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)olve/main/vocab.json:   0%|          | 0.00/1.04M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e80049e66d3342a68c5d82f85648da6b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)olve/main/merges.txt:   0%|          | 0.00/456k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3ed4189ce9404799859a50d7db27951a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)/main/tokenizer.json:   0%|          | 0.00/1.36M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
      "C:\\Users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages\\transformers\\generation\\utils.py:1353: UserWarning: Using `max_length`'s default (50) to control the generation length. This behaviour is deprecated and will be removed from the config in v5 of Transformers -- we recommend using `max_new_tokens` to control the maximum length of the generation.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[{'generated_text': 'In this course, we will teach you how to create a virtualization scenario where you can run a virtualization environment, without creating a virtual machine and run it as a virtual machine. It can also be used to create a virtual machine and run it'}]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from transformers import pipeline\n",
    "\n",
    "generator = pipeline(\"text-generation\")\n",
    "generator(\"In this course, we will teach you how to\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "c7337857-f763-42d3-969c-84cefbb3013b",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[{'generated_text': 'How to learn pytorch: How do you teach pytorch? (by Tami McConaghy)\\n\\nby Tami Mc Davy\\n\\nYou could do a job where you were paid that.\\n\\nIf you have'}]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "generator(\"How to learn pytorch\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "840d0a9b-3458-4307-a586-8d75241a3aae",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "db43ec6944e646d0b9f9b96525ae89f2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)lve/main/config.json:   0%|          | 0.00/762 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e841036df40746bdacc21338ea4eba8f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading model.safetensors:   0%|          | 0.00/353M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bdf7df4be0034c36a1a84e889ef118a0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)neration_config.json:   0%|          | 0.00/124 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "be52317c9ca2499eb5e80528ac177939",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)olve/main/vocab.json:   0%|          | 0.00/1.04M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c550a985cefa438f89f42c7f66d8f25f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)olve/main/merges.txt:   0%|          | 0.00/456k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e0ea12612e4043f192f4e9bd29245478",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)/main/tokenizer.json:   0%|          | 0.00/1.36M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[{'generated_text': 'In this course, we will teach you how to get started and where to find help from local police departments. Students will also look for support in the'},\n",
       " {'generated_text': 'In this course, we will teach you how to use some of the tools we offer at your university. By learning to get your hands on tools from'}]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from transformers import pipeline\n",
    "\n",
    "generator = pipeline(\"text-generation\", model=\"distilgpt2\")\n",
    "generator(\n",
    "    \"In this course, we will teach you how to\",\n",
    "    max_length=30,\n",
    "    num_return_sequences=2,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "ee21d437-f783-4ad4-ae23-126653c9bc9c",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No model was supplied, defaulted to distilroberta-base and revision ec58a5b (https://huggingface.co/distilroberta-base).\n",
      "Using a pipeline without specifying a model name and revision in production is not recommended.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d95017a02d6e4ab883173e17852a3c49",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)lve/main/config.json:   0%|          | 0.00/480 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4b112e663c874799b71ff4a85f0e20cc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading model.safetensors:   0%|          | 0.00/331M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ba0faacc783c4a3dbc2941bde4423444",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)olve/main/vocab.json:   0%|          | 0.00/899k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a5049018b8304587b696daedc08e3100",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)olve/main/merges.txt:   0%|          | 0.00/456k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e37d0536b5b4450c90e8b04711ad13f7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)/main/tokenizer.json:   0%|          | 0.00/1.36M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "[{'score': 0.1961977779865265,\n",
       "  'token': 30412,\n",
       "  'token_str': ' mathematical',\n",
       "  'sequence': 'This course will teach you all about mathematical models.'},\n",
       " {'score': 0.04052717983722687,\n",
       "  'token': 38163,\n",
       "  'token_str': ' computational',\n",
       "  'sequence': 'This course will teach you all about computational models.'}]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from transformers import pipeline\n",
    "\n",
    "unmasker = pipeline(\"fill-mask\")\n",
    "unmasker(\"This course will teach you all about <mask> models.\", top_k=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "2317f1f0-59a4-4d72-b130-d7b2605a68bc",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No model was supplied, defaulted to dbmdz/bert-large-cased-finetuned-conll03-english and revision f2482bf (https://huggingface.co/dbmdz/bert-large-cased-finetuned-conll03-english).\n",
      "Using a pipeline without specifying a model name and revision in production is not recommended.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "019a6981ba86498cb94aaab1bf097afc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)lve/main/config.json:   0%|          | 0.00/998 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6b012fc38be049f78b5e869d9162480d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading pytorch_model.bin:   0%|          | 0.00/1.33G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "91df82c3c5094306908de6a856657d1e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)okenizer_config.json:   0%|          | 0.00/60.0 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "45c4d4924b9946fa9193aae754cf9141",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)solve/main/vocab.txt:   0%|          | 0.00/213k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages\\transformers\\pipelines\\token_classification.py:169: UserWarning: `grouped_entities` is deprecated and will be removed in version v5.0.0, defaulted to `aggregation_strategy=\"simple\"` instead.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[{'entity_group': 'PER',\n",
       "  'score': 0.9981694,\n",
       "  'word': 'Sylvain',\n",
       "  'start': 11,\n",
       "  'end': 18},\n",
       " {'entity_group': 'ORG',\n",
       "  'score': 0.9796019,\n",
       "  'word': 'Hugging Face',\n",
       "  'start': 33,\n",
       "  'end': 45},\n",
       " {'entity_group': 'LOC',\n",
       "  'score': 0.9932106,\n",
       "  'word': 'Brooklyn',\n",
       "  'start': 49,\n",
       "  'end': 57}]"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from transformers import pipeline\n",
    "\n",
    "ner = pipeline(\"ner\", grouped_entities=True)\n",
    "ner(\"My name is Sylvain and I work at Hugging Face in Brooklyn.\")"
   ]
  },
  {
   "cell_type": "raw",
   "id": "1732a91d-4e52-4e9b-b96d-29a10f5e7c99",
   "metadata": {},
   "source": [
    "pass the option grouped_entities=True in the pipeline creation function to tell the pipeline to regroup together the parts of the sentence that correspond to the same entity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "0d0ba59d-62f6-42f6-86d6-cb5885c4adbe",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No model was supplied, defaulted to distilbert-base-cased-distilled-squad and revision 626af31 (https://huggingface.co/distilbert-base-cased-distilled-squad).\n",
      "Using a pipeline without specifying a model name and revision in production is not recommended.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0ae4eb8e994842479a3be480969764ab",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)lve/main/config.json:   0%|          | 0.00/473 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0202dd2b61cc454c8ff171712e226fb4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading model.safetensors:   0%|          | 0.00/261M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "623ec5fdb66b43a4bc8745d06b6e3c13",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)okenizer_config.json:   0%|          | 0.00/29.0 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4a4102df69584f528fc0c20b5fcf5a93",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)solve/main/vocab.txt:   0%|          | 0.00/213k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fb86539b1d904348bfc650bf00561b34",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)/main/tokenizer.json:   0%|          | 0.00/436k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'score': 0.6949772238731384, 'start': 33, 'end': 45, 'answer': 'Hugging Face'}"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from transformers import pipeline\n",
    "\n",
    "question_answerer = pipeline(\"question-answering\")\n",
    "question_answerer(\n",
    "    question=\"Where do I work?\",\n",
    "    context=\"My name is Sylvain and I work at Hugging Face in Brooklyn\",\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "fc568799-79f3-413f-95c3-3f156bda7326",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No model was supplied, defaulted to sshleifer/distilbart-cnn-12-6 and revision a4f8f3e (https://huggingface.co/sshleifer/distilbart-cnn-12-6).\n",
      "Using a pipeline without specifying a model name and revision in production is not recommended.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "56ae5461bc88402d90c7a2160a8bab02",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)lve/main/config.json:   0%|          | 0.00/1.80k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "960d5d2e1c4d4d0493f547d6df04be3e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading pytorch_model.bin:   0%|          | 0.00/1.22G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "aa5bc8d959de452d8aa7c3f267e15701",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)okenizer_config.json:   0%|          | 0.00/26.0 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6418899dd20248efaefdfbdec223d85e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)olve/main/vocab.json:   0%|          | 0.00/899k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1a01492e0ce24afba1de534422153948",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)olve/main/merges.txt:   0%|          | 0.00/456k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "[{'summary_text': ' America has changed dramatically during recent years . The number of engineering graduates in the U.S. has declined in traditional engineering disciplines such as mechanical, civil,    electrical, chemical, and aeronautical engineering . Rapidly developing economies such as China and India continue to encourage and advance the teaching of engineering .'}]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from transformers import pipeline\n",
    "\n",
    "summarizer = pipeline(\"summarization\")\n",
    "summarizer(\n",
    "    \"\"\"\n",
    "    America has changed dramatically during recent years. Not only has the number of \n",
    "    graduates in traditional engineering disciplines such as mechanical, civil, \n",
    "    electrical, chemical, and aeronautical engineering declined, but in most of \n",
    "    the premier American universities engineering curricula now concentrate on \n",
    "    and encourage largely the study of engineering science. As a result, there \n",
    "    are declining offerings in engineering subjects dealing with infrastructure, \n",
    "    the environment, and related issues, and greater concentration on high \n",
    "    technology subjects, largely supporting increasingly complex scientific \n",
    "    developments. While the latter is important, it should not be at the expense \n",
    "    of more traditional engineering.\n",
    "\n",
    "    Rapidly developing economies such as China and India, as well as other \n",
    "    industrial countries in Europe and Asia, continue to encourage and advance \n",
    "    the teaching of engineering. Both China and India, respectively, graduate \n",
    "    six and eight times as many traditional engineers as does the United States. \n",
    "    Other industrial countries at minimum maintain their output, while America \n",
    "    suffers an increasingly serious decline in the number of engineering graduates \n",
    "    and a lack of well-educated engineers.\n",
    "\"\"\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "036698a6-769e-4fbf-917a-4a6280adb71c",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Looking in indexes: https://pypi.org/simple, https://pypi.ngc.nvidia.com\n",
      "Collecting sentencepiece\n",
      "  Downloading sentencepiece-0.1.99-cp39-cp39-win_amd64.whl (977 kB)\n",
      "     ---------------------------------------- 0.0/977.6 kB ? eta -:--:--\n",
      "     -------------------------------- ---- 860.2/977.6 kB 18.1 MB/s eta 0:00:01\n",
      "     ------------------------------------- 977.6/977.6 kB 20.6 MB/s eta 0:00:00\n",
      "Installing collected packages: sentencepiece\n",
      "Successfully installed sentencepiece-0.1.99\n"
     ]
    }
   ],
   "source": [
    "!pip install sentencepiece #restart jupyter kernel after installation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ddaf032b-fd14-47a8-b855-f758a9d5d641",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4cf8b97e4c254051b6c15d7e8d397a9b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading source.spm:   0%|          | 0.00/806k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e3f1bdd80adf49ffba18fd3a3f8fa52d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading target.spm:   0%|          | 0.00/805k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e3148a9009104248bca97ebf23a46c5f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading vocab.json:   0%|          | 0.00/1.75M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bb5f87754d7748a49a5e940c2421abcc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)cial_tokens_map.json:   0%|          | 0.00/74.0 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages\\transformers\\models\\marian\\tokenization_marian.py:194: UserWarning: Recommended: pip install sacremoses.\n",
      "  warnings.warn(\"Recommended: pip install sacremoses.\")\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[{'translation_text': '我的名字是 Gong,我住在加利福尼亚州。'}]"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from transformers import pipeline\n",
    "#https://huggingface.co/DDDSSS/translation_en-zh\n",
    "translator = pipeline(\"translation\", model=\"DDDSSS/translation_en-zh\")\n",
    "translator(\"My name is Gang and I live in California.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c4ddaf26-dbd9-44cf-b06e-8bf87047a344",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'translation_text': '当今,实质滥用和心理健康服务管理局发布了国家模型标准,帮助他人通过自己的实践经验解决药物使用或心理健康问题。自2015年SAMHSA发布Behvior Health服务中,有41个州认可或运行同行认证计划。国家模型标准并不是旨在取代任何国家认证,而是国家和其他国家的指导,“促进质量,并鼓励往往异端的州同行支持认证之间的对齐和互惠”。'}]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "translator(\"The Substance Abuse and Mental Health Services Administration today released National Model Standards for Peer Support Certification for people who help others address a substance use or mental health issue through their own lived experience. Forty-nine states have endorsed or run peer certification programs since SAMHSA released Core Competencies for Peer Workers in Behavioral Health Services in 2015. The national model standards are not intended as a substitute for any state certifications, but guidance for states and others “to promote quality and encourage alignment and reciprocity across often disparate state peer support certifications,” the agency said.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f92ccded-352d-4700-96c2-5720d320410f",
   "metadata": {},
   "source": [
    "# Behind the Pipeline\n",
    "https://huggingface.co/learn/nlp-course/chapter2/2?fw=pt"
   ]
  },
  {
   "cell_type": "raw",
   "id": "4aa2cc67-f91e-41c4-860a-8fed283822b2",
   "metadata": {},
   "source": [
    "All this preprocessing needs to be done in exactly the same way as when the model was pretrained, so we first need to download that information from the Model Hub. To do this, we use the AutoTokenizer class and its from_pretrained() method. Using the checkpoint name of our model, it will automatically fetch the data associated with the model’s tokenizer and cache it "
   ]
  },
  {
   "cell_type": "raw",
   "id": "28e76d1b-140e-4aae-bcf8-c4863dcf5893",
   "metadata": {},
   "source": [
    "The default checkpoint of the sentiment-analysis pipeline is distilbert-base-uncased-finetuned-sst-2-english \n",
    "ref: https://huggingface.co/distilbert-base-uncased-finetuned-sst-2-english"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b5e7de63-03c1-4896-a00b-dafa2030e4d2",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from transformers import AutoTokenizer\n",
    "\n",
    "checkpoint = \"distilbert-base-uncased-finetuned-sst-2-english\"\n",
    "tokenizer = AutoTokenizer.from_pretrained(checkpoint)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "4eba661c-b665-472c-971e-e88d7c8b6051",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'input_ids': tensor([[  101,  1045,  1005,  2310,  2042,  3403,  2005,  1037, 17662, 12172,\n",
      "          2607,  2026,  2878,  2166,  1012,   102],\n",
      "        [  101,  1045,  5223,  2023,  2061,  2172,   999,   102,     0,     0,\n",
      "             0,     0,     0,     0,     0,     0]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],\n",
      "        [1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0]])}\n"
     ]
    }
   ],
   "source": [
    "raw_inputs = [\n",
    "    \"I've been waiting for a HuggingFace course my whole life.\",\n",
    "    \"I hate this so much!\",\n",
    "]\n",
    "inputs = tokenizer(raw_inputs, padding=True, truncation=True, return_tensors=\"pt\")\n",
    "print(inputs)\n",
    "#To specify the type of tensors we want to get back (PyTorch, TensorFlow, or plain NumPy), we use the return_tensors argument:"
   ]
  },
  {
   "cell_type": "raw",
   "id": "3bf27060-9c4f-443e-92c8-cb14b9f07691",
   "metadata": {},
   "source": [
    "The output itself is a dictionary containing two keys, input_ids and attention_mask. input_ids contains two rows of integers (one for each sentence) that are the unique identifiers of the tokens in each sentence."
   ]
  },
  {
   "cell_type": "raw",
   "id": "18d898aa-cc51-4b58-9533-3a9a1fd016fb",
   "metadata": {},
   "source": [
    "Transformers provides an AutoModel class which also has a from_pretrained() method:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "7e7dc13e-f42e-4301-aee5-056c8b75bacf",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at distilbert-base-uncased-finetuned-sst-2-english were not used when initializing DistilBertModel: ['pre_classifier.bias', 'pre_classifier.weight', 'classifier.weight', 'classifier.bias']\n",
      "- This IS expected if you are initializing DistilBertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing DistilBertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoModel\n",
    "\n",
    "checkpoint = \"distilbert-base-uncased-finetuned-sst-2-english\"\n",
    "model = AutoModel.from_pretrained(checkpoint)"
   ]
  },
  {
   "cell_type": "raw",
   "id": "45c7eac6-bab9-4cca-adfd-417f34b2c829",
   "metadata": {},
   "source": [
    "This architecture contains only the base Transformer module: given some inputs, it outputs what we’ll call hidden states, also known as features.\n",
    "For each model input, we’ll retrieve a high-dimensional vector representing the contextual understanding of that input by the Transformer model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "35a69482-0f91-40db-bb11-d1e9a058931f",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([2, 16, 768])\n"
     ]
    }
   ],
   "source": [
    "outputs = model(**inputs)\n",
    "print(outputs.last_hidden_state.shape)"
   ]
  },
  {
   "cell_type": "raw",
   "id": "59af5251-cada-4fa0-9440-b33a8388841e",
   "metadata": {},
   "source": [
    "the outputs of 🤗 Transformers models behave like namedtuples or dictionaries. You can access the elements by attributes (like we did) or by key (outputs[\"last_hidden_state\"]), or even by index if you know exactly where the thing you are looking for is (outputs[0])."
   ]
  },
  {
   "cell_type": "raw",
   "id": "36880c1b-bcd0-4153-8323-e60c94620627",
   "metadata": {},
   "source": [
    "we will need a model with a sequence classification head (to be able to classify the sentences as positive or negative). So, we won’t actually use the AutoModel class, but AutoModelForSequenceClassification:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "4e34fbfc-b54b-49cf-a4de-75a905778995",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from transformers import AutoModelForSequenceClassification\n",
    "\n",
    "checkpoint = \"distilbert-base-uncased-finetuned-sst-2-english\"\n",
    "model = AutoModelForSequenceClassification.from_pretrained(checkpoint)\n",
    "outputs = model(**inputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "5b3fa00b-7ab3-44e6-b36e-cd2bf0c178bc",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([2, 2])\n"
     ]
    }
   ],
   "source": [
    "print(outputs.logits.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "f7058673-bf36-4ee2-986f-45ddb1e1ed7d",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[-1.5607,  1.6123],\n",
      "        [ 4.1692, -3.3464]], grad_fn=<AddmmBackward0>)\n"
     ]
    }
   ],
   "source": [
    "print(outputs.logits)"
   ]
  },
  {
   "cell_type": "raw",
   "id": "b22d3fd3-7c33-45b5-aff9-a017684ff82b",
   "metadata": {},
   "source": [
    "Input is two sentences and two labels, the result we get from our model is of shape 2 x 2"
   ]
  },
  {
   "cell_type": "raw",
   "id": "7925c7a9-b4ee-4058-ac0e-4283df98620c",
   "metadata": {},
   "source": [
    "Our model predicted [-1.5607, 1.6123] for the first sentence and [ 4.1692, -3.3464] for the second one. Those are not probabilities but logits, the raw, unnormalized scores outputted by the last layer of the model. To be converted to probabilities, they need to go through a SoftMax layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "2f0663c1-4cff-4ca3-97a5-dd23c87f5936",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[4.0195e-02, 9.5980e-01],\n",
      "        [9.9946e-01, 5.4418e-04]], grad_fn=<SoftmaxBackward0>)\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "predictions = torch.nn.functional.softmax(outputs.logits, dim=-1)\n",
    "print(predictions)"
   ]
  },
  {
   "cell_type": "raw",
   "id": "55057b82-d31c-4ae6-9efd-156af6174ea8",
   "metadata": {},
   "source": [
    "we can see that the model predicted [0.0402, 0.9598] for the first sentence and [0.9995, 0.0005] for the second one. These are recognizable probability scores."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "530f0d7b-c39e-4efc-9a34-6a718a860c4f",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: 'NEGATIVE', 1: 'POSITIVE'}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.config.id2label"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "007ef3fd-7da6-4cc9-8b12-499e65716dc4",
   "metadata": {},
   "source": [
    "# Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "904f6db9-0364-41b0-81c7-7db656e7bcb0",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sat Jun 17 20:05:06 2023       \n",
      "+-----------------------------------------------------------------------------+\n",
      "| NVIDIA-SMI 528.24       Driver Version: 528.24       CUDA Version: 12.0     |\n",
      "|-------------------------------+----------------------+----------------------+\n",
      "| GPU  Name            TCC/WDDM | Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
      "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
      "|                               |                      |               MIG M. |\n",
      "|===============================+======================+======================|\n",
      "|   0  NVIDIA GeForce ... WDDM  | 00000000:01:00.0 Off |                  N/A |\n",
      "|  0%   40C    P8    19W / 350W |    967MiB / 24576MiB |      0%      Default |\n",
      "|                               |                      |                  N/A |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "                                                                               \n",
      "+-----------------------------------------------------------------------------+\n",
      "| Processes:                                                                  |\n",
      "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
      "|        ID   ID                                                   Usage      |\n",
      "|=============================================================================|\n",
      "|    0   N/A  N/A      1532    C+G   ...e\\Current\\LogiOverlay.exe    N/A      |\n",
      "|    0   N/A  N/A      2044    C+G   ...dows\\System32\\LogonUI.exe    N/A      |\n",
      "|    0   N/A  N/A      2056    C+G   C:\\Windows\\System32\\dwm.exe     N/A      |\n",
      "|    0   N/A  N/A      2368    C+G   ...oft\\OneDrive\\OneDrive.exe    N/A      |\n",
      "|    0   N/A  N/A      7152    C+G   ...y\\ShellExperienceHost.exe    N/A      |\n",
      "|    0   N/A  N/A      9444    C+G   ...logioptionsplus_agent.exe    N/A      |\n",
      "|    0   N/A  N/A     10568    C+G   C:\\Windows\\explorer.exe         N/A      |\n",
      "|    0   N/A  N/A     11968    C+G   ...lPanel\\SystemSettings.exe    N/A      |\n",
      "|    0   N/A  N/A     13392    C+G   ...n1h2txyewy\\SearchHost.exe    N/A      |\n",
      "|    0   N/A  N/A     13420    C+G   ...artMenuExperienceHost.exe    N/A      |\n",
      "|    0   N/A  N/A     13676    C+G   ...8wekyb3d8bbwe\\GameBar.exe    N/A      |\n",
      "|    0   N/A  N/A     13996    C+G   ...perience\\NVIDIA Share.exe    N/A      |\n",
      "|    0   N/A  N/A     15732    C+G   ...e\\PhoneExperienceHost.exe    N/A      |\n",
      "|    0   N/A  N/A     15808    C+G   ...cw5n1h2txyewy\\LockApp.exe    N/A      |\n",
      "|    0   N/A  N/A     17052    C+G   ...2txyewy\\TextInputHost.exe    N/A      |\n",
      "|    0   N/A  N/A     17676    C+G   ...me\\Application\\chrome.exe    N/A      |\n",
      "|    0   N/A  N/A     18316    C+G   ...8wekyb3d8bbwe\\Cortana.exe    N/A      |\n",
      "|    0   N/A  N/A     18916    C+G   ...823.43\\msedgewebview2.exe    N/A      |\n",
      "|    0   N/A  N/A     20444    C+G   ...ontend\\Docker Desktop.exe    N/A      |\n",
      "|    0   N/A  N/A     23572    C+G   ...lienwareMobileConnect.exe    N/A      |\n",
      "|    0   N/A  N/A     25028    C+G   ...8bbwe\\WindowsTerminal.exe    N/A      |\n",
      "|    0   N/A  N/A     25868    C+G   ...reMobileConnectClient.exe    N/A      |\n",
      "|    0   N/A  N/A     27100    C+G   ...oft\\OneDrive\\OneDrive.exe    N/A      |\n",
      "|    0   N/A  N/A     27748    C+G   ...ekyb3d8bbwe\\HxOutlook.exe    N/A      |\n",
      "|    0   N/A  N/A     27792    C+G   ...4__htrsf667h5kn2\\AWCC.exe    N/A      |\n",
      "+-----------------------------------------------------------------------------+\n"
     ]
    }
   ],
   "source": [
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "raw",
   "id": "57ffa2a5-ed88-428c-910e-e0515f01f83c",
   "metadata": {},
   "source": [
    "The AutoModel class and all of its relatives are actually simple wrappers over the wide variety of models available in the library. It’s a clever wrapper as it can automatically guess the appropriate model architecture for your checkpoint, and then instantiates a model with this architecture.\n",
    "\n",
    "However, if you know the type of model you want to use, you can use the class that defines its architecture directly. Let’s take a look at how this works with a BERT model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d5671296-d6da-425f-ba26-025501d09f16",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from transformers import BertConfig, BertModel\n",
    "\n",
    "# Building the config\n",
    "config = BertConfig()\n",
    "\n",
    "# Building the model from the config\n",
    "model = BertModel(config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "1d1b7616-ce72-4679-b7d2-23cfed0910da",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BertConfig {\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"classifier_dropout\": null,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"layer_norm_eps\": 1e-12,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"model_type\": \"bert\",\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"position_embedding_type\": \"absolute\",\n",
      "  \"transformers_version\": \"4.30.2\",\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"use_cache\": true,\n",
      "  \"vocab_size\": 30522\n",
      "}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(config)"
   ]
  },
  {
   "cell_type": "raw",
   "id": "d8b18653-312a-44ef-bd56-567afa9d71aa",
   "metadata": {},
   "source": [
    "Loading a Transformer model that is already trained is simple — we can do this using the from_pretrained() method:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "84ad5faf-d4cf-46f4-8eea-5e9181626057",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e17dadaa52854237acb755ed4632d8f9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)lve/main/config.json:   0%|          | 0.00/570 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e0e635b65f98470391d44fa828d5fa65",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading model.safetensors:   0%|          | 0.00/436M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at bert-base-cased were not used when initializing BertModel: ['cls.predictions.transform.dense.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.bias', 'cls.seq_relationship.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.bias']\n",
      "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    }
   ],
   "source": [
    "from transformers import BertModel\n",
    "\n",
    "model = BertModel.from_pretrained(\"bert-base-cased\")"
   ]
  },
  {
   "cell_type": "raw",
   "id": "0ce529a7-e409-4afa-ac02-1dd4a510cd14",
   "metadata": {},
   "source": [
    "In the code sample above we didn’t use BertConfig, and instead loaded a pretrained model via the bert-base-cased identifier. https://huggingface.co/bert-base-cased"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "0c4e919f-1b73-43a3-ac33-78e92e431ace",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "model.save_pretrained(\"./output/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "619a3c53-f72e-4f0a-80ee-2a487470c3de",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Volume in drive C is OS\n",
      " Volume Serial Number is 5603-1EBA\n",
      "\n",
      " Directory of C:\\Users\\lkk68\\Documents\\GitHub\\DeepDataMiningLearning\\output\n",
      "\n",
      "06/17/2023  08:06 PM    <DIR>          .\n",
      "06/17/2023  08:06 PM    <DIR>          ..\n",
      "06/17/2023  08:06 PM               682 config.json\n",
      "06/17/2023  08:06 PM       433,309,485 pytorch_model.bin\n",
      "               2 File(s)    433,310,167 bytes\n",
      "               2 Dir(s)  215,302,275,072 bytes free\n"
     ]
    }
   ],
   "source": [
    "!dir output"
   ]
  },
  {
   "cell_type": "raw",
   "id": "14d15ef6-4b44-4265-9897-10d423e8ff44",
   "metadata": {},
   "source": [
    "This saves two files to your disk: config.json pytorch_model.bin"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6e601adc-0c2c-45a3-b60c-8564c4e85b59",
   "metadata": {},
   "source": [
    "# Tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "50d67012-d151-4065-9f0c-c822768cb2ee",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Jim', 'Henson', 'was', 'a', 'puppeteer']\n"
     ]
    }
   ],
   "source": [
    "tokenized_text = \"Jim Henson was a puppeteer\".split()\n",
    "print(tokenized_text)"
   ]
  },
  {
   "cell_type": "raw",
   "id": "5abe48fb-909a-4ec5-b89b-c35afe67b0c7",
   "metadata": {},
   "source": [
    "Subword tokenization algorithms rely on the principle that frequently used words should not be split into smaller subwords, but rare words should be decomposed into meaningful subwords."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "8e8c9b2a-5097-4203-b078-67684d44be31",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c0888890a9914c24bb39cdaa01c8311f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)solve/main/vocab.txt:   0%|          | 0.00/213k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5dd95af29b834e1c97130e951603fe98",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)okenizer_config.json:   0%|          | 0.00/29.0 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from transformers import BertTokenizer\n",
    "\n",
    "tokenizer = BertTokenizer.from_pretrained(\"bert-base-cased\")"
   ]
  },
  {
   "cell_type": "raw",
   "id": "a8091a37-2a13-4542-9a5e-84fcaf6aa7e5",
   "metadata": {},
   "source": [
    "Similar to AutoModel, the AutoTokenizer class will grab the proper tokenizer class in the library based on the checkpoint name, and can be used directly with any checkpoint:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "59abc462-7f80-4a56-a51e-b21d75411144",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3aa2191e4e7b4673a28e53fe82d47874",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)/main/tokenizer.json:   0%|          | 0.00/436k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from transformers import AutoTokenizer\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"bert-base-cased\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "26234c52-1351-40e2-aafb-ecef37028506",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'input_ids': [101, 7993, 170, 13809, 23763, 2443, 1110, 3014, 102], 'token_type_ids': [0, 0, 0, 0, 0, 0, 0, 0, 0], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1]}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer(\"Using a Transformer network is simple\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "2fac9352-1565-44b0-90e8-8a53a7e6350c",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('./output\\\\tokenizer_config.json',\n",
       " './output\\\\special_tokens_map.json',\n",
       " './output\\\\vocab.txt',\n",
       " './output\\\\added_tokens.json',\n",
       " './output\\\\tokenizer.json')"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.save_pretrained(\"./output\")"
   ]
  },
  {
   "cell_type": "raw",
   "id": "6dd6cee2-5b59-4d85-932f-55870d1eca66",
   "metadata": {},
   "source": [
    "Translating text to numbers is known as encoding. Encoding is done in a two-step process: the tokenization, followed by the conversion to input IDs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "12f4719c-fb9f-435d-9d2d-d8ce44e0b190",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Using', 'a', 'Trans', '##former', 'network', 'is', 'simple']\n"
     ]
    }
   ],
   "source": [
    "sequence = \"Using a Transformer network is simple\"\n",
    "tokens = tokenizer.tokenize(sequence)\n",
    "\n",
    "print(tokens)"
   ]
  },
  {
   "cell_type": "raw",
   "id": "ccc5c204-5522-499c-8ace-3f2de7f05fd0",
   "metadata": {},
   "source": [
    "This tokenizer is a subword tokenizer: it splits the words until it obtains tokens that can be represented by its vocabulary. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "2b4ff2f9-f38f-4fdf-9ce0-4a62e4e30f74",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[7993, 170, 13809, 23763, 2443, 1110, 3014]\n"
     ]
    }
   ],
   "source": [
    "#The conversion to input IDs is handled by the convert_tokens_to_ids() tokenizer method:\n",
    "ids = tokenizer.convert_tokens_to_ids(tokens)\n",
    "\n",
    "print(ids)"
   ]
  },
  {
   "cell_type": "raw",
   "id": "f5a4d582-6a37-4053-b503-04a1cff281db",
   "metadata": {},
   "source": [
    "Decoding is going the other way around: from vocabulary indices, we want to get a string. This can be done with the decode() method as follows:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "f7e053b0-0056-420f-9a6d-6b4ed19a54ee",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using a transformer network is simple\n"
     ]
    }
   ],
   "source": [
    "decoded_string = tokenizer.decode([7993, 170, 11303, 1200, 2443, 1110, 3014])\n",
    "print(decoded_string)"
   ]
  },
  {
   "cell_type": "raw",
   "id": "6f3c07f8-1027-4703-aa80-f780b7f09338",
   "metadata": {},
   "source": [
    "Handling multiple sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "821ebe94-36ea-4374-9d89-6310688bd552",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input IDs: tensor([[ 1045,  1005,  2310,  2042,  3403,  2005,  1037, 17662, 12172,  2607,\n",
      "          2026,  2878,  2166,  1012]])\n",
      "Logits: tensor([[-2.7276,  2.8789]], grad_fn=<AddmmBackward0>)\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from transformers import AutoTokenizer, AutoModelForSequenceClassification\n",
    "\n",
    "checkpoint = \"distilbert-base-uncased-finetuned-sst-2-english\"\n",
    "tokenizer = AutoTokenizer.from_pretrained(checkpoint)\n",
    "model = AutoModelForSequenceClassification.from_pretrained(checkpoint)\n",
    "\n",
    "sequence = \"I've been waiting for a HuggingFace course my whole life.\"\n",
    "\n",
    "tokens = tokenizer.tokenize(sequence)\n",
    "ids = tokenizer.convert_tokens_to_ids(tokens)\n",
    "\n",
    "input_ids = torch.tensor([ids]) #add a new dimension\n",
    "print(\"Input IDs:\", input_ids)\n",
    "\n",
    "output = model(input_ids)\n",
    "print(\"Logits:\", output.logits)"
   ]
  },
  {
   "cell_type": "raw",
   "id": "a66a7463-f8a4-469f-808a-c44282126f9e",
   "metadata": {},
   "source": [
    "Batching allows the model to work when you feed it multiple sentences. "
   ]
  },
  {
   "cell_type": "raw",
   "id": "16b69005-769e-41d5-9647-33bb0a7311a9",
   "metadata": {},
   "source": [
    "When you’re trying to batch together two (or more) sentences, they might be of different lengths. To work around this problem, we usually pad the inputs. Padding makes sure all our sentences have the same length by adding a special word called the padding token to the sentences with fewer values. The padding token ID can be found in tokenizer.pad_token_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a87a4801-b1a2-4c24-926e-7e5d998d4c79",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[ 1.5694, -1.3895]], grad_fn=<AddmmBackward0>)\n",
      "tensor([[ 0.5803, -0.4125]], grad_fn=<AddmmBackward0>)\n",
      "tensor([[ 1.5694, -1.3895],\n",
      "        [ 1.3374, -1.2163]], grad_fn=<AddmmBackward0>)\n"
     ]
    }
   ],
   "source": [
    "sequence1_ids = [[200, 200, 200]]\n",
    "sequence2_ids = [[200, 200]]\n",
    "batched_ids = [\n",
    "    [200, 200, 200],\n",
    "    [200, 200, tokenizer.pad_token_id],\n",
    "]\n",
    "\n",
    "print(model(torch.tensor(sequence1_ids)).logits)\n",
    "print(model(torch.tensor(sequence2_ids)).logits)\n",
    "print(model(torch.tensor(batched_ids)).logits)"
   ]
  },
  {
   "cell_type": "raw",
   "id": "4798a747-98e7-4f72-a6be-54ca9a861e42",
   "metadata": {},
   "source": [
    "To get the same result when passing individual sentences of different lengths through the model or when passing a batch with the same sentences and padding applied, we need to tell those attention layers to ignore the padding tokens. This is done by using an attention mask."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e5b128e7-c258-4dd2-a54e-26bd9af2ef44",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[ 1.5694, -1.3895],\n",
      "        [ 0.5803, -0.4125]], grad_fn=<AddmmBackward0>)\n"
     ]
    }
   ],
   "source": [
    "batched_ids = [\n",
    "    [200, 200, 200],\n",
    "    [200, 200, tokenizer.pad_token_id],\n",
    "]\n",
    "\n",
    "attention_mask = [\n",
    "    [1, 1, 1],\n",
    "    [1, 1, 0],\n",
    "]\n",
    "\n",
    "outputs = model(torch.tensor(batched_ids), attention_mask=torch.tensor(attention_mask))\n",
    "print(outputs.logits)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "2e0d8f37-0a50-498f-b5bd-9e8fd57dcc0c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from transformers import AutoTokenizer\n",
    "\n",
    "checkpoint = \"distilbert-base-uncased-finetuned-sst-2-english\"\n",
    "tokenizer = AutoTokenizer.from_pretrained(checkpoint)\n",
    "\n",
    "sequence = \"I've been waiting for a HuggingFace course my whole life.\"\n",
    "\n",
    "model_inputs = tokenizer(sequence)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "c4875497-c6fc-48eb-859c-2cc9ee40d6c6",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sequences = [\"I've been waiting for a HuggingFace course my whole life.\", \"So have I!\"]\n",
    "\n",
    "model_inputs = tokenizer(sequences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "5b125fcd-a24c-427f-aade-a149fce75d56",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'input_ids': [[101, 1045, 1005, 2310, 2042, 3403, 2005, 1037, 17662, 12172, 2607, 2026, 2878, 2166, 1012, 102], [101, 2061, 2031, 1045, 999, 102]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1]]}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "ffea8b05-7cdb-4f4c-843e-77b38c936aa0",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "16\n",
      "512\n",
      "16\n"
     ]
    }
   ],
   "source": [
    "# Will pad the sequences up to the maximum sequence length\n",
    "model_inputs = tokenizer(sequences, padding=\"longest\")\n",
    "print(len(model_inputs['input_ids'][0]))\n",
    "\n",
    "# Will pad the sequences up to the model max length\n",
    "# (512 for BERT or DistilBERT)\n",
    "model_inputs = tokenizer(sequences, padding=\"max_length\")\n",
    "print(len(model_inputs['input_ids'][0]))\n",
    "\n",
    "# Will pad the sequences up to the specified max length\n",
    "model_inputs = tokenizer(sequences, padding=\"max_length\", max_length=8)\n",
    "print(len(model_inputs['input_ids'][0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "6e25f79e-b914-45cc-8fbb-b571ce8489d2",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "16\n",
      "8\n"
     ]
    }
   ],
   "source": [
    "sequences = [\"I've been waiting for a HuggingFace course my whole life.\", \"So have I!\"]\n",
    "\n",
    "# Will truncate the sequences that are longer than the model max length\n",
    "# (512 for BERT or DistilBERT)\n",
    "model_inputs = tokenizer(sequences, truncation=True)\n",
    "print(len(model_inputs['input_ids'][0]))\n",
    "\n",
    "# Will truncate the sequences that are longer than the specified max length\n",
    "model_inputs = tokenizer(sequences, max_length=8, truncation=True)\n",
    "print(len(model_inputs['input_ids'][0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "b46561c5-27b7-43f1-8b5a-fbc02bfa41f7",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'transformers.tokenization_utils_base.BatchEncoding'>\n"
     ]
    }
   ],
   "source": [
    "sequences = [\"I've been waiting for a HuggingFace course my whole life.\", \"So have I!\"]\n",
    "\n",
    "# Returns PyTorch tensors\n",
    "model_inputs = tokenizer(sequences, padding=True, return_tensors=\"pt\")\n",
    "print(type(model_inputs))\n",
    "\n",
    "# Returns TensorFlow tensors (Tensorflow need to be installed)\n",
    "# model_inputs = tokenizer(sequences, padding=True, return_tensors=\"tf\")\n",
    "# print(type(model_inputs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "05336943-00f9-4308-8b26-d36d09845f0b",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tokenizers.Encoding"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(model_inputs[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "4ade12e5-73df-4e33-b922-f5de45d6a164",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'transformers.tokenization_utils_base.BatchEncoding'>\n"
     ]
    }
   ],
   "source": [
    "# Returns NumPy arrays\n",
    "model_inputs = tokenizer(sequences, padding=True, return_tensors=\"np\")\n",
    "print(type(model_inputs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "388d6714-aab3-4c12-b6a3-ef1467c0e90b",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[101, 1045, 1005, 2310, 2042, 3403, 2005, 1037, 17662, 12172, 2607, 2026, 2878, 2166, 1012, 102]\n",
      "[1045, 1005, 2310, 2042, 3403, 2005, 1037, 17662, 12172, 2607, 2026, 2878, 2166, 1012]\n"
     ]
    }
   ],
   "source": [
    "sequence = \"I've been waiting for a HuggingFace course my whole life.\"\n",
    "\n",
    "model_inputs = tokenizer(sequence)\n",
    "print(model_inputs[\"input_ids\"])\n",
    "\n",
    "tokens = tokenizer.tokenize(sequence)\n",
    "ids = tokenizer.convert_tokens_to_ids(tokens)\n",
    "print(ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "c4d2839d-a3c4-450a-af6d-02b86dbc776a",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CLS] i've been waiting for a huggingface course my whole life. [SEP]\n",
      "i've been waiting for a huggingface course my whole life.\n"
     ]
    }
   ],
   "source": [
    "print(tokenizer.decode(model_inputs[\"input_ids\"]))\n",
    "print(tokenizer.decode(ids))"
   ]
  },
  {
   "cell_type": "raw",
   "id": "c0e4aafd-2169-4d1a-9ef0-976d4f07456a",
   "metadata": {},
   "source": [
    "The tokenizer added the special word [CLS] at the beginning and the special word [SEP] at the end. This is because the model was pretrained with those, so to get the same results for inference we need to add them as well."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "b66f19a1-acc4-4a90-85c2-5c149f71f835",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "from transformers import AutoTokenizer, AutoModelForSequenceClassification\n",
    "\n",
    "checkpoint = \"distilbert-base-uncased-finetuned-sst-2-english\"\n",
    "tokenizer = AutoTokenizer.from_pretrained(checkpoint)\n",
    "model = AutoModelForSequenceClassification.from_pretrained(checkpoint)\n",
    "sequences = [\"I've been waiting for a HuggingFace course my whole life.\", \"So have I!\"]\n",
    "\n",
    "tokens = tokenizer(sequences, padding=True, truncation=True, return_tensors=\"pt\")\n",
    "output = model(**tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "a8e3cb7a-893c-476d-ab4f-8a3525cc22f0",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SequenceClassifierOutput(loss=None, logits=tensor([[-1.5607,  1.6123],\n",
       "        [-3.6183,  3.9137]], grad_fn=<AddmmBackward0>), hidden_states=None, attentions=None)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a8bd493b-f8d0-48de-b411-eafa84661758",
   "metadata": {},
   "source": [
    "# Fine-tune a pretrained model\n",
    "https://huggingface.co/learn/nlp-course/chapter3/1?fw=pt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "e9c2bad0-95ec-4f9c-9704-ad94bcc754fc",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dfe74707557a4107814178563dad664b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)okenizer_config.json:   0%|          | 0.00/28.0 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6e6c563748ce41ffaca847cc35c79b8d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)lve/main/config.json:   0%|          | 0.00/570 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "000ed8e92c604d008e73763b60eea6d0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)solve/main/vocab.txt:   0%|          | 0.00/232k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6d58b9acbd904cc79676544b8173926d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)/main/tokenizer.json:   0%|          | 0.00/466k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6278a463e94e47ae919fe69aa0a55590",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading model.safetensors:   0%|          | 0.00/440M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForSequenceClassification: ['cls.seq_relationship.bias', 'cls.predictions.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.seq_relationship.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.LayerNorm.weight']\n",
      "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
      "C:\\Users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages\\transformers\\optimization.py:411: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from transformers import AdamW, AutoTokenizer, AutoModelForSequenceClassification\n",
    "\n",
    "# Same as before\n",
    "checkpoint = \"bert-base-uncased\"\n",
    "tokenizer = AutoTokenizer.from_pretrained(checkpoint)\n",
    "model = AutoModelForSequenceClassification.from_pretrained(checkpoint)\n",
    "sequences = [\n",
    "    \"I've been waiting for a HuggingFace course my whole life.\",\n",
    "    \"This course is amazing!\",\n",
    "]\n",
    "batch = tokenizer(sequences, padding=True, truncation=True, return_tensors=\"pt\")\n",
    "\n",
    "# This is new\n",
    "batch[\"labels\"] = torch.tensor([1, 1])\n",
    "\n",
    "optimizer = AdamW(model.parameters())\n",
    "loss = model(**batch).loss\n",
    "loss.backward()\n",
    "optimizer.step()"
   ]
  },
  {
   "cell_type": "raw",
   "id": "e5306b54-3e13-498f-a441-42c2dc69e815",
   "metadata": {},
   "source": [
    "Datasets: https://huggingface.co/docs/datasets/quickstart"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "2aa8d124-13bb-4e78-b5b0-d05f99beba68",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Looking in indexes: https://pypi.org/simple, https://pypi.ngc.nvidia.com\n",
      "Collecting datasets\n",
      "  Downloading datasets-2.13.0-py3-none-any.whl (485 kB)\n",
      "     ---------------------------------------- 0.0/485.6 kB ? eta -:--:--\n",
      "     -------------------- ----------------- 266.2/485.6 kB 8.0 MB/s eta 0:00:01\n",
      "     -------------------------------------- 485.6/485.6 kB 7.5 MB/s eta 0:00:00\n",
      "Collecting multiprocess\n",
      "  Downloading multiprocess-0.70.14-py39-none-any.whl (132 kB)\n",
      "     ---------------------------------------- 0.0/132.9 kB ? eta -:--:--\n",
      "     -------------------------------------- 132.9/132.9 kB 8.2 MB/s eta 0:00:00\n",
      "Collecting pandas\n",
      "  Downloading pandas-2.0.2-cp39-cp39-win_amd64.whl (10.7 MB)\n",
      "     ---------------------------------------- 0.0/10.7 MB ? eta -:--:--\n",
      "     -- ------------------------------------- 0.6/10.7 MB 12.4 MB/s eta 0:00:01\n",
      "     ----- ---------------------------------- 1.4/10.7 MB 18.4 MB/s eta 0:00:01\n",
      "     --------- ------------------------------ 2.6/10.7 MB 18.3 MB/s eta 0:00:01\n",
      "     -------------- ------------------------- 4.0/10.7 MB 21.2 MB/s eta 0:00:01\n",
      "     --------------------- ------------------ 5.9/10.7 MB 25.2 MB/s eta 0:00:01\n",
      "     ------------------------------- -------- 8.3/10.7 MB 29.7 MB/s eta 0:00:01\n",
      "     --------------------------------------- 10.7/10.7 MB 36.3 MB/s eta 0:00:00\n",
      "Requirement already satisfied: requests>=2.19.0 in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from datasets) (2.28.1)\n",
      "Requirement already satisfied: fsspec[http]>=2021.11.1 in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from datasets) (2023.6.0)\n",
      "Requirement already satisfied: pyyaml>=5.1 in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from datasets) (6.0)\n",
      "Collecting dill<0.3.7,>=0.3.0\n",
      "  Downloading dill-0.3.6-py3-none-any.whl (110 kB)\n",
      "     ---------------------------------------- 0.0/110.5 kB ? eta -:--:--\n",
      "     ---------------------------------------- 110.5/110.5 kB ? eta 0:00:00\n",
      "Requirement already satisfied: aiohttp in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from datasets) (3.8.3)\n",
      "Requirement already satisfied: tqdm>=4.62.1 in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from datasets) (4.65.0)\n",
      "Collecting xxhash\n",
      "  Downloading xxhash-3.2.0-cp39-cp39-win_amd64.whl (30 kB)\n",
      "Requirement already satisfied: numpy>=1.17 in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from datasets) (1.23.5)\n",
      "Collecting pyarrow>=8.0.0\n",
      "  Downloading pyarrow-12.0.1-cp39-cp39-win_amd64.whl (21.5 MB)\n",
      "     ---------------------------------------- 0.0/21.5 MB ? eta -:--:--\n",
      "     ------- -------------------------------- 3.9/21.5 MB 83.0 MB/s eta 0:00:01\n",
      "     --------------- ------------------------ 8.4/21.5 MB 89.8 MB/s eta 0:00:01\n",
      "     ----------------------- --------------- 13.1/21.5 MB 93.0 MB/s eta 0:00:01\n",
      "     ------------------------------ -------- 17.0/21.5 MB 93.0 MB/s eta 0:00:01\n",
      "     --------------------------------------  21.2/21.5 MB 81.8 MB/s eta 0:00:01\n",
      "     --------------------------------------- 21.5/21.5 MB 93.0 MB/s eta 0:00:00\n",
      "Requirement already satisfied: packaging in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from datasets) (23.0)\n",
      "Requirement already satisfied: huggingface-hub<1.0.0,>=0.11.0 in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from datasets) (0.15.1)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from aiohttp->datasets) (1.3.3)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from aiohttp->datasets) (1.2.0)\n",
      "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from aiohttp->datasets) (4.0.2)\n",
      "Requirement already satisfied: attrs>=17.3.0 in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from aiohttp->datasets) (22.2.0)\n",
      "Requirement already satisfied: charset-normalizer<3.0,>=2.0 in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from aiohttp->datasets) (2.0.4)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from aiohttp->datasets) (6.0.2)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from aiohttp->datasets) (1.8.1)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from huggingface-hub<1.0.0,>=0.11.0->datasets) (4.4.0)\n",
      "Requirement already satisfied: filelock in c:\\users\\lkk68\\appdata\\roaming\\python\\python39\\site-packages (from huggingface-hub<1.0.0,>=0.11.0->datasets) (3.4.2)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from requests>=2.19.0->datasets) (2022.12.7)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from requests>=2.19.0->datasets) (1.26.14)\n",
      "Requirement already satisfied: idna<4,>=2.5 in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from requests>=2.19.0->datasets) (3.4)\n",
      "Requirement already satisfied: colorama in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from tqdm>=4.62.1->datasets) (0.4.6)\n",
      "Collecting pytz>=2020.1\n",
      "  Downloading pytz-2023.3-py2.py3-none-any.whl (502 kB)\n",
      "     ---------------------------------------- 0.0/502.3 kB ? eta -:--:--\n",
      "     ---------------------------------------- 502.3/502.3 kB ? eta 0:00:00\n",
      "Collecting tzdata>=2022.1\n",
      "  Downloading tzdata-2023.3-py2.py3-none-any.whl (341 kB)\n",
      "     ---------------------------------------- 0.0/341.8 kB ? eta -:--:--\n",
      "     ---------------------------------------- 341.8/341.8 kB ? eta 0:00:00\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from pandas->datasets) (2.8.2)\n",
      "Requirement already satisfied: six>=1.5 in c:\\users\\lkk68\\appdata\\roaming\\python\\python39\\site-packages (from python-dateutil>=2.8.2->pandas->datasets) (1.16.0)\n",
      "Installing collected packages: pytz, xxhash, tzdata, pyarrow, dill, pandas, multiprocess, datasets\n",
      "Successfully installed datasets-2.13.0 dill-0.3.6 multiprocess-0.70.14 pandas-2.0.2 pyarrow-12.0.1 pytz-2023.3 tzdata-2023.3 xxhash-3.2.0\n"
     ]
    }
   ],
   "source": [
    "!pip install datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "824d61d0-e087-437c-af8b-d26995f04036",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "81606efa709f44ab87cda2b79eb19f1d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading builder script: 0.00B [00:00, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2b6782225afb4dc09a889f9f26947e5d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading metadata: 0.00B [00:00, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "99e4ac28ed2d43339a8ecf8be88c8910",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading readme: 0.00B [00:00, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Found cached dataset glue (C:/Users/lkk68/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "da345e49cc5d49af9c58f97c4ce08572",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['sentence1', 'sentence2', 'label', 'idx'],\n",
       "        num_rows: 3668\n",
       "    })\n",
       "    validation: Dataset({\n",
       "        features: ['sentence1', 'sentence2', 'label', 'idx'],\n",
       "        num_rows: 408\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['sentence1', 'sentence2', 'label', 'idx'],\n",
       "        num_rows: 1725\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from datasets import load_dataset\n",
    "\n",
    "raw_datasets = load_dataset(\"glue\", \"mrpc\")\n",
    "raw_datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "31ecb489-5aa9-4859-b403-09d2c1ca79dc",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'sentence1': 'Amrozi accused his brother , whom he called \" the witness \" , of deliberately distorting his evidence .',\n",
       " 'sentence2': 'Referring to him as only \" the witness \" , Amrozi accused his brother of deliberately distorting his evidence .',\n",
       " 'label': 1,\n",
       " 'idx': 0}"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_train_dataset = raw_datasets[\"train\"]\n",
    "raw_train_dataset[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b2117bc1-0dde-446d-81ce-959283349ce1",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3668"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(raw_train_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "651e767a-8697-4e3b-825c-7e30648d2ef4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'sentence1': Value(dtype='string', id=None),\n",
       " 'sentence2': Value(dtype='string', id=None),\n",
       " 'label': ClassLabel(names=['not_equivalent', 'equivalent'], id=None),\n",
       " 'idx': Value(dtype='int32', id=None)}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_train_dataset.features"
   ]
  },
  {
   "cell_type": "raw",
   "id": "25600304-dee9-4793-a6b2-30c8f68d59fd",
   "metadata": {},
   "source": [
    "the mapping of integers to label name is stored in the names folder. 0 corresponds to not_equivalent, and 1 corresponds to equivalent."
   ]
  },
  {
   "cell_type": "raw",
   "id": "3fcd3d7f-97da-4e48-9e00-3a526c14e2b4",
   "metadata": {},
   "source": [
    "we can directly tokenize all the first sentences and all the second sentences of each pair like this:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "06be4d1b-320c-4dd0-981b-c026683061ff",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from transformers import AutoTokenizer\n",
    "\n",
    "checkpoint = \"bert-base-uncased\"\n",
    "tokenizer = AutoTokenizer.from_pretrained(checkpoint)\n",
    "tokenized_sentences_1 = tokenizer(raw_datasets[\"train\"][\"sentence1\"])\n",
    "tokenized_sentences_2 = tokenizer(raw_datasets[\"train\"][\"sentence2\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "1d6d4fa6-7299-430a-9a1c-ee2c7ad032c7",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(tokenized_sentences_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "36b9f487-cdf1-42b7-a784-8c3bf49abba8",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'input_ids': [101, 2023, 2003, 1996, 2034, 6251, 1012, 102, 2023, 2003, 1996, 2117, 2028, 1012, 102], 'token_type_ids': [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inputs = tokenizer(\"This is the first sentence.\", \"This is the second one.\")\n",
    "inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "56fce239-ca7f-4b4f-a6d1-4ab5f00778b3",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['[CLS]',\n",
       " 'this',\n",
       " 'is',\n",
       " 'the',\n",
       " 'first',\n",
       " 'sentence',\n",
       " '.',\n",
       " '[SEP]',\n",
       " 'this',\n",
       " 'is',\n",
       " 'the',\n",
       " 'second',\n",
       " 'one',\n",
       " '.',\n",
       " '[SEP]']"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.convert_ids_to_tokens(inputs[\"input_ids\"])"
   ]
  },
  {
   "cell_type": "raw",
   "id": "78ef8e86-eba7-4ae2-ae62-a64c6597c7ce",
   "metadata": {},
   "source": [
    "the parts of the input corresponding to [CLS] sentence1 [SEP] all have a token type ID of 0, while the other parts, corresponding to sentence2 [SEP], all have a token type ID of 1."
   ]
  },
  {
   "cell_type": "raw",
   "id": "d50ad826-1660-4fdd-a793-ad7fceff3224",
   "metadata": {},
   "source": [
    "we can feed the tokenizer a list of pairs of sentences by giving it the list of first sentences, then the list of second sentences. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "27bb01f9-b10e-4e2a-9584-1b0524a45352",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "tokenized_dataset = tokenizer(\n",
    "    raw_datasets[\"train\"][\"sentence1\"],\n",
    "    raw_datasets[\"train\"][\"sentence2\"],\n",
    "    padding=True,\n",
    "    truncation=True,\n",
    ")"
   ]
  },
  {
   "cell_type": "raw",
   "id": "fa1758b6-e199-493b-895f-e6a13527d26e",
   "metadata": {},
   "source": [
    "To keep the data as a dataset, we will use the Dataset.map() method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "892c3021-0fd1-4942-82f9-66a760eec40e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def tokenize_function(example):\n",
    "    return tokenizer(example[\"sentence1\"], example[\"sentence2\"], truncation=True)"
   ]
  },
  {
   "cell_type": "raw",
   "id": "02c394cd-c3fe-45f2-89d5-6dca5f425041",
   "metadata": {},
   "source": [
    "This function takes a dictionary (like the items of our dataset) and returns a new dictionary with the keys input_ids, attention_mask, and token_type_ids. \n",
    "\n",
    "we only need to pad to the maximum length in that batch, and not the maximum length in the entire dataset. This can save a lot of time and processing power when the inputs have very variable lengths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "9de8a1c5-9be8-4c4e-86cf-698e9e83baa8",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/3668 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/408 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/1725 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['sentence1', 'sentence2', 'label', 'idx', 'input_ids', 'token_type_ids', 'attention_mask'],\n",
       "        num_rows: 3668\n",
       "    })\n",
       "    validation: Dataset({\n",
       "        features: ['sentence1', 'sentence2', 'label', 'idx', 'input_ids', 'token_type_ids', 'attention_mask'],\n",
       "        num_rows: 408\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['sentence1', 'sentence2', 'label', 'idx', 'input_ids', 'token_type_ids', 'attention_mask'],\n",
       "        num_rows: 1725\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenized_datasets = raw_datasets.map(tokenize_function, batched=True)\n",
    "tokenized_datasets"
   ]
  },
  {
   "cell_type": "raw",
   "id": "7136272b-ecff-4e55-87a8-560b7519479e",
   "metadata": {},
   "source": [
    "it also works if the example dictionary contains several samples (each key as a list of sentences) since the tokenizer works on lists of pairs of sentences. This will allow us to use the option batched=True in our call to map()"
   ]
  },
  {
   "cell_type": "raw",
   "id": "e303f38b-b4e4-4706-8128-057d80413f8a",
   "metadata": {},
   "source": [
    "The function that is responsible for putting together samples inside a batch is called a collate function. It’s an argument you can pass when you build a DataLoader, the default being a function that will just convert your samples to PyTorch tensors and concatenate them (recursively if your elements are lists, tuples, or dictionaries). "
   ]
  },
  {
   "cell_type": "raw",
   "id": "c7bcdf20-da7c-43cf-92d6-98d12d3b8707",
   "metadata": {},
   "source": [
    "we have to define a collate function that will apply the correct amount of padding to the items of the dataset we want to batch together."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "c2cd2099-0fdd-45d2-a7a7-702661964f6a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from transformers import DataCollatorWithPadding\n",
    "\n",
    "data_collator = DataCollatorWithPadding(tokenizer=tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "7108fa5c-fa2b-4982-b0dd-83ca4e9d185f",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3668"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(tokenized_datasets[\"train\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "8a630ab1-3b76-4f6e-8106-38a89e87afd5",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "samples = tokenized_datasets[\"train\"][:8] #select 8 samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "28ad4119-4b25-47d4-97a0-ab87e8884e53",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'sentence1': ['Amrozi accused his brother , whom he called \" the witness \" , of deliberately distorting his evidence .',\n",
       "  \"Yucaipa owned Dominick 's before selling the chain to Safeway in 1998 for $ 2.5 billion .\",\n",
       "  'They had published an advertisement on the Internet on June 10 , offering the cargo for sale , he added .',\n",
       "  'Around 0335 GMT , Tab shares were up 19 cents , or 4.4 % , at A $ 4.56 , having earlier set a record high of A $ 4.57 .',\n",
       "  'The stock rose $ 2.11 , or about 11 percent , to close Friday at $ 21.51 on the New York Stock Exchange .',\n",
       "  'Revenue in the first quarter of the year dropped 15 percent from the same period a year earlier .',\n",
       "  'The Nasdaq had a weekly gain of 17.27 , or 1.2 percent , closing at 1,520.15 on Friday .',\n",
       "  'The DVD-CCA then appealed to the state Supreme Court .'],\n",
       " 'sentence2': ['Referring to him as only \" the witness \" , Amrozi accused his brother of deliberately distorting his evidence .',\n",
       "  \"Yucaipa bought Dominick 's in 1995 for $ 693 million and sold it to Safeway for $ 1.8 billion in 1998 .\",\n",
       "  \"On June 10 , the ship 's owners had published an advertisement on the Internet , offering the explosives for sale .\",\n",
       "  'Tab shares jumped 20 cents , or 4.6 % , to set a record closing high at A $ 4.57 .',\n",
       "  'PG & E Corp. shares jumped $ 1.63 or 8 percent to $ 21.03 on the New York Stock Exchange on Friday .',\n",
       "  \"With the scandal hanging over Stewart 's company , revenue the first quarter of the year dropped 15 percent from the same period a year earlier .\",\n",
       "  'The tech-laced Nasdaq Composite .IXIC rallied 30.46 points , or 2.04 percent , to 1,520.15 .',\n",
       "  'The DVD CCA appealed that decision to the U.S. Supreme Court .'],\n",
       " 'label': [1, 0, 1, 0, 1, 1, 0, 1],\n",
       " 'idx': [0, 1, 2, 3, 4, 5, 6, 7],\n",
       " 'input_ids': [[101,\n",
       "   2572,\n",
       "   3217,\n",
       "   5831,\n",
       "   5496,\n",
       "   2010,\n",
       "   2567,\n",
       "   1010,\n",
       "   3183,\n",
       "   2002,\n",
       "   2170,\n",
       "   1000,\n",
       "   1996,\n",
       "   7409,\n",
       "   1000,\n",
       "   1010,\n",
       "   1997,\n",
       "   9969,\n",
       "   4487,\n",
       "   23809,\n",
       "   3436,\n",
       "   2010,\n",
       "   3350,\n",
       "   1012,\n",
       "   102,\n",
       "   7727,\n",
       "   2000,\n",
       "   2032,\n",
       "   2004,\n",
       "   2069,\n",
       "   1000,\n",
       "   1996,\n",
       "   7409,\n",
       "   1000,\n",
       "   1010,\n",
       "   2572,\n",
       "   3217,\n",
       "   5831,\n",
       "   5496,\n",
       "   2010,\n",
       "   2567,\n",
       "   1997,\n",
       "   9969,\n",
       "   4487,\n",
       "   23809,\n",
       "   3436,\n",
       "   2010,\n",
       "   3350,\n",
       "   1012,\n",
       "   102],\n",
       "  [101,\n",
       "   9805,\n",
       "   3540,\n",
       "   11514,\n",
       "   2050,\n",
       "   3079,\n",
       "   11282,\n",
       "   2243,\n",
       "   1005,\n",
       "   1055,\n",
       "   2077,\n",
       "   4855,\n",
       "   1996,\n",
       "   4677,\n",
       "   2000,\n",
       "   3647,\n",
       "   4576,\n",
       "   1999,\n",
       "   2687,\n",
       "   2005,\n",
       "   1002,\n",
       "   1016,\n",
       "   1012,\n",
       "   1019,\n",
       "   4551,\n",
       "   1012,\n",
       "   102,\n",
       "   9805,\n",
       "   3540,\n",
       "   11514,\n",
       "   2050,\n",
       "   4149,\n",
       "   11282,\n",
       "   2243,\n",
       "   1005,\n",
       "   1055,\n",
       "   1999,\n",
       "   2786,\n",
       "   2005,\n",
       "   1002,\n",
       "   6353,\n",
       "   2509,\n",
       "   2454,\n",
       "   1998,\n",
       "   2853,\n",
       "   2009,\n",
       "   2000,\n",
       "   3647,\n",
       "   4576,\n",
       "   2005,\n",
       "   1002,\n",
       "   1015,\n",
       "   1012,\n",
       "   1022,\n",
       "   4551,\n",
       "   1999,\n",
       "   2687,\n",
       "   1012,\n",
       "   102],\n",
       "  [101,\n",
       "   2027,\n",
       "   2018,\n",
       "   2405,\n",
       "   2019,\n",
       "   15147,\n",
       "   2006,\n",
       "   1996,\n",
       "   4274,\n",
       "   2006,\n",
       "   2238,\n",
       "   2184,\n",
       "   1010,\n",
       "   5378,\n",
       "   1996,\n",
       "   6636,\n",
       "   2005,\n",
       "   5096,\n",
       "   1010,\n",
       "   2002,\n",
       "   2794,\n",
       "   1012,\n",
       "   102,\n",
       "   2006,\n",
       "   2238,\n",
       "   2184,\n",
       "   1010,\n",
       "   1996,\n",
       "   2911,\n",
       "   1005,\n",
       "   1055,\n",
       "   5608,\n",
       "   2018,\n",
       "   2405,\n",
       "   2019,\n",
       "   15147,\n",
       "   2006,\n",
       "   1996,\n",
       "   4274,\n",
       "   1010,\n",
       "   5378,\n",
       "   1996,\n",
       "   14792,\n",
       "   2005,\n",
       "   5096,\n",
       "   1012,\n",
       "   102],\n",
       "  [101,\n",
       "   2105,\n",
       "   6021,\n",
       "   19481,\n",
       "   13938,\n",
       "   2102,\n",
       "   1010,\n",
       "   21628,\n",
       "   6661,\n",
       "   2020,\n",
       "   2039,\n",
       "   2539,\n",
       "   16653,\n",
       "   1010,\n",
       "   2030,\n",
       "   1018,\n",
       "   1012,\n",
       "   1018,\n",
       "   1003,\n",
       "   1010,\n",
       "   2012,\n",
       "   1037,\n",
       "   1002,\n",
       "   1018,\n",
       "   1012,\n",
       "   5179,\n",
       "   1010,\n",
       "   2383,\n",
       "   3041,\n",
       "   2275,\n",
       "   1037,\n",
       "   2501,\n",
       "   2152,\n",
       "   1997,\n",
       "   1037,\n",
       "   1002,\n",
       "   1018,\n",
       "   1012,\n",
       "   5401,\n",
       "   1012,\n",
       "   102,\n",
       "   21628,\n",
       "   6661,\n",
       "   5598,\n",
       "   2322,\n",
       "   16653,\n",
       "   1010,\n",
       "   2030,\n",
       "   1018,\n",
       "   1012,\n",
       "   1020,\n",
       "   1003,\n",
       "   1010,\n",
       "   2000,\n",
       "   2275,\n",
       "   1037,\n",
       "   2501,\n",
       "   5494,\n",
       "   2152,\n",
       "   2012,\n",
       "   1037,\n",
       "   1002,\n",
       "   1018,\n",
       "   1012,\n",
       "   5401,\n",
       "   1012,\n",
       "   102],\n",
       "  [101,\n",
       "   1996,\n",
       "   4518,\n",
       "   3123,\n",
       "   1002,\n",
       "   1016,\n",
       "   1012,\n",
       "   2340,\n",
       "   1010,\n",
       "   2030,\n",
       "   2055,\n",
       "   2340,\n",
       "   3867,\n",
       "   1010,\n",
       "   2000,\n",
       "   2485,\n",
       "   5958,\n",
       "   2012,\n",
       "   1002,\n",
       "   2538,\n",
       "   1012,\n",
       "   4868,\n",
       "   2006,\n",
       "   1996,\n",
       "   2047,\n",
       "   2259,\n",
       "   4518,\n",
       "   3863,\n",
       "   1012,\n",
       "   102,\n",
       "   18720,\n",
       "   1004,\n",
       "   1041,\n",
       "   13058,\n",
       "   1012,\n",
       "   6661,\n",
       "   5598,\n",
       "   1002,\n",
       "   1015,\n",
       "   1012,\n",
       "   6191,\n",
       "   2030,\n",
       "   1022,\n",
       "   3867,\n",
       "   2000,\n",
       "   1002,\n",
       "   2538,\n",
       "   1012,\n",
       "   6021,\n",
       "   2006,\n",
       "   1996,\n",
       "   2047,\n",
       "   2259,\n",
       "   4518,\n",
       "   3863,\n",
       "   2006,\n",
       "   5958,\n",
       "   1012,\n",
       "   102],\n",
       "  [101,\n",
       "   6599,\n",
       "   1999,\n",
       "   1996,\n",
       "   2034,\n",
       "   4284,\n",
       "   1997,\n",
       "   1996,\n",
       "   2095,\n",
       "   3333,\n",
       "   2321,\n",
       "   3867,\n",
       "   2013,\n",
       "   1996,\n",
       "   2168,\n",
       "   2558,\n",
       "   1037,\n",
       "   2095,\n",
       "   3041,\n",
       "   1012,\n",
       "   102,\n",
       "   2007,\n",
       "   1996,\n",
       "   9446,\n",
       "   5689,\n",
       "   2058,\n",
       "   5954,\n",
       "   1005,\n",
       "   1055,\n",
       "   2194,\n",
       "   1010,\n",
       "   6599,\n",
       "   1996,\n",
       "   2034,\n",
       "   4284,\n",
       "   1997,\n",
       "   1996,\n",
       "   2095,\n",
       "   3333,\n",
       "   2321,\n",
       "   3867,\n",
       "   2013,\n",
       "   1996,\n",
       "   2168,\n",
       "   2558,\n",
       "   1037,\n",
       "   2095,\n",
       "   3041,\n",
       "   1012,\n",
       "   102],\n",
       "  [101,\n",
       "   1996,\n",
       "   17235,\n",
       "   2850,\n",
       "   4160,\n",
       "   2018,\n",
       "   1037,\n",
       "   4882,\n",
       "   5114,\n",
       "   1997,\n",
       "   2459,\n",
       "   1012,\n",
       "   2676,\n",
       "   1010,\n",
       "   2030,\n",
       "   1015,\n",
       "   1012,\n",
       "   1016,\n",
       "   3867,\n",
       "   1010,\n",
       "   5494,\n",
       "   2012,\n",
       "   1015,\n",
       "   1010,\n",
       "   19611,\n",
       "   1012,\n",
       "   2321,\n",
       "   2006,\n",
       "   5958,\n",
       "   1012,\n",
       "   102,\n",
       "   1996,\n",
       "   6627,\n",
       "   1011,\n",
       "   17958,\n",
       "   17235,\n",
       "   2850,\n",
       "   4160,\n",
       "   12490,\n",
       "   1012,\n",
       "   11814,\n",
       "   2594,\n",
       "   24356,\n",
       "   2382,\n",
       "   1012,\n",
       "   4805,\n",
       "   2685,\n",
       "   1010,\n",
       "   2030,\n",
       "   1016,\n",
       "   1012,\n",
       "   5840,\n",
       "   3867,\n",
       "   1010,\n",
       "   2000,\n",
       "   1015,\n",
       "   1010,\n",
       "   19611,\n",
       "   1012,\n",
       "   2321,\n",
       "   1012,\n",
       "   102],\n",
       "  [101,\n",
       "   1996,\n",
       "   4966,\n",
       "   1011,\n",
       "   10507,\n",
       "   2050,\n",
       "   2059,\n",
       "   12068,\n",
       "   2000,\n",
       "   1996,\n",
       "   2110,\n",
       "   4259,\n",
       "   2457,\n",
       "   1012,\n",
       "   102,\n",
       "   1996,\n",
       "   4966,\n",
       "   10507,\n",
       "   2050,\n",
       "   12068,\n",
       "   2008,\n",
       "   3247,\n",
       "   2000,\n",
       "   1996,\n",
       "   1057,\n",
       "   1012,\n",
       "   1055,\n",
       "   1012,\n",
       "   4259,\n",
       "   2457,\n",
       "   1012,\n",
       "   102]],\n",
       " 'token_type_ids': [[0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1],\n",
       "  [0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1],\n",
       "  [0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1],\n",
       "  [0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1],\n",
       "  [0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1],\n",
       "  [0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1],\n",
       "  [0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1],\n",
       "  [0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1]],\n",
       " 'attention_mask': [[1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1],\n",
       "  [1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1],\n",
       "  [1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1],\n",
       "  [1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1],\n",
       "  [1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1],\n",
       "  [1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1],\n",
       "  [1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1],\n",
       "  [1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1]]}"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "15cbf261-5c07-4836-a65c-e155c30ac4c9",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "samples = {k: v for k, v in samples.items() if k not in [\"idx\", \"sentence1\", \"sentence2\"]} #remove the columns idx, sentence1, and sentence2 as they won’t be needed and contain strings "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "34e7b18e-f86a-4b75-97b4-b7f959a3a841",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[50, 59, 47, 67, 59, 50, 62, 32]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[len(x) for x in samples[\"input_ids\"]]"
   ]
  },
  {
   "cell_type": "raw",
   "id": "73f2e77a-a986-48d8-8942-26812db33e47",
   "metadata": {},
   "source": [
    "we get samples of varying length, from 32 to 67. Dynamic padding means the samples in this batch should all be padded to a length of 67, the maximum length inside the batch."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "16fe77e5-ce3a-4c16-ba33-39aef0351446",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "You're using a BertTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'input_ids': torch.Size([8, 67]),\n",
       " 'token_type_ids': torch.Size([8, 67]),\n",
       " 'attention_mask': torch.Size([8, 67]),\n",
       " 'labels': torch.Size([8])}"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch = data_collator(samples)\n",
    "{k: v.shape for k, v in batch.items()}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "daf6372c-36d3-496c-ad57-99f9a78fa062",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "54e2ca91-36ad-4093-9750-04d888ec5e3e",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Looking in indexes: https://pypi.org/simple, https://pypi.ngc.nvidia.com\n",
      "Collecting accelerate\n",
      "  Downloading accelerate-0.20.3-py3-none-any.whl (227 kB)\n",
      "     ---------------------------------------- 0.0/227.6 kB ? eta -:--:--\n",
      "     ------------------------------------- 227.6/227.6 kB 13.6 MB/s eta 0:00:00\n",
      "Requirement already satisfied: numpy>=1.17 in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from accelerate) (1.23.5)\n",
      "Requirement already satisfied: packaging>=20.0 in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from accelerate) (23.0)\n",
      "Requirement already satisfied: pyyaml in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from accelerate) (6.0)\n",
      "Requirement already satisfied: torch>=1.6.0 in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from accelerate) (2.0.0)\n",
      "Requirement already satisfied: psutil in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from accelerate) (5.9.4)\n",
      "Requirement already satisfied: filelock in c:\\users\\lkk68\\appdata\\roaming\\python\\python39\\site-packages (from torch>=1.6.0->accelerate) (3.4.2)\n",
      "Requirement already satisfied: typing-extensions in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from torch>=1.6.0->accelerate) (4.4.0)\n",
      "Requirement already satisfied: sympy in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from torch>=1.6.0->accelerate) (1.11.1)\n",
      "Requirement already satisfied: networkx in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from torch>=1.6.0->accelerate) (2.8.4)\n",
      "Requirement already satisfied: jinja2 in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from torch>=1.6.0->accelerate) (3.1.2)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from jinja2->torch>=1.6.0->accelerate) (2.1.1)\n",
      "Requirement already satisfied: mpmath>=0.19 in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from sympy->torch>=1.6.0->accelerate) (1.2.1)\n",
      "Installing collected packages: accelerate\n",
      "Successfully installed accelerate-0.20.3\n"
     ]
    }
   ],
   "source": [
    "!pip install accelerate -U"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "7464317d-cf03-43bc-b05d-2878061e1eee",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Found cached dataset glue (C:/Users/lkk68/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3cdfcb3ccaed462c9c1f47530b2f2fa5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/3668 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading cached processed dataset at C:\\Users\\lkk68\\.cache\\huggingface\\datasets\\glue\\mrpc\\1.0.0\\dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad\\cache-3dfcda83adde2396.arrow\n",
      "Loading cached processed dataset at C:\\Users\\lkk68\\.cache\\huggingface\\datasets\\glue\\mrpc\\1.0.0\\dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad\\cache-11e216b6f01e882d.arrow\n"
     ]
    }
   ],
   "source": [
    "from datasets import load_dataset\n",
    "from transformers import AutoTokenizer, DataCollatorWithPadding\n",
    "\n",
    "raw_datasets = load_dataset(\"glue\", \"mrpc\")\n",
    "checkpoint = \"bert-base-uncased\"\n",
    "tokenizer = AutoTokenizer.from_pretrained(checkpoint)\n",
    "\n",
    "\n",
    "def tokenize_function(example):\n",
    "    return tokenizer(example[\"sentence1\"], example[\"sentence2\"], truncation=True)\n",
    "\n",
    "\n",
    "tokenized_datasets = raw_datasets.map(tokenize_function, batched=True)\n",
    "data_collator = DataCollatorWithPadding(tokenizer=tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ad29f5dc-cdb3-41d0-811e-50d461ad44f4",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from transformers import TrainingArguments\n",
    "\n",
    "training_args = TrainingArguments(\"test-trainer\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "8ca3bdb5-7473-4022-b722-fed303d9e73b",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForSequenceClassification: ['cls.predictions.transform.dense.weight', 'cls.predictions.bias', 'cls.seq_relationship.bias', 'cls.seq_relationship.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.dense.bias']\n",
      "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoModelForSequenceClassification\n",
    "\n",
    "model = AutoModelForSequenceClassification.from_pretrained(checkpoint, num_labels=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "f21dc103-98b3-4689-8ee6-e109d123facb",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from transformers import Trainer\n",
    "\n",
    "trainer = Trainer(\n",
    "    model,\n",
    "    training_args,\n",
    "    train_dataset=tokenized_datasets[\"train\"],\n",
    "    eval_dataset=tokenized_datasets[\"validation\"],\n",
    "    data_collator=data_collator,\n",
    "    tokenizer=tokenizer,\n",
    ")"
   ]
  },
  {
   "cell_type": "raw",
   "id": "f8bb1631-6de3-4b65-a06d-d22a810b80f2",
   "metadata": {},
   "source": [
    "To fine-tune the model on our dataset, we just have to call the train() method of our Trainer:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "5b9c0a7d-8b7a-4f89-9e6f-dc18f4c12015",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages\\transformers\\optimization.py:411: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  warnings.warn(\n",
      "You're using a BertTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='1377' max='1377' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [1377/1377 01:54, Epoch 3/3]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.541500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.325300</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=1377, training_loss=0.35906219482421875, metrics={'train_runtime': 120.2271, 'train_samples_per_second': 91.527, 'train_steps_per_second': 11.453, 'total_flos': 406183858377360.0, 'train_loss': 0.35906219482421875, 'epoch': 3.0})"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "241a1f5e-4fde-49eb-8a19-b0bcbd38cfd5",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(408, 2) (408,)\n"
     ]
    }
   ],
   "source": [
    "predictions = trainer.predict(tokenized_datasets[\"validation\"])\n",
    "print(predictions.predictions.shape, predictions.label_ids.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "c3e93e1a-e073-4713-a770-e6c442a842b9",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "preds = np.argmax(predictions.predictions, axis=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "a6c2d979-7043-4066-be88-3eb02d73ccbf",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Looking in indexes: https://pypi.org/simple, https://pypi.ngc.nvidia.com\n",
      "Collecting evaluate\n",
      "  Downloading evaluate-0.4.0-py3-none-any.whl (81 kB)\n",
      "     ---------------------------------------- 0.0/81.4 kB ? eta -:--:--\n",
      "     ---------------------------------------- 81.4/81.4 kB 4.4 MB/s eta 0:00:00\n",
      "Requirement already satisfied: requests>=2.19.0 in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from evaluate) (2.28.1)\n",
      "Requirement already satisfied: packaging in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from evaluate) (23.0)\n",
      "Requirement already satisfied: dill in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from evaluate) (0.3.6)\n",
      "Requirement already satisfied: tqdm>=4.62.1 in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from evaluate) (4.65.0)\n",
      "Requirement already satisfied: xxhash in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from evaluate) (3.2.0)\n",
      "Requirement already satisfied: huggingface-hub>=0.7.0 in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from evaluate) (0.15.1)\n",
      "Requirement already satisfied: multiprocess in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from evaluate) (0.70.14)\n",
      "Requirement already satisfied: datasets>=2.0.0 in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from evaluate) (2.13.0)\n",
      "Requirement already satisfied: fsspec[http]>=2021.05.0 in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from evaluate) (2023.6.0)\n",
      "Requirement already satisfied: pandas in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from evaluate) (2.0.2)\n",
      "Collecting responses<0.19\n",
      "  Downloading responses-0.18.0-py3-none-any.whl (38 kB)\n",
      "Requirement already satisfied: numpy>=1.17 in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from evaluate) (1.23.5)\n",
      "Requirement already satisfied: pyyaml>=5.1 in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from datasets>=2.0.0->evaluate) (6.0)\n",
      "Requirement already satisfied: pyarrow>=8.0.0 in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from datasets>=2.0.0->evaluate) (12.0.1)\n",
      "Requirement already satisfied: aiohttp in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from datasets>=2.0.0->evaluate) (3.8.3)\n",
      "Requirement already satisfied: filelock in c:\\users\\lkk68\\appdata\\roaming\\python\\python39\\site-packages (from huggingface-hub>=0.7.0->evaluate) (3.4.2)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from huggingface-hub>=0.7.0->evaluate) (4.4.0)\n",
      "Requirement already satisfied: idna<4,>=2.5 in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from requests>=2.19.0->evaluate) (3.4)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from requests>=2.19.0->evaluate) (1.26.14)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from requests>=2.19.0->evaluate) (2022.12.7)\n",
      "Requirement already satisfied: charset-normalizer<3,>=2 in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from requests>=2.19.0->evaluate) (2.0.4)\n",
      "Requirement already satisfied: colorama in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from tqdm>=4.62.1->evaluate) (0.4.6)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from pandas->evaluate) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from pandas->evaluate) (2023.3)\n",
      "Requirement already satisfied: tzdata>=2022.1 in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from pandas->evaluate) (2023.3)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from aiohttp->datasets>=2.0.0->evaluate) (1.8.1)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from aiohttp->datasets>=2.0.0->evaluate) (1.2.0)\n",
      "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from aiohttp->datasets>=2.0.0->evaluate) (4.0.2)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from aiohttp->datasets>=2.0.0->evaluate) (6.0.2)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from aiohttp->datasets>=2.0.0->evaluate) (1.3.3)\n",
      "Requirement already satisfied: attrs>=17.3.0 in c:\\users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages (from aiohttp->datasets>=2.0.0->evaluate) (22.2.0)\n",
      "Requirement already satisfied: six>=1.5 in c:\\users\\lkk68\\appdata\\roaming\\python\\python39\\site-packages (from python-dateutil>=2.8.2->pandas->evaluate) (1.16.0)\n",
      "Installing collected packages: responses, evaluate\n",
      "Successfully installed evaluate-0.4.0 responses-0.18.0\n"
     ]
    }
   ],
   "source": [
    "!pip install evaluate"
   ]
  },
  {
   "cell_type": "raw",
   "id": "acec8e72-4876-4932-8103-6e7482195fba",
   "metadata": {},
   "source": [
    "https://github.com/huggingface/evaluate/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "000bf90d-c4f6-40c6-9580-bbc54fe458f5",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "eecb4c15677147099872433b774d0ace",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading builder script: 0.00B [00:00, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'accuracy': 0.8504901960784313, 'f1': 0.8957264957264958}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import evaluate\n",
    "\n",
    "metric = evaluate.load(\"glue\", \"mrpc\")\n",
    "metric.compute(predictions=preds, references=predictions.label_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "ecadd6ff-13ee-4255-a2ac-ab8a2dc1a3b3",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "#put things together\n",
    "def compute_metrics(eval_preds):\n",
    "    metric = evaluate.load(\"glue\", \"mrpc\")\n",
    "    logits, labels = eval_preds\n",
    "    predictions = np.argmax(logits, axis=-1)\n",
    "    return metric.compute(predictions=predictions, references=labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "3191fb79-5bbf-45cf-b67d-e30604b0800a",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForSequenceClassification: ['cls.predictions.transform.dense.weight', 'cls.predictions.bias', 'cls.seq_relationship.bias', 'cls.seq_relationship.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.dense.bias']\n",
      "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "training_args = TrainingArguments(\"test-trainer\", evaluation_strategy=\"epoch\")\n",
    "model = AutoModelForSequenceClassification.from_pretrained(checkpoint, num_labels=2)\n",
    "\n",
    "trainer = Trainer(\n",
    "    model,\n",
    "    training_args,\n",
    "    train_dataset=tokenized_datasets[\"train\"],\n",
    "    eval_dataset=tokenized_datasets[\"validation\"],\n",
    "    data_collator=data_collator,\n",
    "    tokenizer=tokenizer,\n",
    "    compute_metrics=compute_metrics,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "0a47ea2a-8315-4c05-b440-6b7d97118e92",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages\\transformers\\optimization.py:411: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='1377' max='1377' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [1377/1377 01:58, Epoch 3/3]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>F1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.374698</td>\n",
       "      <td>0.835784</td>\n",
       "      <td>0.885470</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.514900</td>\n",
       "      <td>0.456679</td>\n",
       "      <td>0.848039</td>\n",
       "      <td>0.891228</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.292700</td>\n",
       "      <td>0.722507</td>\n",
       "      <td>0.860294</td>\n",
       "      <td>0.901554</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=1377, training_loss=0.33171884928740014, metrics={'train_runtime': 119.0597, 'train_samples_per_second': 92.424, 'train_steps_per_second': 11.566, 'total_flos': 406183858377360.0, 'train_loss': 0.33171884928740014, 'epoch': 3.0})"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dadbe3aa-f413-4f04-b7c4-420b7b6e78b1",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "mycondapy39",
   "language": "python",
   "name": "mycondapy39"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}