nlp/huggingfaceLM.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "USE_HPC=False\n",
    "if USE_HPC:\n",
    "    mycache_dir=\"/data/cmpe249-fa23/Huggingfacecache\"\n",
    "    os.environ['TRANSFORMERS_CACHE'] = mycache_dir\n",
    "    os.environ['HF_HOME'] = mycache_dir\n",
    "    os.environ['HF_DATASETS_CACHE'] = mycache_dir\n",
    "    os.environ['http_proxy'] = \"http://172.16.1.2:3128\"\n",
    "    os.environ['HTTP_PROXY'] = \"http://172.16.1.2:3128\"\n",
    "    os.environ['https_proxy'] = \"https://172.16.1.2:3128\"\n",
    "    os.environ['HTTPS_PROXY'] = \"https://172.16.1.2:3128\"\n",
    "    trainoutput=\"/data/cmpe249-fa23/trainoutput/huggingface\"\n",
    "    taskname=\"eli5asksciencemodeling\"\n",
    "else:\n",
    "    trainoutput=\"./output\"\n",
    "    taskname=\"eli5asksciencemodeling\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Masked language modeling\n",
    "Masked language modeling predicts a masked token in a sequence, and the model can attend to tokens bidirectionally. This means the model has full access to the tokens on the left and right. Masked language modeling is great for tasks that require a good contextual understanding of an entire sequence. BERT is an example of a masked language model.\n",
    "https://huggingface.co/docs/transformers/tasks/masked_language_modeling"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Finetune DistilRoBERTa on the r/askscience subset of the ELI5 dataset: https://huggingface.co/datasets/eli5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1400522365a540f5bd3e271262664143",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading builder script:   0%|          | 0.00/18.2k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e6e0a6871c314f78b008aabe1b62106c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading metadata:   0%|          | 0.00/6.36k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "475a47a513ba48f5b64d47c2d459b8ac",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading readme:   0%|          | 0.00/15.8k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading and preparing dataset eli5/LFQA_reddit to C:/Users/lkk68/.cache/huggingface/datasets/eli5/LFQA_reddit/1.0.0/17574e5502a10f41bbd17beba83e22475b499fa62caa1384a3d093fc856fe6fa...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d4eac5e4f53c41b7a9e7f3f0e505e3c2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/3.50k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2bfd043ecfc24071bb269dfeab7f143a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/576M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ab4d823411104fc2b5dbeb9d38b30029",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/21.1M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8f20276070814a7b8f2ff765c3a54215",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/53.0M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d33e3d5e576b4ef4bcf57e35767b646b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/286M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7328b4f4d466456bab3d1ccf90c06021",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/9.65M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "12725fae8039424482fec173dc6457f1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/17.7M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8ffe8bf8323f4bb99c7040de7c2ef0c9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/330M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "10ad357082e742d39fa0bd7474adb469",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/18.7M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f6589dbbb71a4a2397885ca58bc8a88a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/36.2M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset eli5 downloaded and prepared to C:/Users/lkk68/.cache/huggingface/datasets/eli5/LFQA_reddit/1.0.0/17574e5502a10f41bbd17beba83e22475b499fa62caa1384a3d093fc856fe6fa. Subsequent calls will reuse this data.\n"
     ]
    }
   ],
   "source": [
    "from datasets import load_dataset\n",
    "\n",
    "eli5 = load_dataset(\"eli5\", split=\"train_asks[:5000]\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "eli5 = eli5.train_test_split(test_size=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'q_id': 'iukwf',\n",
       " 'title': 'Statics question',\n",
       " 'selftext': \"I'm taking statics and mechanics of materials and had a conceptual question that my professor didn't answer confidently. (new prof.) Anyway if you had say a cube in space and pushed it perpendicular to one side by the corner, would it experience translational motion and a moment or just translation motion?\",\n",
       " 'document': '',\n",
       " 'subreddit': 'askscience',\n",
       " 'answers': {'a_id': ['c26ra0j'],\n",
       "  'text': ['A static force applied eccentric to the center of mass (c.m.) of an initially stationary cube in space would result in rotation and translation. In terms of statics, think about the \"penalty\" of \"moving\" the force *from* the corner over *to* the c.m., where this \"penalty\" is the moment.\\nAs a terrestrial analogy, try pushing a ballon at an eccentric location.'],\n",
       "  'score': [8]},\n",
       " 'title_urls': {'url': []},\n",
       " 'selftext_urls': {'url': []},\n",
       " 'answers_urls': {'url': []}}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eli5[\"train\"][0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You’re only really interested in the text field (nested inside answers). What’s cool about language modeling tasks is you don’t need labels (also known as an unsupervised task) because the next word is the label."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['q_id', 'title', 'selftext', 'document', 'subreddit', 'answers.a_id', 'answers.text', 'answers.score', 'title_urls.url', 'selftext_urls.url', 'answers_urls.url'],\n",
       "        num_rows: 4000\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['q_id', 'title', 'selftext', 'document', 'subreddit', 'answers.a_id', 'answers.text', 'answers.score', 'title_urls.url', 'selftext_urls.url', 'answers_urls.url'],\n",
       "        num_rows: 1000\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#e xtract the text subfield from its nested structure with the flatten method:\n",
    "eli5 = eli5.flatten()\n",
    "eli5"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Each subfield is now a separate column as indicated by the answers prefix, and the text field is a list now"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'q_id': 'iukwf',\n",
       " 'title': 'Statics question',\n",
       " 'selftext': \"I'm taking statics and mechanics of materials and had a conceptual question that my professor didn't answer confidently. (new prof.) Anyway if you had say a cube in space and pushed it perpendicular to one side by the corner, would it experience translational motion and a moment or just translation motion?\",\n",
       " 'document': '',\n",
       " 'subreddit': 'askscience',\n",
       " 'answers.a_id': ['c26ra0j'],\n",
       " 'answers.text': ['A static force applied eccentric to the center of mass (c.m.) of an initially stationary cube in space would result in rotation and translation. In terms of statics, think about the \"penalty\" of \"moving\" the force *from* the corner over *to* the c.m., where this \"penalty\" is the moment.\\nAs a terrestrial analogy, try pushing a ballon at an eccentric location.'],\n",
       " 'answers.score': [8],\n",
       " 'title_urls.url': [],\n",
       " 'selftext_urls.url': [],\n",
       " 'answers_urls.url': []}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eli5[\"train\"][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7d91e73c618243968156541ebcbf10c7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)lve/main/config.json:   0%|          | 0.00/480 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages\\huggingface_hub\\file_download.py:133: UserWarning: `huggingface_hub` cache-system uses symlinks by default to efficiently store duplicated files but your machine does not support them in C:\\Users\\lkk68\\.cache\\huggingface\\hub. Caching files will still work but in a degraded version that might require more space on your disk. This warning can be disabled by setting the `HF_HUB_DISABLE_SYMLINKS_WARNING` environment variable. For more details, see https://huggingface.co/docs/huggingface_hub/how-to-cache#limitations.\n",
      "To support symlinks on Windows, you either need to activate Developer Mode or to run Python as an administrator. In order to see activate developer mode, see this article: https://docs.microsoft.com/en-us/windows/apps/get-started/enable-your-device-for-development\n",
      "  warnings.warn(message)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4a5ea790e9954bc389db682eb6855c9d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)olve/main/vocab.json:   0%|          | 0.00/899k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fb73a958543947a2a0d5c356024bce37",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)olve/main/merges.txt:   0%|          | 0.00/456k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "67c0f4109d19466db872c2305623efb2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)/main/tokenizer.json:   0%|          | 0.00/1.36M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from transformers import AutoTokenizer\n",
    "modelname=\"distilroberta-base\"\n",
    "if USE_HPC:\n",
    "    localpath=os.path.join(mycache_dir, modelname)\n",
    "    tokenizer = AutoTokenizer.from_pretrained(localpath)\n",
    "else:\n",
    "    tokenizer = AutoTokenizer.from_pretrained(modelname)#, cache_dir=mycache_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'input_ids': [[0, 347, 26593, 9866, 6, 8, 9, 768, 70, 8980, 3611, 32, 10, 5043, 9, 1164, 4, 1437, 616, 8825, 2368, 52, 206, 9, 9281, 7823, 25, 45059, 5224, 4748, 6, 42, 16, 444, 31, 1528, 4, 1437, 152, 16, 596, 2480, 40, 20147, 23, 10, 723, 5181, 23, 723, 9985, 36, 5412, 16, 7735, 8, 42, 16, 23, 18198, 3569, 6, 53, 1364, 25, 10, 205, 1246, 322, 50118, 50118, 11773, 18, 41, 1246, 9, 10, 48498, 14, 3374, 55, 11942, 2088, 223, 239, 9985, 35, 18134, 42703, 1215, 288, 47426, 50118, 50118, 15791, 1043, 9314, 33, 10, 30862, 45172, 14, 16, 10, 5043, 9, 5, 10875, 9, 80, 4204, 12957, 6, 98, 26640, 74, 7280, 14, 10875, 6, 2992, 24, 18, 30862, 45172, 4, 50118, 50118, 15248, 21491, 994, 32, 67, 10, 5043, 9, 49, 3989, 6, 98, 26640, 115, 483, 7, 10, 464, 89, 4, 50118, 50118, 100, 206, 5, 144, 505, 233, 74, 28, 14, 5, 3650, 9, 6214, 43951, 1022, 223, 1164, 6, 8, 14, 74, 22646, 17948, 5, 8917, 183, 4, 50118, 1215, 42703, 1215, 134, 1215, 2], [0, 133, 1164, 64, 464, 5, 3184, 9, 3183, 4, 9387, 59, 11720, 8, 20992, 1459, 4, 1773, 5, 9326, 16, 45, 95, 65, 9486, 42019, 1468, 24, 40, 676, 25406, 26640, 4, 50118, 50118, 133, 1164, 23, 5, 2576, 9, 5, 5466, 9, 21217, 16, 593, 7, 204, 12, 245, 153, 43118, 29492, 4, 2]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer(eli5[\"train\"][0]['answers.text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "examples=eli5[\"train\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4000"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(examples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "listexamples = [\" \".join(x) for x in examples[\"answers.text\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4000"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(listexamples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Token indices sequence length is longer than the specified maximum sequence length for this model (575 > 512). Running this sequence through the model will result in indexing errors\n"
     ]
    }
   ],
   "source": [
    "token_train=tokenizer(listexamples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "token_train=tokenizer(listexamples, padding='max_length', truncation=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TokenizerWrapper:\n",
    "    def __init__(self, tokenizer):\n",
    "        self.tokenizer = tokenizer\n",
    "    \n",
    "    def tokenize_function(self, examples):\n",
    "        return self.tokenizer(\n",
    "            [\" \".join(x) for x in examples[\"answers.text\"]],\n",
    "            padding=\"max_length\",\n",
    "            truncation=True,\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer_wrapper = TokenizerWrapper(tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3d96d2e9599d40cab32eaaad1c5558a9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map (num_proc=3):   0%|          | 0/4000 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "53e53f3100f24d99b9bfffd39ba0ec01",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map (num_proc=3):   0%|          | 0/1000 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "tokenized_dataset = eli5.map(tokenizer_wrapper.tokenize_function, batched=True, num_proc=3, remove_columns=eli5[\"train\"].column_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['input_ids', 'attention_mask'],\n",
       "        num_rows: 4000\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['input_ids', 'attention_mask'],\n",
       "        num_rows: 1000\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenized_dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "concatenate all the sequences\n",
    "split the concatenated sequences into shorter chunks defined by block_size, which should be both shorter than the maximum input length and short enough for your GPU RAM."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def group_texts(examples):\n",
    "    block_size = 128\n",
    "    # Concatenate all texts.\n",
    "    #print(examples.keys())\n",
    "    concatenated_examples = {k: sum(examples[k], []) for k in examples.keys()}\n",
    "    total_length = len(concatenated_examples[list(examples.keys())[0]])\n",
    "    #print('total_length:', total_length)\n",
    "    # We drop the small remainder, we could add padding if the model supported it instead of this drop, you can\n",
    "    # customize this part to your needs.\n",
    "    if total_length >= block_size:\n",
    "        total_length = (total_length // block_size) * block_size\n",
    "    # Split by chunks of block_size.\n",
    "    result = {\n",
    "        k: [t[i : i + block_size] for i in range(0, total_length, block_size)]\n",
    "        for k, t in concatenated_examples.items()\n",
    "    }\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "04fe0f9961464654852dd1ee8c718dc6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map (num_proc=4):   0%|          | 0/4000 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8ed35803019b4b44b01dad2a80e999c8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map (num_proc=4):   0%|          | 0/1000 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "lm_dataset = tokenized_dataset.map(group_texts, batched=True, num_proc=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['input_ids', 'attention_mask'],\n",
       "        num_rows: 16000\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['input_ids', 'attention_mask'],\n",
       "        num_rows: 4000\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lm_dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Use the end-of-sequence token as the padding token and specify mlm_probability to randomly mask tokens each time you iterate over the data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import DataCollatorForLanguageModeling\n",
    "\n",
    "#tokenizer.pad_token = tokenizer.eos_token\n",
    "data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm_probability=0.15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "def29a0dfd714dd3844c679a3434f39a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading model.safetensors:   0%|          | 0.00/331M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from transformers import AutoModelForMaskedLM\n",
    "if USE_HPC:\n",
    "    localpath=os.path.join(mycache_dir, modelname) #modelname=\"distilroberta-base\"\n",
    "    model = AutoModelForMaskedLM.from_pretrained(localpath)\n",
    "else:\n",
    "    model = AutoModelForMaskedLM.from_pretrained(modelname)#\"distilroberta-base\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import TrainingArguments\n",
    "\n",
    "training_args = TrainingArguments(\n",
    "    output_dir=os.path.join(trainoutput, modelname, taskname), #\"./output/my_awesome_eli5_mlm_model\",\n",
    "    evaluation_strategy=\"epoch\",\n",
    "    learning_rate=2e-5,\n",
    "    num_train_epochs=10,\n",
    "    weight_decay=0.01,\n",
    "    push_to_hub=True,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "aace46aeb59340e3b9788446af2320b9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HTML(value='<center> <img\\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.sv…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from huggingface_hub import notebook_login\n",
    "notebook_login()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Cloning https://huggingface.co/lkk688/eli5asksciencemodeling into local empty directory.\n"
     ]
    }
   ],
   "source": [
    "from transformers import Trainer\n",
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=training_args,\n",
    "    train_dataset=lm_dataset[\"train\"],\n",
    "    eval_dataset=lm_dataset[\"test\"],\n",
    "    data_collator=data_collator,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages\\transformers\\optimization.py:411: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "45d23f1055494f0fa7970db981583919",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/20000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "You're using a RobertaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 2.331, 'learning_rate': 1.95e-05, 'epoch': 0.25}\n",
      "{'loss': 2.231, 'learning_rate': 1.9e-05, 'epoch': 0.5}\n",
      "{'loss': 2.2261, 'learning_rate': 1.8500000000000002e-05, 'epoch': 0.75}\n",
      "{'loss': 2.2115, 'learning_rate': 1.8e-05, 'epoch': 1.0}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f247bba7382a420593de415f719b6ed8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/500 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 1.992519497871399, 'eval_runtime': 7.6104, 'eval_samples_per_second': 525.598, 'eval_steps_per_second': 65.7, 'epoch': 1.0}\n",
      "{'loss': 2.1583, 'learning_rate': 1.7500000000000002e-05, 'epoch': 1.25}\n",
      "{'loss': 2.1447, 'learning_rate': 1.7e-05, 'epoch': 1.5}\n",
      "{'loss': 2.1256, 'learning_rate': 1.65e-05, 'epoch': 1.75}\n",
      "{'loss': 2.0949, 'learning_rate': 1.6000000000000003e-05, 'epoch': 2.0}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "710c1bfcccdd45ab8bf7cae4968ea703",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/500 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 2.0107181072235107, 'eval_runtime': 7.3769, 'eval_samples_per_second': 542.236, 'eval_steps_per_second': 67.78, 'epoch': 2.0}\n",
      "{'loss': 2.1094, 'learning_rate': 1.55e-05, 'epoch': 2.25}\n",
      "{'loss': 2.1017, 'learning_rate': 1.5000000000000002e-05, 'epoch': 2.5}\n",
      "{'loss': 2.0655, 'learning_rate': 1.45e-05, 'epoch': 2.75}\n",
      "{'loss': 2.105, 'learning_rate': 1.4e-05, 'epoch': 3.0}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9726db8d93af484e89ddd0dc384e2fd6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/500 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 1.9571186304092407, 'eval_runtime': 7.1642, 'eval_samples_per_second': 558.33, 'eval_steps_per_second': 69.791, 'epoch': 3.0}\n",
      "{'loss': 2.0274, 'learning_rate': 1.3500000000000001e-05, 'epoch': 3.25}\n",
      "{'loss': 2.0416, 'learning_rate': 1.3000000000000001e-05, 'epoch': 3.5}\n",
      "{'loss': 2.0081, 'learning_rate': 1.25e-05, 'epoch': 3.75}\n",
      "{'loss': 2.0432, 'learning_rate': 1.2e-05, 'epoch': 4.0}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2f5dbf1e2d8d426cba563aa5372e7360",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/500 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 1.9548934698104858, 'eval_runtime': 6.9702, 'eval_samples_per_second': 573.87, 'eval_steps_per_second': 71.734, 'epoch': 4.0}\n",
      "{'loss': 1.9858, 'learning_rate': 1.15e-05, 'epoch': 4.25}\n",
      "{'loss': 1.9906, 'learning_rate': 1.1000000000000001e-05, 'epoch': 4.5}\n",
      "{'loss': 2.0183, 'learning_rate': 1.0500000000000001e-05, 'epoch': 4.75}\n",
      "{'loss': 1.9822, 'learning_rate': 1e-05, 'epoch': 5.0}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ba620dadcb574e5ca416606ead4f78f0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/500 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 1.915711760520935, 'eval_runtime': 6.7874, 'eval_samples_per_second': 589.324, 'eval_steps_per_second': 73.665, 'epoch': 5.0}\n",
      "{'loss': 1.9621, 'learning_rate': 9.5e-06, 'epoch': 5.25}\n",
      "{'loss': 1.9472, 'learning_rate': 9e-06, 'epoch': 5.5}\n",
      "{'loss': 1.945, 'learning_rate': 8.5e-06, 'epoch': 5.75}\n",
      "{'loss': 1.9467, 'learning_rate': 8.000000000000001e-06, 'epoch': 6.0}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6e03d963953f4a9aba79d119ff983559",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/500 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 1.9151719808578491, 'eval_runtime': 6.7893, 'eval_samples_per_second': 589.164, 'eval_steps_per_second': 73.645, 'epoch': 6.0}\n",
      "{'loss': 1.9108, 'learning_rate': 7.500000000000001e-06, 'epoch': 6.25}\n",
      "{'loss': 1.9699, 'learning_rate': 7e-06, 'epoch': 6.5}\n",
      "{'loss': 1.934, 'learning_rate': 6.5000000000000004e-06, 'epoch': 6.75}\n",
      "{'loss': 1.9089, 'learning_rate': 6e-06, 'epoch': 7.0}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "710b38d7790749788ade1f8231786d2c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/500 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 1.9423062801361084, 'eval_runtime': 6.8958, 'eval_samples_per_second': 580.064, 'eval_steps_per_second': 72.508, 'epoch': 7.0}\n",
      "{'loss': 1.8791, 'learning_rate': 5.500000000000001e-06, 'epoch': 7.25}\n",
      "{'loss': 1.9094, 'learning_rate': 5e-06, 'epoch': 7.5}\n",
      "{'loss': 1.8813, 'learning_rate': 4.5e-06, 'epoch': 7.75}\n",
      "{'loss': 1.9251, 'learning_rate': 4.000000000000001e-06, 'epoch': 8.0}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "03c642fcdeec47ac9c4b2102026de9b4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/500 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 1.8912537097930908, 'eval_runtime': 6.9772, 'eval_samples_per_second': 573.297, 'eval_steps_per_second': 71.662, 'epoch': 8.0}\n",
      "{'loss': 1.9213, 'learning_rate': 3.5e-06, 'epoch': 8.25}\n",
      "{'loss': 1.8877, 'learning_rate': 3e-06, 'epoch': 8.5}\n",
      "{'loss': 1.874, 'learning_rate': 2.5e-06, 'epoch': 8.75}\n",
      "{'loss': 1.9104, 'learning_rate': 2.0000000000000003e-06, 'epoch': 9.0}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "669f2a8ae1ae41d4bce9c1355cd45a93",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/500 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 1.8430209159851074, 'eval_runtime': 7.049, 'eval_samples_per_second': 567.457, 'eval_steps_per_second': 70.932, 'epoch': 9.0}\n",
      "{'loss': 1.8848, 'learning_rate': 1.5e-06, 'epoch': 9.25}\n",
      "{'loss': 1.8816, 'learning_rate': 1.0000000000000002e-06, 'epoch': 9.5}\n",
      "{'loss': 1.8729, 'learning_rate': 5.000000000000001e-07, 'epoch': 9.75}\n",
      "{'loss': 1.8856, 'learning_rate': 0.0, 'epoch': 10.0}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e2b5ba2b9c9c4ffbb6520cd9e3cecd23",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/500 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 1.8816810846328735, 'eval_runtime': 6.8146, 'eval_samples_per_second': 586.976, 'eval_steps_per_second': 73.372, 'epoch': 10.0}\n",
      "{'train_runtime': 1486.8466, 'train_samples_per_second': 107.61, 'train_steps_per_second': 13.451, 'train_loss': 2.0059987731933595, 'epoch': 10.0}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=20000, training_loss=2.0059987731933595, metrics={'train_runtime': 1486.8466, 'train_samples_per_second': 107.61, 'train_steps_per_second': 13.451, 'train_loss': 2.0059987731933595, 'epoch': 10.0})"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2aaca8a1689e4de8afb4c1a15a5270ec",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/500 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Perplexity: 6.64\n"
     ]
    }
   ],
   "source": [
    "import math\n",
    "\n",
    "eval_results = trainer.evaluate()\n",
    "print(f\"Perplexity: {math.exp(eval_results['eval_loss']):.2f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "text = \"The Milky Way is a <mask> galaxy.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs = tokenizer(text, return_tensors=\"pt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([6])"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "mask_token_index = torch.where(inputs[\"input_ids\"] == tokenizer.mask_token_id)[1]\n",
    "mask_token_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda', index=0)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs=inputs.to('cuda')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "logits = model(**inputs).logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[ 3.5256, -4.3728,  5.6675,  ..., -2.0497, -1.7589,  5.0884],\n",
       "         [ 4.5299, -3.5006, 13.1255,  ..., -0.5527,  0.3152,  6.1676],\n",
       "         [-1.2306, -3.3158,  2.4637,  ..., -2.3856, -2.2163,  1.2203],\n",
       "         ...,\n",
       "         [-3.2669, -4.2269,  2.3295,  ..., -3.7995, -2.9306,  1.1487],\n",
       "         [-4.2636, -4.8338,  7.9587,  ..., -2.0692, -1.4231, -1.2669],\n",
       "         [ 3.5487, -4.9553, 20.0545,  ..., -0.8984, -4.0654,  5.8322]]],\n",
       "       device='cuda:0', grad_fn=<ViewBackward0>)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 10, 50265])"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logits.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-3.6496, -3.9652,  3.4165,  ..., -2.4433, -4.3573,  0.9159]],\n",
       "       device='cuda:0', grad_fn=<IndexBackward0>)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mask_token_logits = logits[0, mask_token_index, :]\n",
    "mask_token_logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 50265])"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mask_token_logits.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Then return the three masked tokens with the highest probability and print them out:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "top_3_tokens = torch.topk(mask_token_logits, 3, dim=1).indices[0].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[21300, 2232, 30794]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_3_tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The Milky Way is a  spiral galaxy.\n",
      "The Milky Way is a  massive galaxy.\n",
      "The Milky Way is a  dwarf galaxy.\n"
     ]
    }
   ],
   "source": [
    "for token in top_3_tokens:\n",
    "    print(text.replace(tokenizer.mask_token, tokenizer.decode([token])))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Causal Language modeling\n",
    "Causal language models are frequently used for text generation. Causal language modeling predicts the next token in a sequence of tokens, and the model can only attend to tokens on the left. This means the model cannot see future tokens. GPT-2 is an example of a causal language model.\n",
    "https://huggingface.co/docs/transformers/tasks/language_modeling"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Finetune DistilGPT2 on the r/askscience subset of the ELI5 dataset: https://huggingface.co/datasets/eli5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Found cached dataset eli5 (C:/Users/lkk68/.cache/huggingface/datasets/eli5/LFQA_reddit/1.0.0/17574e5502a10f41bbd17beba83e22475b499fa62caa1384a3d093fc856fe6fa)\n"
     ]
    }
   ],
   "source": [
    "from datasets import load_dataset\n",
    "\n",
    "eli5 = load_dataset(\"eli5\", split=\"train_asks[:5000]\")\n",
    "eli5 = eli5.train_test_split(test_size=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "eli5 = eli5.flatten()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'q_id': '14p632',\n",
       " 'title': 'Can someone explain why this is a valid method of measuring the obesity rate?',\n",
       " 'selftext': '',\n",
       " 'document': '',\n",
       " 'subreddit': 'askscience',\n",
       " 'answers.a_id': ['c7f5e1w'],\n",
       " 'answers.text': [\"The article is using information directly from the CDC: [Prevalence of Obesity Among Children and Adolescents: United States, Trends 1963-1965 Through 2007-2008](_URL_1_). As far as I can tell, the cutoff for obesity was fixed to the 2000 charts, so it makes sense to compare obesity rates across different years. Those charts are based on data gathered from 1963-1994. (You can find the charts and methods [here](_URL_0_).) Notice that it hits around the exact 5% obesity (which you'd expect using the 95th percentile) in the '70s.\"],\n",
       " 'answers.score': [3],\n",
       " 'title_urls.url': [],\n",
       " 'selftext_urls.url': [],\n",
       " 'answers_urls.url': ['http://www.cdc.gov/growthcharts/',\n",
       "  'http://www.cdc.gov/nchs/data/hestat/obesity_child_07_08/obesity_child_07_08.htm']}"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eli5[\"train\"][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1cf03b51bed540c6b77e902f9b3a2a38",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)olve/main/vocab.json:   0%|          | 0.00/1.04M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "db8bf15f2f6640668f13e2b598297256",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)olve/main/merges.txt:   0%|          | 0.00/456k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8745595c36274abaab25d1bcc890108e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)lve/main/config.json:   0%|          | 0.00/762 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from transformers import GPT2Tokenizer\n",
    "tokenizer = GPT2Tokenizer.from_pretrained(\"distilgpt2\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'<|endoftext|>'"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.eos_token"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using pad_token, but it is not set yet.\n"
     ]
    }
   ],
   "source": [
    "tokenizer.pad_token"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer.pad_token = tokenizer.eos_token"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1e707577aaba450b925cc768d168acf7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)/main/tokenizer.json:   0%|          | 0.00/1.36M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from transformers import AutoTokenizer\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"distilgpt2\") #different"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TokenizerWrapper:\n",
    "    def __init__(self, tokenizer):\n",
    "        self.tokenizer = tokenizer\n",
    "    \n",
    "    def tokenize_function(self, examples):\n",
    "        return self.tokenizer(\n",
    "            [\" \".join(x) for x in examples[\"answers.text\"]],\n",
    "            padding=\"max_length\",\n",
    "            truncation=True,\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer_wrapper = TokenizerWrapper(tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "23d86bee34c540e0b2fe5342dfc2b225",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map (num_proc=3):   0%|          | 0/4000 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2f0a90661cc14d72b37e40ddb60d2f79",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map (num_proc=3):   0%|          | 0/1000 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "tokenized_dataset = eli5.map(tokenizer_wrapper.tokenize_function, batched=True, num_proc=3, remove_columns=eli5[\"train\"].column_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "def group_texts(examples):\n",
    "    block_size = 128\n",
    "    # Concatenate all texts.\n",
    "    #print(examples.keys())\n",
    "    concatenated_examples = {k: sum(examples[k], []) for k in examples.keys()}\n",
    "    total_length = len(concatenated_examples[list(examples.keys())[0]])\n",
    "    #print('total_length:', total_length)\n",
    "    # We drop the small remainder, we could add padding if the model supported it instead of this drop, you can\n",
    "    # customize this part to your needs.\n",
    "    if total_length >= block_size:\n",
    "        total_length = (total_length // block_size) * block_size\n",
    "    # Split by chunks of block_size.\n",
    "    result = {\n",
    "        k: [t[i : i + block_size] for i in range(0, total_length, block_size)]\n",
    "        for k, t in concatenated_examples.items()\n",
    "    }\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2e09f0f8e10e45769cd4d5a342a6bea7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map (num_proc=4):   0%|          | 0/4000 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0961e9143f8743ac9c132c616a257b5c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map (num_proc=4):   0%|          | 0/1000 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "lm_dataset = tokenized_dataset.map(group_texts, batched=True, num_proc=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['input_ids', 'attention_mask'],\n",
       "        num_rows: 32000\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['input_ids', 'attention_mask'],\n",
       "        num_rows: 8000\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#use the same processed dataset used in MASKED LM\n",
    "lm_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "def addlabels(examples):\n",
    "    examples[\"labels\"] = examples[\"input_ids\"].copy()\n",
    "    return examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d31a6cc1e7434edba4d66753a68637a9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/32000 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "daf7d6aca7fc4eaea74bbfbc22607a27",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/8000 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "lm_datasetlabels = lm_dataset.map(addlabels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['input_ids', 'attention_mask', 'labels'],\n",
       "        num_rows: 32000\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['input_ids', 'attention_mask', 'labels'],\n",
       "        num_rows: 8000\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lm_datasetlabels"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Use the end-of-sequence token as the padding token and set mlm=False. This will use the inputs as labels shifted to the right by one element:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import DataCollatorForLanguageModeling\n",
    "\n",
    "tokenizer.pad_token = tokenizer.eos_token\n",
    "data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0edfa486d7934aadbed1155e97ce3c20",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading model.safetensors:   0%|          | 0.00/353M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ca55021210fb415292aaba46f4d7c76d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)neration_config.json:   0%|          | 0.00/124 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from transformers import AutoModelForCausalLM, TrainingArguments, Trainer\n",
    "\n",
    "model_gpt2 = AutoModelForCausalLM.from_pretrained(\"distilgpt2\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_args = TrainingArguments(\n",
    "    output_dir=\"./output/my_awesome_eli5_clm-model\",\n",
    "    evaluation_strategy=\"epoch\",\n",
    "    learning_rate=2e-5,\n",
    "    weight_decay=0.01,\n",
    "    push_to_hub=False,\n",
    "    num_train_epochs=3\n",
    ")\n",
    "\n",
    "trainer = Trainer(\n",
    "    model=model_gpt2,\n",
    "    args=training_args,\n",
    "    train_dataset=lm_datasetlabels[\"train\"],\n",
    "    eval_dataset=lm_datasetlabels[\"test\"],\n",
    "    data_collator=data_collator,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages\\transformers\\optimization.py:411: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b1ab1d2614d548c08a7916dde239b25d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/12000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "You're using a GPT2TokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 3.9358, 'learning_rate': 1.916666666666667e-05, 'epoch': 0.12}\n",
      "{'loss': 3.9018, 'learning_rate': 1.8333333333333333e-05, 'epoch': 0.25}\n",
      "{'loss': 3.8573, 'learning_rate': 1.7500000000000002e-05, 'epoch': 0.38}\n",
      "{'loss': 3.8621, 'learning_rate': 1.6666666666666667e-05, 'epoch': 0.5}\n",
      "{'loss': 3.847, 'learning_rate': 1.5833333333333333e-05, 'epoch': 0.62}\n",
      "{'loss': 3.8301, 'learning_rate': 1.5000000000000002e-05, 'epoch': 0.75}\n",
      "{'loss': 3.8177, 'learning_rate': 1.416666666666667e-05, 'epoch': 0.88}\n",
      "{'loss': 3.7892, 'learning_rate': 1.3333333333333333e-05, 'epoch': 1.0}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a26b147c7cea480a85a40b382c409bda",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 3.701878309249878, 'eval_runtime': 16.7099, 'eval_samples_per_second': 478.758, 'eval_steps_per_second': 59.845, 'epoch': 1.0}\n",
      "{'loss': 3.6895, 'learning_rate': 1.25e-05, 'epoch': 1.12}\n",
      "{'loss': 3.6914, 'learning_rate': 1.1666666666666668e-05, 'epoch': 1.25}\n",
      "{'loss': 3.699, 'learning_rate': 1.0833333333333334e-05, 'epoch': 1.38}\n",
      "{'loss': 3.6817, 'learning_rate': 1e-05, 'epoch': 1.5}\n",
      "{'loss': 3.6866, 'learning_rate': 9.166666666666666e-06, 'epoch': 1.62}\n",
      "{'loss': 3.6908, 'learning_rate': 8.333333333333334e-06, 'epoch': 1.75}\n",
      "{'loss': 3.685, 'learning_rate': 7.500000000000001e-06, 'epoch': 1.88}\n",
      "{'loss': 3.6747, 'learning_rate': 6.666666666666667e-06, 'epoch': 2.0}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b238fbd9c9464ba0a96b9a44aff8a271",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 3.686300277709961, 'eval_runtime': 16.6824, 'eval_samples_per_second': 479.548, 'eval_steps_per_second': 59.944, 'epoch': 2.0}\n",
      "{'loss': 3.6389, 'learning_rate': 5.833333333333334e-06, 'epoch': 2.12}\n",
      "{'loss': 3.6055, 'learning_rate': 5e-06, 'epoch': 2.25}\n",
      "{'loss': 3.6351, 'learning_rate': 4.166666666666667e-06, 'epoch': 2.38}\n",
      "{'loss': 3.6183, 'learning_rate': 3.3333333333333333e-06, 'epoch': 2.5}\n",
      "{'loss': 3.6152, 'learning_rate': 2.5e-06, 'epoch': 2.62}\n",
      "{'loss': 3.6212, 'learning_rate': 1.6666666666666667e-06, 'epoch': 2.75}\n",
      "{'loss': 3.6252, 'learning_rate': 8.333333333333333e-07, 'epoch': 2.88}\n",
      "{'loss': 3.6395, 'learning_rate': 0.0, 'epoch': 3.0}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "81000a628efa49809416a753993f9871",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 3.6846420764923096, 'eval_runtime': 16.7049, 'eval_samples_per_second': 478.9, 'eval_steps_per_second': 59.863, 'epoch': 3.0}\n",
      "{'train_runtime': 780.2897, 'train_samples_per_second': 123.031, 'train_steps_per_second': 15.379, 'train_loss': 3.7224405619303385, 'epoch': 3.0}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=12000, training_loss=3.7224405619303385, metrics={'train_runtime': 780.2897, 'train_samples_per_second': 123.031, 'train_steps_per_second': 15.379, 'train_loss': 3.7224405619303385, 'epoch': 3.0})"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "47f4e5258aaa41a78720d5ec513ae2e3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Perplexity: 39.83\n"
     ]
    }
   ],
   "source": [
    "import math\n",
    "\n",
    "eval_results = trainer.evaluate()\n",
    "print(f\"Perplexity: {math.exp(eval_results['eval_loss']):.2f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "trainer.push_to_hub()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt = \"Somatic hypermutation allows the immune system to\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "A matching Triton is not available, some optimizations will not be enabled.\n",
      "Error caught was: No module named 'triton'\n",
      "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
      "c:\\Users\\lkk68\\.conda\\envs\\mycondapy39\\lib\\site-packages\\transformers\\generation\\utils.py:1353: UserWarning: Using `max_length`'s default (50) to control the generation length. This behaviour is deprecated and will be removed from the config in v5 of Transformers -- we recommend using `max_new_tokens` to control the maximum length of the generation.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[{'generated_text': 'Somatic hypermutation allows the immune system to distinguish different pathogens such as leukaemia from leukaemia in humans. However, we did not investigate any of the possible pathogenic diseases in the immune system because of these issues.\\n\\n\\n'}]"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from transformers import pipeline\n",
    "\n",
    "generator = pipeline(\"text-generation\", model=\"distilgpt2\")\n",
    "generator(prompt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs = tokenizer(prompt, return_tensors=\"pt\").input_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs=inputs.to('cuda')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda', index=0)"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inputs.device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda', index=0)"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_gpt2.device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "outputs = model_gpt2.generate(inputs, max_new_tokens=100, do_sample=True, top_k=50, top_p=0.95, pad_token_id=generator.tokenizer.eos_token_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[   50, 13730,  8718,    76,  7094,  3578,   262, 10900,  1080,   284,\n",
       "           787,   517,  4050, 13820,   621,   262,   517, 16378,  3513,   714,\n",
       "           307,   329,   262,  5827,    13,   220,   770,   318,   257,  1180,\n",
       "          3164,   422,   366,  3866,  1151,   425,  5010,   460,   470,   670,\n",
       "             1,   357,  4480,   281,  4050,  2563,   852,   262,   717,    11,\n",
       "           290,   262,  5827,   743,   307,   287,   262,  1218,   737,   220,\n",
       "           770,  5983,   284,   517,  3871,  4203,  6563,   546,  2263,   257,\n",
       "          2563,    13,   220,   770,    11,   287,   584,  2456,    11,  5983,\n",
       "           284,   517,  3871,  4203,  1365,   546,  2263,  5010,    11,   543,\n",
       "          2482,   287,   281,  2620,   287,   262,  1271,   286,  1744, 13820,\n",
       "            13,   220,   770,  5983,   284,  3649,   511,  6628,   287,   428]],\n",
       "       device='cuda:0')"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Somatic hypermutation allows the immune system to make more effective treatments than the more costly treatment could be for the patient.  This is a different approach from \"preventive drugs can\\'t work\" (with an effective drug being the first, and the patient may be in the second).  This leads to more patients feeling confident about taking a drug.  This, in other words, leads to more patients feeling better about taking drugs, which results in an increase in the number of possible treatments.  This leads to increasing their confidence in this']"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.batch_decode(outputs, skip_special_tokens=True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "mycondapy310",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}