concise_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "71fb263e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "import torch\n",
    "import torchtext as text\n",
    "import sys\n",
    "import tqdm\n",
    "from datasets import load_dataset\n",
    "import pandas as pd\n",
    "import string\n",
    "from torchtext.data import get_tokenizer\n",
    "from vocab import *\n",
    "from utils import *\n",
    "from constants import *\n",
    "import pickle\n",
    "\n",
    "from tqdm.notebook import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "3de0ed9b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Reusing dataset squad (/tmp/xdg-cache/huggingface/datasets/squad/plain_text/1.0.0/d6ec3ceb99ca480ce37cdd35555d6cb2511d223b9150cce08a837ef62ffea453)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "68b4cf5788f24e25b9e75e756ca4d846",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "dataset = load_dataset(\"squad\")\n",
    "\n",
    "train = {'passage': [], 'question': [], 'answer': []}\n",
    "for i in range(len(dataset['train'])):\n",
    "    datum = dataset['train'][i]\n",
    "    for j in range(len(datum['answers']['text'])):\n",
    "        train['passage'].append(datum['context'])\n",
    "        train['question'].append(datum['question'])\n",
    "        train['answer'].append(datum['answers']['text'][j])\n",
    "\n",
    "train = pd.DataFrame(train)\n",
    "\n",
    "val = {'passage': [], 'question': [], 'answer': []}\n",
    "for datum in dataset['validation']:\n",
    "    for elem in datum['answers']['text']:\n",
    "        ans_id = 0\n",
    "        val['passage'].append(datum['context'])\n",
    "        val['question'].append(datum['question'])\n",
    "        val['answer'].append(elem)\n",
    "\n",
    "val = pd.DataFrame(val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "39705f27",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_short_dic = {'passage': [], 'answer': [], 'question': []}\n",
    "for idx in range(len(train)):\n",
    "    row = train.iloc[idx]\n",
    "    passage, answer, question =  row['passage'].lower(), row['answer'].lower(), row['question'].lower()\n",
    "    \n",
    "    prefix, ans, suffix = passage.partition(answer)\n",
    "    context = ' '.join(prefix.split(' ')[-10:]) + ans + ' '.join(suffix.split(' ')[:10])\n",
    "    train_short_dic['passage'].append(context)\n",
    "    train_short_dic['answer'].append(answer)\n",
    "    train_short_dic['question'].append(question)\n",
    "train_short = pd.DataFrame(train_short_dic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "90ca2491",
   "metadata": {},
   "outputs": [],
   "source": [
    "val_short_dic = {'passage': [], 'answer': [], 'question': []}\n",
    "for idx in range(len(val)):\n",
    "    row = val.iloc[idx]\n",
    "    passage, answer, question =  row['passage'].lower(), row['answer'].lower(), row['question'].lower()\n",
    "    \n",
    "    prefix, ans, suffix = passage.partition(answer)\n",
    "    context = ' '.join(prefix.split(' ')[-10:]) + ans + ' '.join(suffix.split(' ')[:10])\n",
    "    val_short_dic['passage'].append(context)\n",
    "    val_short_dic['answer'].append(answer)\n",
    "    val_short_dic['question'].append(question)\n",
    "val_short = pd.DataFrame(val_short_dic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "5653498b",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Split Val into val and test\n",
    "\n",
    "val_short = val_short.sample(frac=1).reset_index(drop=True)\n",
    "test_short = val_short[:10000]\n",
    "val_short = val_short[10000:]\n",
    "\n",
    "train_short.to_csv('./data/train_short.csv', index=False)\n",
    "val_short.to_csv('./data/val_short.csv', index=False)\n",
    "test_short.to_csv('./data/test_short.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "af2b0465",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loaded vocab\n"
     ]
    }
   ],
   "source": [
    "vocab = load_vocab()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "76d8347a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "68"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer = get_tokenizer('basic_english')\n",
    "train_short['passage'].apply(lambda x: len(tokenizer(x))).max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "03cce2ed",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_processed_short_data(df, tokenizer):\n",
    "        data = []\n",
    "        for idx in tqdm(range(len(df))):\n",
    "            pass_tokens = ['<start>'] + tokenizer(df.iloc[idx][\"passage\"]) + ['<end>']\n",
    "            ans_tokens = ['<start>'] + tokenizer(df.iloc[idx][\"answer\"]) + ['<end>']\n",
    "            q_tokens = ['<start>'] + tokenizer(df.iloc[idx][\"question\"]) + ['<end>']\n",
    "\n",
    "            pass_len = MAX_SHORT_PASSAGE_LEN + 2 # +2 for start and end tokens\n",
    "            ans_len = MAX_SHORT_ANSWER_LEN + 2\n",
    "            q_len = MAX_SHORT_QUESTION_LEN + 2\n",
    "\n",
    "            passage = [vocab(word) for word in pass_tokens]\n",
    "            answer = [vocab(word) for word in ans_tokens]\n",
    "            question = [vocab(word) for word in q_tokens]\n",
    "\n",
    "            # padding to same length\n",
    "            pass_idxs = torch.zeros(pass_len)\n",
    "            ans_idxs = torch.zeros(ans_len)\n",
    "            q_idxs = torch.zeros(q_len)\n",
    "\n",
    "            pass_idxs[:len(passage)] = torch.FloatTensor(passage)\n",
    "            ans_idxs[:len(answer)] = torch.FloatTensor(answer)\n",
    "            q_idxs[:len(question)] = torch.FloatTensor(question)\n",
    "\n",
    "            data.append((pass_idxs, ans_idxs, q_idxs))\n",
    "        return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "fb49a594",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2cfa21d02e974c129f19604feef6c22b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/87599 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d22ab9c80593477bb9dcf7ff7c642b8d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/24726 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d6278e9a72124a7aac3662132d8ceeb6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/10000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "tokenizer = get_tokenizer(\"basic_english\")\n",
    "train_short_processed = get_processed_short_data(train_short, tokenizer)\n",
    "val_short_processed = get_processed_short_data(val_short, tokenizer)\n",
    "test_short_processed = get_processed_short_data(test_short, tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "80fdfb14",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([70])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_short_processed[0][0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "124d24c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('./data/train_short_processed.pickle', 'wb') as train_file:\n",
    "    pickle.dump(train_short_processed, train_file)\n",
    "\n",
    "with open('./data/val_short_processed.pickle', 'wb') as val_file:\n",
    "    pickle.dump(val_short_processed, val_file)\n",
    "\n",
    "with open('./data/test_short_processed.pickle', 'wb') as test_file:\n",
    "    pickle.dump(test_short_processed, test_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c50526c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('./data/train_short_processed.pickle', 'rb') as train_file:\n",
    "    temp = pickle.load(train_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "c2909557",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([70])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "temp[0][0].shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3dcce0bf",
   "metadata": {},
   "source": [
    "### EDA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "f9f568cf",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('./data/train_short.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "fe282bb9",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = get_tokenizer('basic_english')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "c1118fc5",
   "metadata": {},
   "outputs": [],
   "source": [
    "q_lengths = df['question'].apply(lambda x: len(tokenizer(x)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "6579c47a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAD4CAYAAADsKpHdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAATqUlEQVR4nO3db4yd5Xnn8e8vdkpcCIQ/YWRhdocKq1vAG1Is4ord1RBni7eJCi9AckWLWXllCVEplVy1dt9UXckSvKB00Ra0VshiSFtj0WaxgmgXmY52VyIQ06ZrDLGYDS5M7cVLIBRHhc3Qqy/OPcrxMPacmfH8OYfvRzo6z7nOcz9zXz62f/Pcz5kzqSokSfrEUk9AkrQ8GAiSJMBAkCQ1BoIkCTAQJEnNyqWewFxdcsklNTw8PON+P/rRjzj33HMXfkKLZJD6GaReYLD6GaRewH66vfjii29V1Wene65vA2F4eJiDBw/OuN/o6CgjIyMLP6FFMkj9DFIvMFj9DFIvYD/dkvzt6Z5zyUiSBBgIkqTGQJAkAQaCJKkxECRJgIEgSWoMBEkSYCBIkhoDQZIE9PFPKver4R1PzWv89nUT3DmHYxy958vz+rqSBp9nCJIkwECQJDUGgiQJMBAkSY2BIEkCDARJUmMgSJIAA0GS1BgIkiTAQJAkNQaCJAkwECRJTU+BkORokkNJvpvkYKtdlOSZJK+2+wu79t+ZZCzJkSQ3ddWva8cZS/JAkrT6OUkeb/Xnkwyf5T4lSTOYzRnCjVV1bVWtb493AAeqai1woD0myVXAZuBqYBPwYJIVbcxDwDZgbbttavWtwDtVdSVwP3Dv3FuSJM3FfJaMbgb2tO09wC1d9b1V9UFVvQaMAdcnWQ2cX1XPVVUBj04ZM3msJ4CNk2cPkqTFkc7/zTPslLwGvAMU8F+qaneSH1bVZ7r2eaeqLkzyn4FvV9U3Wv1h4GngKHBPVX2p1f818NtV9ZUkLwGbqmq8Pfd/gC9U1VtT5rGNzhkGQ0ND1+3du3fGuZ88eZLzzjtvxv0Wy6G/e3de44dWwZv/MPtx6y67YF5fdyEst9dmvgapn0HqBeyn24033vhi10rPKXr9BTk3VNWxJJcCzyT53hn2ne47+zpD/UxjTi1U7QZ2A6xfv75GRkbOOGmA0dFRetlvsczll9t0275ugvsOzf73Gh29fWReX3chLLfXZr4GqZ9B6gXsp1c9LRlV1bF2fwL4JnA98GZbBqLdn2i7jwOXdw1fAxxr9TXT1E8Zk2QlcAHw9uzbkSTN1YyBkOTcJJ+e3AZ+EXgJ2A9sabttAZ5s2/uBze2dQ1fQuXj8QlUdB95LsqFdH7hjypjJY90KPFu9rGVJks6aXtYehoBvtmu8K4E/rqo/T/IdYF+SrcDrwG0AVXU4yT7gZWACuLuqPmzHugt4BFhF57rC063+MPBYkjE6Zwabz0JvkqRZmDEQqur7wOemqf8A2HiaMbuAXdPUDwLXTFN/nxYokqSl4U8qS5IAA0GS1BgIkiTAQJAkNQaCJAkwECRJjYEgSQIMBElSYyBIkgADQZLUGAiSJMBAkCQ1BoIkCTAQJEmNgSBJAgwESVJjIEiSAANBktQYCJIkwECQJDUGgiQJMBAkSY2BIEkCDARJUmMgSJIAA0GS1BgIkiTAQJAkNQaCJAmYRSAkWZHkr5N8qz2+KMkzSV5t9xd27bszyViSI0lu6qpfl+RQe+6BJGn1c5I83urPJxk+iz1KknowmzOErwKvdD3eARyoqrXAgfaYJFcBm4GrgU3Ag0lWtDEPAduAte22qdW3Au9U1ZXA/cC9c+pGkjRnPQVCkjXAl4GvdZVvBva07T3ALV31vVX1QVW9BowB1ydZDZxfVc9VVQGPThkzeawngI2TZw+SpMWxssf9/gD4LeDTXbWhqjoOUFXHk1za6pcB3+7ab7zVfty2p9Ynx7zRjjWR5F3gYuCt7kkk2UbnDIOhoSFGR0dnnPjJkyd72m+xbF83Ma/xQ6vmdozl9Gcwabm9NvM1SP0MUi9gP72aMRCSfAU4UVUvJhnp4ZjTfWdfZ6ifacyphardwG6A9evX18jIzNMZHR2ll/0Wy507nprX+O3rJrjvUK85/hNHbx+Z19ddCMvttZmvQepnkHoB++lVL/+z3AD8cpJfAj4FnJ/kG8CbSVa3s4PVwIm2/zhwedf4NcCxVl8zTb17zHiSlcAFwNtz7EmSNAczXkOoqp1VtaaqhulcLH62qn4V2A9sabttAZ5s2/uBze2dQ1fQuXj8Qlteei/JhnZ94I4pYyaPdWv7Gh85Q5AkLZzZrz38xD3AviRbgdeB2wCq6nCSfcDLwARwd1V92MbcBTwCrAKebjeAh4HHkozROTPYPI95SZLmYFaBUFWjwGjb/gGw8TT77QJ2TVM/CFwzTf19WqBIkpaGP6ksSQIMBElSYyBIkgADQZLUGAiSJMBAkCQ1BoIkCTAQJEmNgSBJAgwESVJjIEiSAANBktQYCJIkwECQJDUGgiQJMBAkSY2BIEkCDARJUmMgSJIAA0GS1BgIkiTAQJAkNQaCJAkwECRJjYEgSQIMBElSYyBIkgADQZLUGAiSJKCHQEjyqSQvJPmbJIeT/F6rX5TkmSSvtvsLu8bsTDKW5EiSm7rq1yU51J57IEla/Zwkj7f680mGF6BXSdIZ9HKG8AHwxar6HHAtsCnJBmAHcKCq1gIH2mOSXAVsBq4GNgEPJlnRjvUQsA1Y226bWn0r8E5VXQncD9w7/9YkSbMxYyBUx8n28JPtVsDNwJ5W3wPc0rZvBvZW1QdV9RowBlyfZDVwflU9V1UFPDplzOSxngA2Tp49SJIWx8pedmrf4b8IXAn8YVU9n2Soqo4DVNXxJJe23S8Dvt01fLzVfty2p9Ynx7zRjjWR5F3gYuCtKfPYRucMg6GhIUZHR2ec+8mTJ3vab7FsXzcxr/FDq+Z2jOX0ZzBpub028zVI/QxSL2A/veopEKrqQ+DaJJ8BvpnkmjPsPt139nWG+pnGTJ3HbmA3wPr162tkZOQM0+gYHR2ll/0Wy507nprX+O3rJrjvUE8v2ymO3j4yr6+7EJbbazNfg9TPIPUC9tOrWb3LqKp+CIzSWft/sy0D0e5PtN3Ggcu7hq0BjrX6mmnqp4xJshK4AHh7NnOTJM1PL+8y+mw7MyDJKuBLwPeA/cCWttsW4Mm2vR/Y3N45dAWdi8cvtOWl95JsaNcH7pgyZvJYtwLPtusMkqRF0svaw2pgT7uO8AlgX1V9K8lzwL4kW4HXgdsAqupwkn3Ay8AEcHdbcgK4C3gEWAU83W4ADwOPJRmjc2aw+Ww0J0nq3YyBUFX/G/j8NPUfABtPM2YXsGua+kHgI9cfqup9WqBIkpaGP6ksSQIMBElSYyBIkgADQZLUGAiSJMBAkCQ1BoIkCTAQJEmNgSBJAgwESVJjIEiSAANBktQYCJIkwECQJDUGgiQJMBAkSc3sf1u7+tLwjqeW7GsfvefLS/a1JfXOMwRJEmAgSJIaA0GSBBgIkqTGQJAkAQaCJKkxECRJgIEgSWoMBEkSYCBIkhoDQZIEGAiSpGbGQEhyeZK/TPJKksNJvtrqFyV5Jsmr7f7CrjE7k4wlOZLkpq76dUkOteceSJJWPyfJ463+fJLhBehVknQGvZwhTADbq+rngA3A3UmuAnYAB6pqLXCgPaY9txm4GtgEPJhkRTvWQ8A2YG27bWr1rcA7VXUlcD9w71noTZI0CzMGQlUdr6q/atvvAa8AlwE3A3vabnuAW9r2zcDeqvqgql4DxoDrk6wGzq+q56qqgEenjJk81hPAxsmzB0nS4pjV70NoSzmfB54HhqrqOHRCI8mlbbfLgG93DRtvtR+37an1yTFvtGNNJHkXuBh4a8rX30bnDIOhoSFGR0dnnPPJkyd72m+xbF83Ma/xQ6vmf4zFdro//+X22szXIPUzSL2A/fSq50BIch7wp8BvVNXfn+Eb+OmeqDPUzzTm1ELVbmA3wPr162tkZGSGWXf+M+plv8Vy5zx/Uc32dRPcd6i/fq/R0dtHpq0vt9dmvgapn0HqBeynVz29yyjJJ+mEwR9V1Z+18pttGYh2f6LVx4HLu4avAY61+ppp6qeMSbISuAB4e7bNSJLmrpd3GQV4GHilqn6/66n9wJa2vQV4squ+ub1z6Ao6F49faMtL7yXZ0I55x5Qxk8e6FXi2XWeQJC2SXtYebgB+DTiU5Lut9jvAPcC+JFuB14HbAKrqcJJ9wMt03qF0d1V92MbdBTwCrAKebjfoBM5jScbonBlsnl9bkqTZmjEQqup/Mf0aP8DG04zZBeyapn4QuGaa+vu0QJEkLQ1/UlmSBBgIkqTGQJAkAQaCJKkxECRJgIEgSWoMBEkSYCBIkhoDQZIEGAiSpMZAkCQBBoIkqTEQJEmAgSBJagwESRJgIEiSGgNBkgQYCJKkxkCQJAEGgiSpMRAkSYCBIElqDARJEmAgSJIaA0GSBBgIkqTGQJAkAQaCJKkxECRJQA+BkOTrSU4keamrdlGSZ5K82u4v7HpuZ5KxJEeS3NRVvy7JofbcA0nS6uckebzVn08yfJZ7lCT1oJczhEeATVNqO4ADVbUWONAek+QqYDNwdRvzYJIVbcxDwDZgbbtNHnMr8E5VXQncD9w712YkSXM3YyBU1f8A3p5SvhnY07b3ALd01fdW1QdV9RowBlyfZDVwflU9V1UFPDplzOSxngA2Tp49SJIWz1yvIQxV1XGAdn9pq18GvNG133irXda2p9ZPGVNVE8C7wMVznJckaY5WnuXjTfedfZ2hfqYxHz14so3OshNDQ0OMjo7OOKGTJ0/2tN9i2b5uYl7jh1bN/xiL7XR//svttZmvQepnkHoB++nVXAPhzSSrq+p4Ww460erjwOVd+60BjrX6mmnq3WPGk6wELuCjS1QAVNVuYDfA+vXra2RkZMaJjo6O0st+i+XOHU/Na/z2dRPcd+hs5/jCOnr7yLT15fbazNcg9TNIvYD99GquS0b7gS1tewvwZFd9c3vn0BV0Lh6/0JaV3kuyoV0fuGPKmMlj3Qo8264zSJIW0Yzfaib5E2AEuCTJOPC7wD3AviRbgdeB2wCq6nCSfcDLwARwd1V92A51F513LK0Cnm43gIeBx5KM0Tkz2HxWOpMkzcqMgVBVv3KapzaeZv9dwK5p6geBa6apv08LFEnS0vEnlSVJgIEgSWoMBEkSYCBIkhoDQZIEGAiSpMZAkCQBBoIkqTEQJEmAgSBJagwESRJgIEiSGgNBkgQYCJKkpr9+9dZZMjzP31omSYPIMwRJEmAgSJIaA0GSBBgIkqTmY3lRWYvrdBfxt6+b4M4FvMB/9J4vL9ixpUHkGYIkCTAQJEmNgSBJAgwESVJjIEiSAANBktQYCJIkwECQJDUGgiQJMBAkSc2yCYQkm5IcSTKWZMdSz0eSPm6WxWcZJVkB/CHwb4Fx4DtJ9lfVy0s7M/Wzxf5FSJOfzeRnKKlfLZczhOuBsar6flX9f2AvcPMSz0mSPlZSVUs9B5LcCmyqqv/QHv8a8IWq+vUp+20DtrWHPwsc6eHwlwBvncXpLrVB6meQeoHB6meQegH76fbPq+qz0z2xLJaMgExT+0hSVdVuYPesDpwcrKr1c53YcjNI/QxSLzBY/QxSL2A/vVouS0bjwOVdj9cAx5ZoLpL0sbRcAuE7wNokVyT5KWAzsH+J5yRJHyvLYsmoqiaS/DrwF8AK4OtVdfgsHX5WS0x9YJD6GaReYLD6GaRewH56siwuKkuSlt5yWTKSJC0xA0GSBAx4IPTzx2Ek+XqSE0le6qpdlOSZJK+2+wuXco6zkeTyJH+Z5JUkh5N8tdX7rqckn0ryQpK/ab38Xqv3XS/dkqxI8tdJvtUe920/SY4mOZTku0kOtlpf9pPkM0meSPK99u/nFxaql4ENhK6Pw/h3wFXAryS5amlnNSuPAJum1HYAB6pqLXCgPe4XE8D2qvo5YANwd3s9+rGnD4AvVtXngGuBTUk20J+9dPsq8ErX437v58aqurbr/fr92s9/Av68qv4F8Dk6r9HC9FJVA3kDfgH4i67HO4GdSz2vWfYwDLzU9fgIsLptrwaOLPUc59Hbk3Q+u6qvewJ+Gvgr4Av93Audn/05AHwR+Far9XM/R4FLptT6rh/gfOA12huAFrqXgT1DAC4D3uh6PN5q/Wyoqo4DtPtLl3g+c5JkGPg88Dx92lNbXvkucAJ4pqr6tpfmD4DfAv6xq9bP/RTw35O82D7yBvqzn58B/h/wX9ty3teSnMsC9TLIgdDTx2FocSU5D/hT4Deq6u+Xej5zVVUfVtW1dL6zvj7JNUs8pTlL8hXgRFW9uNRzOYtuqKqfp7NkfHeSf7PUE5qjlcDPAw9V1eeBH7GAS12DHAiD+HEYbyZZDdDuTyzxfGYlySfphMEfVdWftXJf91RVPwRG6Vzv6ddebgB+OclROp80/MUk36B/+6GqjrX7E8A36Xyicj/2Mw6MtzNQgCfoBMSC9DLIgTCIH4exH9jStrfQWYfvC0kCPAy8UlW/3/VU3/WU5LNJPtO2VwFfAr5HH/YCUFU7q2pNVQ3T+XfybFX9Kn3aT5Jzk3x6chv4ReAl+rCfqvq/wBtJfraVNgIvs0C9DPRPKif5JTpro5Mfh7FraWfUuyR/AozQ+ZjbN4HfBf4bsA/4Z8DrwG1V9fYSTXFWkvwr4H8Ch/jJOvXv0LmO0Fc9JfmXwB46f68+Aeyrqv+Y5GL6rJepkowAv1lVX+nXfpL8DJ2zAugsufxxVe3q436uBb4G/BTwfeDf0/7ecZZ7GehAkCT1bpCXjCRJs2AgSJIAA0GS1BgIkiTAQJAkNQaCJAkwECRJzT8BP3e9x3zoBIAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "q_lengths.hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "09472726",
   "metadata": {},
   "outputs": [],
   "source": [
    "p_lengths = df['passage'].apply(lambda x: len(tokenizer(x)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "5c379071",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAD4CAYAAAAD6PrjAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAARxklEQVR4nO3cf6zd9X3f8ecrOE0oLQk/wpWF2UyElZXghRSLMmWa7uJucZco5A+QXKWNmZgsRVRKJE+V6T9VJ1kif7TpIjVIVshw0i7Eo+2wgrINmV7thwgU0nQOEAureODh4VFSgjOF5bL3/jif++nhcu1777m+vvccPx/S0fme9/l+vufz9iF53c/3e85JVSFJEsA71noCkqT1w1CQJHWGgiSpMxQkSZ2hIEnqNqz1BEZ15ZVX1ubNm0ca++Mf/5hLLrnk3E5ojU1aT5PWD0xeT5PWD0xeTwv189RTT71SVe8705ixDYXNmzfz5JNPjjR2ZmaG6enpczuhNTZpPU1aPzB5PU1aPzB5PS3UT5L/cbYxnj6SJHWGgiSpMxQkSZ2hIEnqDAVJUmcoSJI6Q0GS1BkKkqTOUJAkdWP7jWYtz+a9D6/Zax+/5+Nr9tqSlseVgiSpMxQkSZ2hIEnqDAVJUmcoSJI6Q0GS1BkKkqTOUJAkdYaCJKkzFCRJnaEgSeoMBUlSZyhIkjpDQZLUGQqSpM5QkCR1hoIkqTMUJEmdoSBJ6gwFSVJnKEiSuiWHQpKLkvxFkm+1x5cneSTJc+3+sqF9705yLMnRJB8bqt+U5Eh77ktJ0urvSvLNVn88yeZz2KMkaYmWs1L4HPDs0OO9wOGq2gIcbo9Jcj2wE/ggsAP4cpKL2ph7gd3Alnbb0ep3Aj+squuALwJfGKkbSdKKLCkUkmwCPg58Zah8K3CgbR8APjVUf6Cq3qiq54FjwM1JNgKXVtVjVVXA1+aNmTvWg8D2uVWEJOn82bDE/X4f+E3g54dqU1V1EqCqTia5qtWvBr4ztN+JVvtp255fnxvzYjvWbJLXgCuAV4YnkWQ3g5UGU1NTzMzMLHH6b3X69OmRx65Xi/W0Z+vs+ZvMPKP8W1+I79G4mbR+YPJ6GqWfRUMhySeAU1X1VJLpJRxzob/w6yz1s415a6FqP7AfYNu2bTU9vZTpvN3MzAyjjl2vFuvpjr0Pn7/JzHP809PLHnMhvkfjZtL6gcnraZR+lrJS+AjwyST/DHg3cGmSPwReTrKxrRI2Aqfa/ieAa4bGbwJeavVNC9SHx5xIsgF4D/DqsjqRJK3YotcUquruqtpUVZsZXEB+tKp+DTgE7Gq77QIeatuHgJ3tE0XXMrig/EQ71fR6klva9YLPzBszd6zb2mu8baUgSVpdS72msJB7gINJ7gReAG4HqKqnkxwEngFmgbuq6s025rPA/cDFwLfbDeA+4OtJjjFYIexcwbwkSSNaVihU1Qww07b/Gth+hv32AfsWqD8J3LBA/Se0UJEkrR2/0SxJ6gwFSVJnKEiSOkNBktQZCpKkzlCQJHWGgiSpMxQkSZ2hIEnqDAVJUmcoSJI6Q0GS1BkKkqTOUJAkdYaCJKkzFCRJnaEgSeoMBUlSZyhIkjpDQZLUGQqSpM5QkCR1hoIkqTMUJEmdoSBJ6gwFSVJnKEiSOkNBktQZCpKkzlCQJHWGgiSpMxQkSZ2hIEnqDAVJUmcoSJI6Q0GS1C0aCkneneSJJH+Z5Okkv9Pqlyd5JMlz7f6yoTF3JzmW5GiSjw3Vb0pypD33pSRp9Xcl+WarP55k8yr0KklaxFJWCm8AH62qDwE3AjuS3ALsBQ5X1RbgcHtMkuuBncAHgR3Al5Nc1I51L7Ab2NJuO1r9TuCHVXUd8EXgCytvTZK0XIuGQg2cbg/f2W4F3AocaPUDwKfa9q3AA1X1RlU9DxwDbk6yEbi0qh6rqgK+Nm/M3LEeBLbPrSIkSefPhqXs1P7Sfwq4DviDqno8yVRVnQSoqpNJrmq7Xw18Z2j4iVb7adueX58b82I71myS14ArgFfmzWM3g5UGU1NTzMzMLLHNtzp9+vTIY9erxXras3X2/E1mnlH+rS/E92jcTFo/MHk9jdLPkkKhqt4EbkzyXuBPk9xwlt0X+gu/zlI/25j589gP7AfYtm1bTU9Pn2UaZzYzM8OoY9erxXq6Y+/D528y8xz/9PSyx1yI79G4mbR+YPJ6GqWfZX36qKr+BphhcC3g5XZKiHZ/qu12ArhmaNgm4KVW37RA/S1jkmwA3gO8upy5SZJWbimfPnpfWyGQ5GLgl4EfAIeAXW23XcBDbfsQsLN9ouhaBheUn2inml5Pcku7XvCZeWPmjnUb8Gi77iBJOo+WcvpoI3CgXVd4B3Cwqr6V5DHgYJI7gReA2wGq6ukkB4FngFngrnb6CeCzwP3AxcC32w3gPuDrSY4xWCHsPBfNSZKWZ9FQqKr/Dnx4gfpfA9vPMGYfsG+B+pPA265HVNVPaKEiSVo7fqNZktQZCpKkzlCQJHWGgiSpMxQkSZ2hIEnqDAVJUmcoSJI6Q0GS1BkKkqTOUJAkdYaCJKkzFCRJnaEgSeoMBUlSZyhIkjpDQZLUGQqSpM5QkCR1hoIkqTMUJEmdoSBJ6gwFSVJnKEiSOkNBktQZCpKkbsNaT+BCs3nvw6ty3D1bZ7ljlY4t6cLhSkGS1BkKkqTOUJAkdYaCJKkzFCRJnaEgSeoMBUlSZyhIkjpDQZLUGQqSpG7RUEhyTZI/S/JskqeTfK7VL0/ySJLn2v1lQ2PuTnIsydEkHxuq35TkSHvuS0nS6u9K8s1WfzzJ5lXoVZK0iKWsFGaBPVX1C8AtwF1Jrgf2AoeragtwuD2mPbcT+CCwA/hykovase4FdgNb2m1Hq98J/LCqrgO+CHzhHPQmSVqmRUOhqk5W1Xfb9uvAs8DVwK3AgbbbAeBTbftW4IGqeqOqngeOATcn2QhcWlWPVVUBX5s3Zu5YDwLb51YRkqTzZ1m/ktpO63wYeByYqqqTMAiOJFe13a4GvjM07ESr/bRtz6/PjXmxHWs2yWvAFcAr815/N4OVBlNTU8zMzCxn+t3p06dHHrtSe7bOrspxpy5evWOv1Cj/1mv5Hq2WSetp0vqByetplH6WHApJfg74Y+DzVfWjs/whv9ATdZb62ca8tVC1H9gPsG3btpqenl5k1gubmZlh1LErtVo/b71n6yy/e2R9/hL68U9PL3vMWr5Hq2XSepq0fmDyehqlnyV9+ijJOxkEwh9V1Z+08svtlBDt/lSrnwCuGRq+CXip1TctUH/LmCQbgPcAry6rE0nSii3l00cB7gOerarfG3rqELCrbe8CHhqq72yfKLqWwQXlJ9qppteT3NKO+Zl5Y+aOdRvwaLvuIEk6j5ZyvuEjwK8DR5J8r9V+C7gHOJjkTuAF4HaAqno6yUHgGQafXLqrqt5s4z4L3A9cDHy73WAQOl9PcozBCmHnytqSJI1i0VCoqv/Kwuf8AbafYcw+YN8C9SeBGxao/4QWKpKkteM3miVJnaEgSeoMBUlSZyhIkjpDQZLUGQqSpM5QkCR1hoIkqTMUJEmdoSBJ6gwFSVJnKEiSOkNBktQZCpKkzlCQJHWGgiSpMxQkSZ2hIEnqDAVJUmcoSJI6Q0GS1BkKkqTOUJAkdYaCJKkzFCRJnaEgSeoMBUlSZyhIkjpDQZLUGQqSpM5QkCR1hoIkqTMUJEmdoSBJ6gwFSVJnKEiSukVDIclXk5xK8v2h2uVJHknyXLu/bOi5u5McS3I0yceG6jclOdKe+1KStPq7knyz1R9Psvkc9yhJWqKlrBTuB3bMq+0FDlfVFuBwe0yS64GdwAfbmC8nuaiNuRfYDWxpt7lj3gn8sKquA74IfGHUZiRJK7NoKFTVfwZenVe+FTjQtg8AnxqqP1BVb1TV88Ax4OYkG4FLq+qxqirga/PGzB3rQWD73CpCknR+bRhx3FRVnQSoqpNJrmr1q4HvDO13otV+2rbn1+fGvNiONZvkNeAK4JX5L5pkN4PVBlNTU8zMzIw0+dOnT488dqX2bJ1dleNOXbx6x16pUf6t1/I9Wi2T1tOk9QOT19Mo/YwaCmey0F/4dZb62ca8vVi1H9gPsG3btpqenh5hioP/kxp17ErdsffhVTnunq2z/O6Rc/12nhvHPz297DFr+R6tlknradL6gcnraZR+Rv300cvtlBDt/lSrnwCuGdpvE/BSq29aoP6WMUk2AO/h7aerJEnnwaihcAjY1bZ3AQ8N1Xe2TxRdy+CC8hPtVNPrSW5p1ws+M2/M3LFuAx5t1x0kSefZoucbknwDmAauTHIC+G3gHuBgkjuBF4DbAarq6SQHgWeAWeCuqnqzHeqzDD7JdDHw7XYDuA/4epJjDFYIO89JZ5KkZVs0FKrqV8/w1PYz7L8P2LdA/UnghgXqP6GFiiRpbfmNZklSZyhIkjpDQZLUrc8PtmuibB7huxl7ts6u+Dsdx+/5+IrGSxciVwqSpO6CXCkc+Z+vrdo3iyVpnLlSkCR1hoIkqTMUJEmdoSBJ6gwFSVJnKEiSOkNBktQZCpKkzlCQJHWGgiSpMxQkSZ2hIEnqDAVJUmcoSJI6Q0GS1BkKkqTOUJAkdYaCJKkzFCRJnaEgSeoMBUlSZyhIkjpDQZLUGQqSpM5QkCR1G9Z6AtJq2bz34TV53eP3fHxNXlc6F1wpSJI6Q0GS1BkKkqTOUJAkdV5ols6xM13g3rN1ljtW+eK3F7m1UutmpZBkR5KjSY4l2bvW85GkC9G6WCkkuQj4A+CfACeAP09yqKqeWduZSePlfH4Md3jl4wplcqyXlcLNwLGq+quq+r/AA8CtazwnSbrgpKrWeg4kuQ3YUVX/oj3+deCXquo35u23G9jdHn4AODriS14JvDLi2PVq0nqatH5g8nqatH5g8npaqJ+/W1XvO9OAdXH6CMgCtbelVVXtB/av+MWSJ6tq20qPs55MWk+T1g9MXk+T1g9MXk+j9LNeTh+dAK4ZerwJeGmN5iJJF6z1Egp/DmxJcm2SnwF2AofWeE6SdMFZF6ePqmo2yW8A/xG4CPhqVT29ii+54lNQ69Ck9TRp/cDk9TRp/cDk9bTsftbFhWZJ0vqwXk4fSZLWAUNBktRNfCgk+WqSU0m+P1S7PMkjSZ5r95et5RyXI8k1Sf4sybNJnk7yuVYf557eneSJJH/ZevqdVh/bnmDwTf0kf5HkW+3xuPdzPMmRJN9L8mSrjW1PSd6b5MEkP2j/e/oHY97PB9p7M3f7UZLPL7eniQ8F4H5gx7zaXuBwVW0BDrfH42IW2FNVvwDcAtyV5HrGu6c3gI9W1YeAG4EdSW5hvHsC+Bzw7NDjce8H4B9X1Y1Dn30f557+NfAfqurvAR9i8F6NbT9VdbS9NzcCNwH/B/hTlttTVU38DdgMfH/o8VFgY9veCBxd6zmuoLeHGPxm1ET0BPws8F3gl8a5JwbftTkMfBT4VquNbT9tzseBK+fVxrIn4FLgedqHbca9nwX6+6fAfxulpwthpbCQqao6CdDur1rj+YwkyWbgw8DjjHlP7VTL94BTwCNVNe49/T7wm8D/G6qNcz8w+JWB/5TkqfaTMzC+Pb0f+N/Av2mn+L6S5BLGt5/5dgLfaNvL6ulCDYWxl+TngD8GPl9VP1rr+axUVb1Zg2XvJuDmJDes8ZRGluQTwKmqemqt53KOfaSqfhH4FQanLf/RWk9oBTYAvwjcW1UfBn7MGJ0qOpv2BeBPAv9ulPEXaii8nGQjQLs/tcbzWZYk72QQCH9UVX/SymPd05yq+htghsF1oHHt6SPAJ5McZ/CLvx9N8oeMbz8AVNVL7f4Ug3PVNzO+PZ0ATrQVKcCDDEJiXPsZ9ivAd6vq5fZ4WT1dqKFwCNjVtncxOC8/FpIEuA94tqp+b+ipce7pfUne27YvBn4Z+AFj2lNV3V1Vm6pqM4Nl/KNV9WuMaT8ASS5J8vNz2wzOWX+fMe2pqv4X8GKSD7TSduAZxrSfeX6Vvz11BMvsaeK/0ZzkG8A0g5+QfRn4beDfAweBvwO8ANxeVa+u0RSXJck/BP4LcIS/PV/9WwyuK4xrT38fOMDgJ07eARysqn+V5ArGtKc5SaaBf1lVnxjnfpK8n8HqAAanXv5tVe0b855uBL4C/AzwV8A/p/33xxj2A5DkZ4EXgfdX1Wuttqz3aOJDQZK0dBfq6SNJ0gIMBUlSZyhIkjpDQZLUGQqSpM5QkCR1hoIkqfv/5K721EcDOcQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "p_lengths.hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "a43bf400",
   "metadata": {},
   "outputs": [],
   "source": [
    "a_lengths = df['answer'].apply(lambda x: len(tokenizer(str(x))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "041f13a4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAD4CAYAAADsKpHdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAWVUlEQVR4nO3db4xd9X3n8fendkJYuhAgYYRsds0KKy1/NmQZsV5ltZqN28VtopgHIE1Ei1l55RWiVSKxqkyfVF3JEjwpDVJBa4UshmYDFm2KFUR3kelVtxKYmDRdYghiNlDw4sVLIIRJBWXodx/c38h3hjt3ZmzPXOP7fklX99zv/f3O/Z2vEn/mnHNnSFUhSdIvDHsBkqRTg4EgSQIMBElSYyBIkgADQZLUrB32Ao7Xpz71qdqwYcPAMT//+c8566yzVmdBpzh7MZf9OMZezHW69+OZZ555o6o+3e+9j2wgbNiwgYMHDw4c0+l0mJiYWJ0FneLsxVz24xh7Mdfp3o8kf7vQe14ykiQBBoIkqTEQJEmAgSBJagwESRJgIEiSGgNBkgQYCJKkxkCQJAEf4d9UPhEbdj46tM9++fYvDu2zJWkQzxAkSYCBIElqDARJEmAgSJIaA0GSBBgIkqTGQJAkAQaCJKlZNBCSfCbJD3oeP0vytSTnJXk8yYvt+dyeObclmUryQpJreupXJXm2vXdXkrT6GUkeavUDSTasyNFKkha0aCBU1QtVdWVVXQlcBfwd8B1gJ7C/qjYC+9trklwKTAKXAVuAu5Osabu7B9gBbGyPLa2+HXirqi4B7gTuOClHJ0lasuVeMtoM/O+q+ltgK7Cn1fcA17btrcCDVfVeVb0ETAFXJ7kQOLuqnqyqAu6fN2d2Xw8Dm2fPHiRJq2O5f8toEvh22x6rqiMAVXUkyQWtvg54qmfO4VZ7v23Pr8/OebXtaybJ28D5wBu9H55kB90zDMbGxuh0OgMXOz093XfMrVfMDJy3khZb80pZqBejyn4cYy/mGuV+LDkQknwc+DJw22JD+9RqQH3QnLmFqt3AboDx8fGamJgYuJBOp0O/MTcN84/b3TAxlM9dqBejyn4cYy/mGuV+LOeS0a8B36+q19vr19tlINrz0VY/DFzUM2898Fqrr+9TnzMnyVrgHODNZaxNknSClhMIX+HY5SKAfcC2tr0NeKSnPtm+OXQx3ZvHT7fLS+8k2dTuD9w4b87svq4Dnmj3GSRJq2RJl4yS/CPgV4H/2FO+HdibZDvwCnA9QFUdSrIXeA6YAW6pqg/anJuB+4AzgcfaA+Be4IEkU3TPDCZP4JgkScdhSYFQVX9H9yZvb+0ndL911G/8LmBXn/pB4PI+9XdpgSJJGg5/U1mSBBgIkqTGQJAkAQaCJKkxECRJgIEgSWoMBEkSYCBIkhoDQZIEGAiSpMZAkCQBBoIkqTEQJEmAgSBJagwESRJgIEiSGgNBkgQYCJKkxkCQJAFLDIQkn0zycJIfJXk+yb9Kcl6Sx5O82J7P7Rl/W5KpJC8kuaanflWSZ9t7dyVJq5+R5KFWP5Bkw0k/UknSQEs9Q/g68OdV9UvAZ4HngZ3A/qraCOxvr0lyKTAJXAZsAe5Osqbt5x5gB7CxPba0+nbgraq6BLgTuOMEj0uStEyLBkKSs4F/A9wLUFV/X1U/BbYCe9qwPcC1bXsr8GBVvVdVLwFTwNVJLgTOrqonq6qA++fNmd3Xw8Dm2bMHSdLqWLuEMf8M+H/Af03yWeAZ4KvAWFUdAaiqI0kuaOPXAU/1zD/cau+37fn12Tmvtn3NJHkbOB94o3chSXbQPcNgbGyMTqczcOHT09N9x9x6xczAeStpsTWvlIV6MarsxzH2Yq5R7sdSAmEt8C+A366qA0m+Trs8tIB+P9nXgPqgOXMLVbuB3QDj4+M1MTExYBndf3z7jblp56MD562kl2+YGMrnLtSLUWU/jrEXc41yP5ZyD+EwcLiqDrTXD9MNiNfbZSDa89Ge8Rf1zF8PvNbq6/vU58xJshY4B3hzuQcjSTp+iwZCVf1f4NUkn2mlzcBzwD5gW6ttAx5p2/uAyfbNoYvp3jx+ul1eeifJpnZ/4MZ5c2b3dR3wRLvPIElaJUu5ZATw28C3knwc+DHw7+mGyd4k24FXgOsBqupQkr10Q2MGuKWqPmj7uRm4DzgTeKw9oHvD+oEkU3TPDCZP8LgkScu0pECoqh8A433e2rzA+F3Arj71g8Dlferv0gJFkjQc/qayJAkwECRJjYEgSQIMBElSYyBIkgADQZLUGAiSJMBAkCQ1BoIkCTAQJEmNgSBJAgwESVJjIEiSAANBktQYCJIkwECQJDUGgiQJMBAkSY2BIEkClhgISV5O8mySHyQ52GrnJXk8yYvt+dye8bclmUryQpJreupXtf1MJbkrSVr9jCQPtfqBJBtO8nFKkhaxnDOEf1tVV1bVeHu9E9hfVRuB/e01SS4FJoHLgC3A3UnWtDn3ADuAje2xpdW3A29V1SXAncAdx39IkqTjcSKXjLYCe9r2HuDanvqDVfVeVb0ETAFXJ7kQOLuqnqyqAu6fN2d2Xw8Dm2fPHiRJq2PtEscV8D+SFPBfqmo3MFZVRwCq6kiSC9rYdcBTPXMPt9r7bXt+fXbOq21fM0neBs4H3uhdRJIddM8wGBsbo9PpDFz09PR03zG3XjEz+GhX0GJrXikL9WJU2Y9j7MVco9yPpQbC56vqtfaP/uNJfjRgbL+f7GtAfdCcuYVuEO0GGB8fr4mJiYGL7nQ69Btz085HB85bSS/fMDGUz12oF6PKfhxjL+Ya5X4s6ZJRVb3Wno8C3wGuBl5vl4Foz0fb8MPART3T1wOvtfr6PvU5c5KsBc4B3lz+4UiSjteigZDkrCT/eHYb+HfAD4F9wLY2bBvwSNveB0y2bw5dTPfm8dPt8tI7STa1+wM3zpszu6/rgCfafQZJ0ipZyiWjMeA77R7vWuC/VdWfJ/kesDfJduAV4HqAqjqUZC/wHDAD3FJVH7R93QzcB5wJPNYeAPcCDySZontmMHkSjk2StAyLBkJV/Rj4bJ/6T4DNC8zZBezqUz8IXN6n/i4tUCRJw+FvKkuSAANBktQYCJIkwECQJDUGgiQJMBAkSY2BIEkCDARJUmMgSJIAA0GS1BgIkiTAQJAkNQaCJAkwECRJjYEgSQIMBElSYyBIkgADQZLULDkQkqxJ8tdJvtten5fk8SQvtudze8belmQqyQtJrumpX5Xk2fbeXWn/oeYkZyR5qNUPJNlwEo9RkrQEyzlD+CrwfM/rncD+qtoI7G+vSXIpMAlcBmwB7k6yps25B9gBbGyPLa2+HXirqi4B7gTuOK6jkSQdtyUFQpL1wBeBb/SUtwJ72vYe4Nqe+oNV9V5VvQRMAVcnuRA4u6qerKoC7p83Z3ZfDwObZ88eJEmrY6lnCH8I/A7wDz21sao6AtCeL2j1dcCrPeMOt9q6tj2/PmdOVc0AbwPnL/UgJEknbu1iA5J8CThaVc8kmVjCPvv9ZF8D6oPmzF/LDrqXnBgbG6PT6QxcyPT0dN8xt14xM3DeSlpszStloV6MKvtxjL2Ya5T7sWggAJ8Hvpzk14FPAGcn+WPg9SQXVtWRdjnoaBt/GLioZ/564LVWX9+n3jvncJK1wDnAm/MXUlW7gd0A4+PjNTExMXDhnU6HfmNu2vnowHkr6eUbJobyuQv1YlTZj2PsxVyj3I9FLxlV1W1Vtb6qNtC9WfxEVf0GsA/Y1oZtAx5p2/uAyfbNoYvp3jx+ul1WeifJpnZ/4MZ5c2b3dV37jA+dIUiSVs5SzhAWcjuwN8l24BXgeoCqOpRkL/AcMAPcUlUftDk3A/cBZwKPtQfAvcADSabonhlMnsC6JEnHYVmBUFUdoNO2fwJsXmDcLmBXn/pB4PI+9XdpgSJJGg5/U1mSBBgIkqTGQJAkAQaCJKkxECRJgIEgSWoMBEkSYCBIkhoDQZIEGAiSpMZAkCQBBoIkqTEQJEmAgSBJagwESRJgIEiSGgNBkgQYCJKkxkCQJAFLCIQkn0jydJK/SXIoye+3+nlJHk/yYns+t2fObUmmkryQ5Jqe+lVJnm3v3ZUkrX5Gkoda/UCSDStwrJKkAZZyhvAe8IWq+ixwJbAlySZgJ7C/qjYC+9trklwKTAKXAVuAu5Osafu6B9gBbGyPLa2+HXirqi4B7gTuOPFDkyQtx6KBUF3T7eXH2qOArcCeVt8DXNu2twIPVtV7VfUSMAVcneRC4OyqerKqCrh/3pzZfT0MbJ49e5AkrY61SxnUfsJ/BrgE+KOqOpBkrKqOAFTVkSQXtOHrgKd6ph9utffb9vz67JxX275mkrwNnA+8MW8dO+ieYTA2Nkan0xm47unp6b5jbr1iZvABr6DF1rxSFurFqLIfx9iLuUa5H0sKhKr6ALgyySeB7yS5fMDwfj/Z14D6oDnz17Eb2A0wPj5eExMTA5bR/ce335ibdj46cN5KevmGiaF87kK9GFX24xh7Mdco92NZ3zKqqp8CHbrX/l9vl4Foz0fbsMPART3T1gOvtfr6PvU5c5KsBc4B3lzO2iRJJ2Yp3zL6dDszIMmZwK8APwL2AdvasG3AI217HzDZvjl0Md2bx0+3y0vvJNnU7g/cOG/O7L6uA55o9xkkSatkKZeMLgT2tPsIvwDsrarvJnkS2JtkO/AKcD1AVR1Kshd4DpgBbmmXnABuBu4DzgQeaw+Ae4EHkkzRPTOYPBkHJ0laukUDoar+F/C5PvWfAJsXmLML2NWnfhD40P2HqnqXFiiSpOHwN5UlSYCBIElqDARJEmAgSJIaA0GSBBgIkqTGQJAkAQaCJKkxECRJgIEgSWoMBEkSYCBIkhoDQZIEGAiSpMZAkCQBBoIkqTEQJEmAgSBJagwESRKwhEBIclGSv0jyfJJDSb7a6ucleTzJi+353J45tyWZSvJCkmt66lcleba9d1eStPoZSR5q9QNJNqzAsUqSBljKGcIMcGtV/TKwCbglyaXATmB/VW0E9rfXtPcmgcuALcDdSda0fd0D7AA2tseWVt8OvFVVlwB3AnechGOTJC3DooFQVUeq6vtt+x3geWAdsBXY04btAa5t21uBB6vqvap6CZgCrk5yIXB2VT1ZVQXcP2/O7L4eBjbPnj1IklbH2uUMbpdyPgccAMaq6gh0QyPJBW3YOuCpnmmHW+39tj2/Pjvn1bavmSRvA+cDb8z7/B10zzAYGxuj0+kMXO/09HTfMbdeMTNw3kpabM0rZaFejCr7cYy9mGuU+7HkQEjyi8CfAF+rqp8N+AG+3xs1oD5oztxC1W5gN8D4+HhNTEwMXHOn06HfmJt2Pjpw3kp6+YaJoXzuQr0YVfbjGHsx1yj3Y0nfMkryMbph8K2q+tNWfr1dBqI9H231w8BFPdPXA6+1+vo+9TlzkqwFzgHeXO7BSJKO31K+ZRTgXuD5qvqDnrf2Adva9jbgkZ76ZPvm0MV0bx4/3S4vvZNkU9vnjfPmzO7rOuCJdp9BkrRKlnLJ6PPAbwLPJvlBq/0ucDuwN8l24BXgeoCqOpRkL/Ac3W8o3VJVH7R5NwP3AWcCj7UHdAPngSRTdM8MJk/ssCRJy7VoIFTVX9H/Gj/A5gXm7AJ29akfBC7vU3+XFiiSpOHwN5UlSYCBIElqDARJEmAgSJIaA0GSBBgIkqTGQJAkAQaCJKkxECRJgIEgSWoMBEkSYCBIkhoDQZIEGAiSpMZAkCQBBoIkqTEQJEmAgSBJagwESRKwhEBI8s0kR5P8sKd2XpLHk7zYns/tee+2JFNJXkhyTU/9qiTPtvfuSpJWPyPJQ61+IMmGk3yMkqQlWMoZwn3Alnm1ncD+qtoI7G+vSXIpMAlc1ubcnWRNm3MPsAPY2B6z+9wOvFVVlwB3Ancc78FIko7fooFQVX8JvDmvvBXY07b3ANf21B+sqveq6iVgCrg6yYXA2VX1ZFUVcP+8ObP7ehjYPHv2IElaPWuPc95YVR0BqKojSS5o9XXAUz3jDrfa+217fn12zqttXzNJ3gbOB96Y/6FJdtA9y2BsbIxOpzNwkdPT033H3HrFzMB5K2mxNa+UhXoxquzHMfZirlHux/EGwkL6/WRfA+qD5ny4WLUb2A0wPj5eExMTAxfT6XToN+amnY8OnLeSXr5hYiifu1AvRpX9OMZezDXK/Tjebxm93i4D0Z6Ptvph4KKeceuB11p9fZ/6nDlJ1gLn8OFLVJKkFXa8gbAP2Na2twGP9NQn2zeHLqZ78/jpdnnpnSSb2v2BG+fNmd3XdcAT7T6DJGkVLXrJKMm3gQngU0kOA78H3A7sTbIdeAW4HqCqDiXZCzwHzAC3VNUHbVc30/3G0pnAY+0BcC/wQJIpumcGkyflyCRJy7JoIFTVVxZ4a/MC43cBu/rUDwKX96m/SwsUSdLw+JvKkiTg5H/LSIvYMKRvON235ayhfK6kjw7PECRJgIEgSWoMBEkSYCBIkhoDQZIEGAiSpMZAkCQBBoIkqTEQJEmAgSBJagwESRLg3zIaGc/+n7eH9l+Ke/n2Lw7lcyUtj2cIkiTAQJAkNQaCJAkwECRJjYEgSQJOoW8ZJdkCfB1YA3yjqm4f8pJ0kgzrvxLnt5uk5TklAiHJGuCPgF8FDgPfS7Kvqp4b7sr0UTYoiG69YmbFvoZrEOmj6pQIBOBqYKqqfgyQ5EFgK2Ag6CNnWGdEx+tkhKMheHpIVQ17DSS5DthSVf+hvf5N4F9W1W/NG7cD2NFefgZ4YZFdfwp44yQv96PKXsxlP46xF3Od7v34p1X16X5vnCpnCOlT+1BSVdVuYPeSd5ocrKrxE1nY6cJezGU/jrEXc41yP06VbxkdBi7qeb0eeG1Ia5GkkXSqBML3gI1JLk7ycWAS2DfkNUnSSDklLhlV1UyS3wL+O92vnX6zqg6dhF0v+fLSCLAXc9mPY+zFXCPbj1PiprIkafhOlUtGkqQhMxAkScBpGghJtiR5IclUkp3DXs9qS/LNJEeT/LCndl6Sx5O82J7PHeYaV0uSi5L8RZLnkxxK8tVWH9V+fCLJ00n+pvXj91t9JPsB3b+UkOSvk3y3vR7ZXpx2gdDzZzB+DbgU+EqSS4e7qlV3H7BlXm0nsL+qNgL72+tRMAPcWlW/DGwCbmn/exjVfrwHfKGqPgtcCWxJsonR7QfAV4Hne16PbC9Ou0Cg589gVNXfA7N/BmNkVNVfAm/OK28F9rTtPcC1q7mmYamqI1X1/bb9Dt3/469jdPtRVTXdXn6sPYoR7UeS9cAXgW/0lEeyF3B6BsI64NWe14dbbdSNVdUR6P4jCVww5PWsuiQbgM8BBxjhfrRLJD8AjgKPV9Uo9+MPgd8B/qGnNqq9OC0DYUl/BkOjJckvAn8CfK2qfjbs9QxTVX1QVVfS/YsAVye5fMhLGookXwKOVtUzw17LqeJ0DAT/DEZ/rye5EKA9Hx3yelZNko/RDYNvVdWftvLI9mNWVf0U6NC93zSK/fg88OUkL9O9tPyFJH/MaPYCOD0DwT+D0d8+YFvb3gY8MsS1rJokAe4Fnq+qP+h5a1T78ekkn2zbZwK/AvyIEexHVd1WVeuragPdfyeeqKrfYAR7Meu0/E3lJL9O99rg7J/B2DXcFa2uJN8GJuj+Gd/Xgd8D/gzYC/wT4BXg+qqaf+P5tJPkXwP/E3iWY9eJf5fufYRR7Mc/p3ujdA3dHwj3VtV/TnI+I9iPWUkmgP9UVV8a5V6cloEgSVq+0/GSkSTpOBgIkiTAQJAkNQaCJAkwECRJjYEgSQIMBElS8/8BG4y+dCxOTaYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "a_lengths.hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "483176a3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    87599.000000\n",
       "mean         3.444023\n",
       "std          3.799090\n",
       "min          1.000000\n",
       "25%          1.000000\n",
       "50%          2.000000\n",
       "75%          4.000000\n",
       "max         46.000000\n",
       "Name: answer, dtype: float64"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a_lengths.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "db6259d2",
   "metadata": {},
   "outputs": [],
   "source": [
    "def max_outlier(lengths):\n",
    "    stats = lengths.describe()\n",
    "    iqr = stats['75%'] - stats['25%']\n",
    "    outlier_thresh = 1.75 * iqr + stats['75%']\n",
    "    return outlier_thresh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "e817b4eb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(20.0, 32.0, 9.25)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max_outlier(q_lengths), max_outlier(p_lengths), max_outlier(a_lengths)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "73046c3a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (clean)",
   "language": "python",
   "name": "python3_clean"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}