inferencia-kernel.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['pickleds', 'gpt2-inferece-w-identity', 'bert-inference-w-identityseed5060', 'modelos-entrenados-toxic', 'ppbert', 'jigsaw-unintended-bias-in-toxicity-classification', 'bert-inference-w-identity-1', 'gpt2-pytorch', 'bert-inference-w-identity', 'bert-inference-w-identity-seed6789', 'bert-inference-w-identity-2', 'bert-pretrained-models', 'gpt2-models']\n"
     ]
    }
   ],
   "source": [
    "# This Python 3 environment comes with many helpful analytics libraries installed\n",
    "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n",
    "# For example, here's several helpful packages to load in \n",
    "\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "import fastai\n",
    "from fastai.train import Learner\n",
    "from fastai.train import DataBunch\n",
    "from fastai.callbacks import *\n",
    "from fastai.basic_data import DatasetType\n",
    "import fastprogress\n",
    "from fastprogress import force_console_behavior\n",
    "import numpy as np\n",
    "from pprint import pprint\n",
    "import pandas as pd\n",
    "import os\n",
    "import time\n",
    "\n",
    "import gc\n",
    "import random\n",
    "from tqdm._tqdm_notebook import tqdm_notebook as tqdm\n",
    "from keras.preprocessing import text, sequence\n",
    "import torch\n",
    "from torch import nn\n",
    "from torch.utils import data\n",
    "from torch.nn import functional as F\n",
    "\n",
    "import os\n",
    "print(os.listdir(\"../input/\"))\n",
    "\n",
    "from nltk.tokenize.treebank import TreebankWordTokenizer\n",
    "\n",
    "tqdm.pandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['pickled-crawl300d2m-for-kernel-competitions', 'pickled-glove840b300d-for-10sec-loading']\n"
     ]
    }
   ],
   "source": [
    "print(os.listdir(\"../input/pickleds/\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_lines(example, max_seq_length,tokenizer):\n",
    "    max_seq_length -=2\n",
    "    all_tokens = []\n",
    "    longer = 0\n",
    "    for text in tqdm(example):\n",
    "        tokens_a = tokenizer.tokenize(text)\n",
    "        if len(tokens_a)>max_seq_length:\n",
    "            tokens_a = tokens_a[:max_seq_length]\n",
    "            longer += 1\n",
    "        one_token = tokenizer.convert_tokens_to_ids([\"[CLS]\"]+tokens_a+[\"[SEP]\"])+[0] * (max_seq_length - len(tokens_a))\n",
    "        all_tokens.append(one_token)\n",
    "    return np.array(all_tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Any results you write to the current directory are saved as output.\n",
    "modelos_path = '../input/modelos-entrenados-toxic/'\n",
    "\n",
    "def seed_everything(seed=123):\n",
    "    random.seed(seed)\n",
    "    os.environ['PYTHONHASHSEED'] = str(seed)\n",
    "    np.random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    torch.cuda.manual_seed(seed)\n",
    "    torch.backends.cudnn.deterministic = True\n",
    "seed_everything()\n",
    "\n",
    "#CRAWL_EMBEDDING_PATH = '../input/pickled-crawl300d2m-for-kernel-competitions/crawl-300d-2M.pkl'\n",
    "#GLOVE_EMBEDDING_PATH = '../input/pickled-glove840b300d-for-10sec-loading/glove.840B.300d.pkl'\n",
    "CRAWL_EMBEDDING_PATH = '../input/pickleds/pickled-crawl300d2m-for-kernel-competitions/crawl-300d-2M.pkl'\n",
    "GLOVE_EMBEDDING_PATH = '../input/pickleds/pickled-glove840b300d-for-10sec-loading/glove.840B.300d.pkl'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [],
   "source": [
    "NUM_MODELS = 2\n",
    "LSTM_UNITS = 128 #128\n",
    "DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS\n",
    "MAX_LEN = 220 #220\n",
    "batch_s = 512\n",
    "EPOCHS = 5\n",
    "learning_rate = 0.001\n",
    "lambda_model = 0.6\n",
    "spatial_dropout = 0.3 #0.3 ///0.01, 0.5 baja\n",
    "EMBED_SIZE = 600\n",
    "\n",
    "symbols_to_isolate = '.,?!-;*\"…:—()%#$&_/@＼・ω+=”“[]^–>\\\\°<~•≠™ˈʊɒ∞§{}·τα❤☺ɡ|¢→̶`❥━┣┫┗Ｏ►★©―ɪ✔®\\x96\\x92●£♥➤´¹☕≈÷♡◐║▬′ɔː€۩۞†μ✒➥═☆ˌ◄½ʻπδηλσερνʃ✬ＳＵＰＥＲＩＴ☻±♍µº¾✓◾؟．⬅℅»Вав❣⋅¿¬♫ＣＭβ█▓▒░⇒⭐›¡₂₃❧▰▔◞▀▂▃▄▅▆▇↙γ̄″☹➡«φ⅓„✋：¥̲̅́∙‛◇✏▷❓❗¶˚˙）сиʿ✨。ɑ\\x80◕！％¯−ﬂﬁ₁²ʌ¼⁴⁄₄⌠♭✘╪▶☭✭♪☔☠♂☃☎✈✌✰❆☙○‣⚓年∎ℒ▪▙☏⅛ｃａｓǀ℮¸ｗ‚∼‖ℳ❄←☼⋆ʒ⊂、⅔¨͡๏⚾⚽Φ×θ￦？（℃⏩☮⚠月✊❌⭕▸■⇌☐☑⚡☄ǫ╭∩╮，例＞ʕɐ̣Δ₀✞┈╱╲▏▕┃╰▊▋╯┳┊≥☒↑☝ɹ✅☛♩☞ＡＪＢ◔◡↓♀⬆̱ℏ\\x91⠀ˤ╚↺⇤∏✾◦♬³の｜／∵∴√Ω¤☜▲↳▫‿⬇✧ｏｖｍ－２０８＇‰≤∕ˆ⚜☁'\n",
    "symbols_to_delete = '\\n🍕\\r🐵😑\\xa0\\ue014\\t\\uf818\\uf04a\\xad😢🐶️\\uf0e0😜😎👊\\u200b\\u200e😁عدويهصقأناخلىبمغر😍💖💵Е👎😀😂\\u202a\\u202c🔥😄🏻💥ᴍʏʀᴇɴᴅᴏᴀᴋʜᴜʟᴛᴄᴘʙғᴊᴡɢ😋👏שלוםבי😱‼\\x81エンジ故障\\u2009🚌ᴵ͞🌟😊😳😧🙀😐😕\\u200f👍😮😃😘אעכח💩💯⛽🚄🏼ஜ😖ᴠ🚲‐😟😈💪🙏🎯🌹😇💔😡\\x7f👌ἐὶήιὲκἀίῃἴξ🙄Ｈ😠\\ufeff\\u2028😉😤⛺🙂\\u3000تحكسة👮💙فزط😏🍾🎉😞\\u2008🏾😅😭👻😥😔😓🏽🎆🍻🍽🎶🌺🤔😪\\x08‑🐰🐇🐱🙆😨🙃💕𝘊𝘦𝘳𝘢𝘵𝘰𝘤𝘺𝘴𝘪𝘧𝘮𝘣💗💚地獄谷улкнПоАН🐾🐕😆ה🔗🚽歌舞伎🙈😴🏿🤗🇺🇸мυтѕ⤵🏆🎃😩\\u200a🌠🐟💫💰💎эпрд\\x95🖐🙅⛲🍰🤐👆🙌\\u2002💛🙁👀🙊🙉\\u2004ˢᵒʳʸᴼᴷᴺʷᵗʰᵉᵘ\\x13🚬🤓\\ue602😵άοόςέὸתמדףנרךצט😒͝🆕👅👥👄🔄🔤👉👤👶👲🔛🎓\\uf0b7\\uf04c\\x9f\\x10成都😣⏺😌🤑🌏😯ех😲Ἰᾶὁ💞🚓🔔📚🏀👐\\u202d💤🍇\\ue613小土豆🏡❔⁉\\u202f👠》कर्मा🇹🇼🌸蔡英文🌞🎲レクサス😛外国人关系Сб💋💀🎄💜🤢َِьыгя不是\\x9c\\x9d🗑\\u2005💃📣👿༼つ༽😰ḷЗз▱ц￼🤣卖温哥华议会下降你失去所有的钱加拿大坏税骗子🐝ツ🎅\\x85🍺آإشء🎵🌎͟ἔ油别克🤡🤥😬🤧й\\u2003🚀🤴ʲшчИОРФДЯМюж😝🖑ὐύύ特殊作戦群щ💨圆明园קℐ🏈😺🌍⏏ệ🍔🐮🍁🍆🍑🌮🌯🤦\\u200d𝓒𝓲𝓿𝓵안영하세요ЖљКћ🍀😫🤤ῦ我出生在了可以说普通话汉语好极🎼🕺🍸🥂🗽🎇🎊🆘🤠👩🖒🚪天一家⚲\\u2006⚭⚆⬭⬯⏖新✀╌🇫🇷🇩🇪🇮🇬🇧😷🇨🇦ХШ🌐\\x1f杀鸡给猴看ʁ𝗪𝗵𝗲𝗻𝘆𝗼𝘂𝗿𝗮𝗹𝗶𝘇𝗯𝘁𝗰𝘀𝘅𝗽𝘄𝗱📺ϖ\\u2000үսᴦᎥһͺ\\u2007հ\\u2001ɩｙｅ൦ｌƽｈ𝐓𝐡𝐞𝐫𝐮𝐝𝐚𝐃𝐜𝐩𝐭𝐢𝐨𝐧Ƅᴨןᑯ໐ΤᏧ௦Іᴑ܁𝐬𝐰𝐲𝐛𝐦𝐯𝐑𝐙𝐣𝐇𝐂𝐘𝟎ԜТᗞ౦〔Ꭻ𝐳𝐔𝐱𝟔𝟓𝐅🐋ﬃ💘💓ё𝘥𝘯𝘶💐🌋🌄🌅𝙬𝙖𝙨𝙤𝙣𝙡𝙮𝙘𝙠𝙚𝙙𝙜𝙧𝙥𝙩𝙪𝙗𝙞𝙝𝙛👺🐷ℋ𝐀𝐥𝐪🚶𝙢Ἱ🤘ͦ💸ج패티Ｗ𝙇ᵻ👂👃ɜ🎫\\uf0a7БУі🚢🚂ગુજરાતીῆ🏃𝓬𝓻𝓴𝓮𝓽𝓼☘﴾̯﴿₽\\ue807𝑻𝒆𝒍𝒕𝒉𝒓𝒖𝒂𝒏𝒅𝒔𝒎𝒗𝒊👽😙\\u200cЛ‒🎾👹⎌🏒⛸公寓养宠物吗🏄🐀🚑🤷操美𝒑𝒚𝒐𝑴🤙🐒欢迎来到阿拉斯ספ𝙫🐈𝒌𝙊𝙭𝙆𝙋𝙍𝘼𝙅ﷻ🦄巨收赢得白鬼愤怒要买额ẽ🚗🐳𝟏𝐟𝟖𝟑𝟕𝒄𝟗𝐠𝙄𝙃👇锟斤拷𝗢𝟳𝟱𝟬⦁マルハニチロ株式社⛷한국어ㄸㅓ니͜ʖ𝘿𝙔₵𝒩ℯ𝒾𝓁𝒶𝓉𝓇𝓊𝓃𝓈𝓅ℴ𝒻𝒽𝓀𝓌𝒸𝓎𝙏ζ𝙟𝘃𝗺𝟮𝟭𝟯𝟲👋🦊多伦🐽🎻🎹⛓🏹🍷🦆为和中友谊祝贺与其想象对法如直接问用自己猜本传教士没积唯认识基督徒曾经让相信耶稣复活死怪他但当们聊些政治题时候战胜因圣把全堂结婚孩恐惧且栗谓这样还♾🎸🤕🤒⛑🎁批判检讨🏝🦁🙋😶쥐스탱트뤼도석유가격인상이경제황을렵게만들지않록잘관리해야합다캐나에서대마초와화약금의품런성분갈때는반드시허된사용🔫👁凸ὰ💲🗯𝙈Ἄ𝒇𝒈𝒘𝒃𝑬𝑶𝕾𝖙𝖗𝖆𝖎𝖌𝖍𝖕𝖊𝖔𝖑𝖉𝖓𝖐𝖜𝖞𝖚𝖇𝕿𝖘𝖄𝖛𝖒𝖋𝖂𝕴𝖟𝖈𝕸👑🚿💡知彼百\\uf005𝙀𝒛𝑲𝑳𝑾𝒋𝟒😦𝙒𝘾𝘽🏐𝘩𝘨ὼṑ𝑱𝑹𝑫𝑵𝑪🇰🇵👾ᓇᒧᔭᐃᐧᐦᑳᐨᓃᓂᑲᐸᑭᑎᓀᐣ🐄🎈🔨🐎🤞🐸💟🎰🌝🛳点击查版🍭𝑥𝑦𝑧ＮＧ👣\\uf020っ🏉ф💭🎥Ξ🐴👨🤳🦍\\x0b🍩𝑯𝒒😗𝟐🏂👳🍗🕉🐲چی𝑮𝗕𝗴🍒ꜥⲣⲏ🐑⏰鉄リ事件ї💊「」\\uf203\\uf09a\\uf222\\ue608\\uf202\\uf099\\uf469\\ue607\\uf410\\ue600燻製シ虚偽屁理屈Г𝑩𝑰𝒀𝑺🌤𝗳𝗜𝗙𝗦𝗧🍊ὺἈἡχῖΛ⤏🇳𝒙ψՁմեռայինրւդձ冬至ὀ𝒁🔹🤚🍎𝑷🐂💅𝘬𝘱𝘸𝘷𝘐𝘭𝘓𝘖𝘹𝘲𝘫کΒώ💢ΜΟΝΑΕ🇱♲𝝈↴💒⊘Ȼ🚴🖕🖤🥘📍👈➕🚫🎨🌑🐻𝐎𝐍𝐊𝑭🤖🎎😼🕷ｇｒｎｔｉｄｕｆｂｋ𝟰🇴🇭🇻🇲𝗞𝗭𝗘𝗤👼📉🍟🍦🌈🔭《🐊🐍\\uf10aლڡ🐦\\U0001f92f\\U0001f92a🐡💳ἱ🙇𝗸𝗟𝗠𝗷🥜さようなら🔼'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_coefs(word, *arr):\n",
    "    return word, np.asarray(arr, dtype='float32')\n",
    "\n",
    "\n",
    "def load_embeddings(path):\n",
    "    with open(path,'rb') as f:\n",
    "        emb_arr = pickle.load(f)\n",
    "    return emb_arr\n",
    "    \n",
    "def build_matrix(word_index, path):\n",
    "    embedding_index = load_embeddings(path)\n",
    "    embedding_matrix = np.zeros((max_features + 1, 300))\n",
    "    unknown_words = []\n",
    "    \n",
    "    for word, i in word_index.items():\n",
    "        if i <= max_features:\n",
    "            try:\n",
    "                embedding_matrix[i] = embedding_index[word]\n",
    "            except KeyError:\n",
    "                try:\n",
    "                    embedding_matrix[i] = embedding_index[word.lower()]\n",
    "                except KeyError:\n",
    "                    try:\n",
    "                        embedding_matrix[i] = embedding_index[word.title()]\n",
    "                    except KeyError:\n",
    "                        unknown_words.append(word)\n",
    "    return embedding_matrix, unknown_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class EMB(nn.Module):\n",
    "    def __init__(self, embedding_matrix):\n",
    "        super(EMB, self).__init__()\n",
    "        embed_size = embedding_matrix.shape[1]\n",
    "        \n",
    "        self.embedding = nn.Embedding(max_features, embed_size)\n",
    "        self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))\n",
    "        self.embedding.weight.requires_grad = False\n",
    "        self.embedding_dropout = SpatialDropout(spatial_dropout)\n",
    "        \n",
    "    def forward(self, x, lengths=None):\n",
    "        h_embedding = self.embedding(x.long())\n",
    "        h_embedding = self.embedding_dropout(h_embedding)\n",
    "        \n",
    "        return out\n",
    "    \n",
    "class RoM(nn.Module):\n",
    "    def __init__(self, num_aux_targets):\n",
    "        super(RoM, self).__init__()\n",
    "        embed_size = EMBED_SIZE\n",
    "        \n",
    "        self.lstm1 = nn.LSTM(embed_size, LSTM_UNITS, bidirectional=True, batch_first=True)\n",
    "        self.lstm2 = nn.LSTM(LSTM_UNITS * 2, LSTM_UNITS, bidirectional=True, batch_first=True)\n",
    "    \n",
    "        self.linear1 = nn.Linear(DENSE_HIDDEN_UNITS, DENSE_HIDDEN_UNITS)\n",
    "        self.linear2 = nn.Linear(DENSE_HIDDEN_UNITS, DENSE_HIDDEN_UNITS)\n",
    "        \n",
    "        self.linear_out = nn.Linear(DENSE_HIDDEN_UNITS, 1)\n",
    "        self.linear_aux_out = nn.Linear(DENSE_HIDDEN_UNITS, num_aux_targets)\n",
    "        \n",
    "    def forward(self, x, lengths=None):\n",
    "        \n",
    "        h_lstm1, _ = self.lstm1(x)\n",
    "        h_lstm2, _ = self.lstm2(h_lstm1)\n",
    "        \n",
    "        # global average pooling\n",
    "        avg_pool = torch.mean(h_lstm2, 1)\n",
    "        # global max pooling\n",
    "        max_pool, _ = torch.max(h_lstm2, 1)\n",
    "        \n",
    "        h_conc = torch.cat((max_pool, avg_pool), 1)\n",
    "        h_conc_linear1  = F.relu(self.linear1(h_conc))\n",
    "        h_conc_linear2  = F.relu(self.linear2(h_conc))\n",
    "        \n",
    "        hidden = h_conc + h_conc_linear1 + h_conc_linear2\n",
    "        \n",
    "        result = self.linear_out(hidden)\n",
    "        aux_result = self.linear_aux_out(hidden)\n",
    "        out = torch.cat([result, aux_result], 1)\n",
    "        \n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sigmoid(x):\n",
    "    return 1 / (1 + np.exp(-x))\n",
    "\n",
    "class SpatialDropout(nn.Dropout2d):\n",
    "    def forward(self, x):\n",
    "        x = x.unsqueeze(2)    # (N, T, 1, K)\n",
    "        x = x.permute(0, 3, 2, 1)  # (N, K, 1, T)\n",
    "        x = super(SpatialDropout, self).forward(x)  # (N, K, 1, T), some features are masked\n",
    "        x = x.permute(0, 3, 2, 1)  # (N, T, 1, K)\n",
    "        x = x.squeeze(2)  # (N, T, K)\n",
    "        return x\n",
    "    \n",
    "class NeuralNet(nn.Module):\n",
    "    def __init__(self, embedding_matrix, num_aux_targets):\n",
    "        super(NeuralNet, self).__init__()\n",
    "        embed_size = embedding_matrix.shape[1]\n",
    "        \n",
    "        self.embedding = nn.Embedding(max_features, embed_size)\n",
    "        self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))\n",
    "        self.embedding.weight.requires_grad = False\n",
    "        self.embedding_dropout = SpatialDropout(spatial_dropout)\n",
    "        \n",
    "        self.lstm1 = nn.LSTM(embed_size, LSTM_UNITS, bidirectional=True, batch_first=True)\n",
    "        self.lstm2 = nn.LSTM(LSTM_UNITS * 2, LSTM_UNITS, bidirectional=True, batch_first=True)\n",
    "    \n",
    "        self.linear1 = nn.Linear(DENSE_HIDDEN_UNITS, DENSE_HIDDEN_UNITS)\n",
    "        self.linear2 = nn.Linear(DENSE_HIDDEN_UNITS, DENSE_HIDDEN_UNITS)\n",
    "        \n",
    "        self.linear_out = nn.Linear(DENSE_HIDDEN_UNITS, 1)\n",
    "        self.linear_aux_out = nn.Linear(DENSE_HIDDEN_UNITS, num_aux_targets)\n",
    "        \n",
    "    def forward(self, x, lengths=None):\n",
    "        h_embedding = self.embedding(x.long())\n",
    "        h_embedding = self.embedding_dropout(h_embedding)\n",
    "        \n",
    "        h_lstm1, _ = self.lstm1(h_embedding)\n",
    "        h_lstm2, _ = self.lstm2(h_lstm1)\n",
    "        \n",
    "        # global average pooling\n",
    "        avg_pool = torch.mean(h_lstm2, 1)\n",
    "        # global max pooling\n",
    "        max_pool, _ = torch.max(h_lstm2, 1)\n",
    "        \n",
    "        h_conc = torch.cat((max_pool, avg_pool), 1)\n",
    "        h_conc_linear1  = F.relu(self.linear1(h_conc))\n",
    "        h_conc_linear2  = F.relu(self.linear2(h_conc))\n",
    "        \n",
    "        hidden = h_conc + h_conc_linear1 + h_conc_linear2\n",
    "        \n",
    "        result = self.linear_out(hidden)\n",
    "        aux_result = self.linear_aux_out(hidden)\n",
    "        out = torch.cat([result, aux_result], 1)\n",
    "        \n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = TreebankWordTokenizer()\n",
    "\n",
    "\n",
    "isolate_dict = {ord(c):f' {c} ' for c in symbols_to_isolate}\n",
    "remove_dict = {ord(c):f'' for c in symbols_to_delete}\n",
    "\n",
    "\n",
    "def handle_punctuation(x):\n",
    "    x = x.translate(remove_dict)\n",
    "    x = x.translate(isolate_dict)\n",
    "    return x\n",
    "\n",
    "def handle_contractions(x):\n",
    "    x = tokenizer.tokenize(x)\n",
    "    return x\n",
    "\n",
    "def fix_quote(x):\n",
    "    x = [x_[1:] if x_.startswith(\"'\") else x_ for x_ in x]\n",
    "    x = ' '.join(x)\n",
    "    return x\n",
    "\n",
    "def preprocess(x):\n",
    "    x = handle_punctuation(x)#movido a abajo\n",
    "    x = handle_contractions(x)#movido a arriba \n",
    "    x = fix_quote(x)\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_df = pd.read_csv('../input/jigsaw-unintended-bias-in-toxicity-classification/test.csv')\n",
    "#test_df = test_df.loc[:100, :] #++\n",
    "IDS_TEST = test_df['id']\n",
    "\n",
    "test_df['comment_text'] = test_df['comment_text'].astype(str) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "REDES\n",
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "027c96a2d5204349816fc6dab856d032",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=97320), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "n unknown words (crawl):  14055\n",
      "n unknown words (glove):  14375\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lstm_test = test_df['comment_text'].progress_apply(lambda x:preprocess(x))\n",
    "\n",
    "max_features = 327576\n",
    "\n",
    "#################################################################################################### \n",
    "tokenizer = text.Tokenizer(num_words = max_features, filters='',lower=False)\n",
    "#3. In the inference kernel build the embedding matrix with only the words of the test set.\n",
    "tokenizer.fit_on_texts(list(lstm_test)) \n",
    "\n",
    "crawl_matrix, unknown_words_crawl = build_matrix(tokenizer.word_index, CRAWL_EMBEDDING_PATH)\n",
    "print('n unknown words (crawl): ', len(unknown_words_crawl))\n",
    "\n",
    "glove_matrix, unknown_words_glove = build_matrix(tokenizer.word_index, GLOVE_EMBEDDING_PATH)\n",
    "print('n unknown words (glove): ', len(unknown_words_glove))\n",
    "\n",
    "max_features = max_features or len(tokenizer.word_index) + 1\n",
    "max_features\n",
    "\n",
    "embedding_matrix = np.concatenate([crawl_matrix, glove_matrix], axis=-1)\n",
    "embedding_matrix.shape\n",
    "\n",
    "del crawl_matrix\n",
    "del glove_matrix\n",
    "gc.collect()\n",
    "\n",
    "batch_size = batch_s\n",
    "x_test = tokenizer.texts_to_sequences(lstm_test)\n",
    "\n",
    "#maxlen = lengths.max() \n",
    "maxlen = 300\n",
    "test_lengths = torch.from_numpy(np.array([len(x) for x in x_test]))\n",
    "x_test_padded = torch.from_numpy(sequence.pad_sequences(x_test, maxlen=maxlen))\n",
    "\n",
    "\n",
    "test_dataset = data.TensorDataset(x_test_padded, test_lengths)\n",
    "#test_collator = SequenceBucketCollator(lambda lenghts: lenghts.max(), sequence_index=0, length_index=1)\n",
    "#test_loader = data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False, collate_fn=test_collator)\n",
    "\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Modelo: model_3.pth\n",
      "Modelo preparado y cargado.\n",
      "Tiempo prediccion: 15.089172840118408\n",
      "Modelo: model_5.pth\n",
      "Modelo preparado y cargado.\n",
      "Tiempo prediccion: 15.061607122421265\n",
      "Modelo: model_4.pth\n",
      "Modelo preparado y cargado.\n",
      "Tiempo prediccion: 15.047003269195557\n",
      "Modelo: model_2.pth\n",
      "Modelo preparado y cargado.\n",
      "Tiempo prediccion: 15.082767248153687\n",
      "Modelo: model_1.pth\n",
      "Modelo preparado y cargado.\n",
      "Tiempo prediccion: 15.044901371002197\n",
      "Modelo: model_0.pth\n",
      "Modelo preparado y cargado.\n",
      "Tiempo prediccion: 15.060503482818604\n"
     ]
    }
   ],
   "source": [
    "test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False)\n",
    "#TODO cargar modelos y calcular medias de las predicciones\n",
    "# meter como primera capa el embedding weigth\n",
    "all_test_preds = []\n",
    "\n",
    "cont = 0\n",
    "for filename in os.listdir(modelos_path):\n",
    "    \n",
    "    if cont > 6:\n",
    "        break\n",
    "    \n",
    "    print('Modelo: '+filename)\n",
    "    #creamos/cargamos los modelos separados\n",
    "    embd_model = EMB(embedding_matrix).cuda()\n",
    "    ron_model = RoM(6).cuda()\n",
    "    ron_model.load_state_dict(torch.load(modelos_path + filename))\n",
    "\n",
    "    # y juntamos\n",
    "    state_dict={}\n",
    "    for key, value in embd_model.state_dict().items():\n",
    "        state_dict[key] = value\n",
    "    \n",
    "    for key, value in ron_model.state_dict().items():\n",
    "        state_dict[key] = value\n",
    "    \n",
    "    del embd_model, ron_model\n",
    "    \n",
    "    model = NeuralNet(embedding_matrix, 6).cuda()\n",
    "    model.load_state_dict(state_dict)\n",
    "    \n",
    "    #++----------\n",
    "    for param in model.parameters():\n",
    "        param.requires_grad = False\n",
    "    model.eval() #++\n",
    "    #++----------\n",
    "    \n",
    "    print('Modelo preparado y cargado.')\n",
    "    test_preds = np.zeros((len(test_dataset), 7))\n",
    "    start = time.time()\n",
    "    for i, x_batch in enumerate(test_loader):\n",
    "            X = x_batch[0].cuda()\n",
    "            y_pred = sigmoid(model(X).detach().cpu().numpy())\n",
    "            test_preds[i * batch_size:(i+1) * batch_size, :] = y_pred\n",
    "\n",
    "    end = time.time()\n",
    "    print('Tiempo prediccion:', end-start)\n",
    "    all_test_preds.append(test_preds)\n",
    "    cont+=1\n",
    "    \n",
    "    gc.collect()\n",
    "    \n",
    "submission_lstm = pd.DataFrame.from_dict({\n",
    "    'id': IDS_TEST,\n",
    "    'prediction': np.mean(all_test_preds, axis=0)[:, 0]\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "warnings.filterwarnings(action='once')\n",
    "device = torch.device('cuda')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "GPT2\n",
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n! pip install ../input/gpt2-pytorch/gpt2-pytorch/pytorch-pretrained-BERT-master\\npackage_dir = \"../input/gpt2-pytorch/gpt2-pytorch/pytorch-pretrained-BERT-master\"\\nsys.path.append(package_dir)\\n\\nfrom pytorch_pretrained_bert import GPT2Config\\nfrom pytorch_pretrained_bert import GPT2Tokenizer\\nfrom pytorch_pretrained_bert import GPT2ClassificationHeadModel\\n'"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "! pip install ../input/gpt2-pytorch/gpt2-pytorch/pytorch-pretrained-BERT-master\n",
    "package_dir = \"../input/gpt2-pytorch/gpt2-pytorch/pytorch-pretrained-BERT-master\"\n",
    "sys.path.append(package_dir)\n",
    "\n",
    "from pytorch_pretrained_bert import GPT2Config\n",
    "from pytorch_pretrained_bert import GPT2Tokenizer\n",
    "from pytorch_pretrained_bert import GPT2ClassificationHeadModel\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_lines2(example, max_seq_length,tokenizer):\n",
    "    max_seq_length -=2\n",
    "    all_tokens = []\n",
    "    longer = 0\n",
    "    for text in tqdm(example):\n",
    "        tokens_a = tokenizer.tokenize(text)\n",
    "        if len(tokens_a)>max_seq_length:\n",
    "            tokens_a = tokens_a[:max_seq_length]\n",
    "            longer += 1\n",
    "        one_token = tokenizer.convert_tokens_to_ids([\"[CLS]\"]+tokens_a+[\"[SEP]\"])+[0] * (max_seq_length - len(tokens_a))\n",
    "        all_tokens.append(one_token)\n",
    "    return np.array(all_tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n%%time\\nMAX_SEQUENCE_LENGTH = 220\\nSEED = 1234\\nBATCH_SIZE_gpt2 = 16\\nBERT_MODEL_PATH = \\'../input/bert-pretrained-models/uncased_l-12_h-768_a-12/uncased_L-12_H-768_A-12/\\' #base ->10xx es large\\n\\nnp.random.seed(SEED)\\ntorch.manual_seed(SEED)\\ntorch.cuda.manual_seed(SEED)\\ntorch.backends.cudnn.deterministic = True\\n\\ngpt2_config = GPT2Config(\\'../input/gpt2-inferece-w-identity/results/\\'+\\'config.json\\')\\ntokenizer = GPT2Tokenizer.from_pretrained(\\'../input/gpt2-models/\\')#++\\n\\nX_test_gpt2 = convert_lines2(test_df[\"comment_text\"].fillna(\"DUMMY_VALUE\"), MAX_SEQUENCE_LENGTH, tokenizer)\\n\\nmodel = GPT2ClassificationHeadModel(gpt2_config,n_class=7)\\nmodel.load_state_dict(torch.load(\"../input/gpt2-inferece-w-identity/results/gpt2_pytorch.bin\"))\\nmodel.to(device)\\nfor param in model.parameters():\\n    param.requires_grad = False\\nmodel.eval()\\n\\ntest_preds_gpt2 = np.zeros((len(X_test_gpt2)))\\ntest_gpt2 = torch.utils.data.TensorDataset(torch.tensor(X_test_gpt2, dtype=torch.long))\\ntest_loader_gpt2 = torch.utils.data.DataLoader(test_gpt2, batch_size=BATCH_SIZE_gpt2, shuffle=False)\\ntk0 = tqdm(test_loader_gpt2)\\nfor i, (x_batch,) in enumerate(tk0):\\n    pred = model(x_batch.to(device))\\n    test_preds_gpt2[i * BATCH_SIZE_gpt2:(i + 1) * BATCH_SIZE_gpt2] = pred[:, 0].detach().cpu().squeeze().numpy()\\n\\ntest_pred_gpt2 = torch.sigmoid(torch.tensor(test_preds_gpt2)).numpy().ravel()\\n\\nsubmission_gpt2 = pd.DataFrame.from_dict({\\n    \\'id\\': test_df[\\'id\\'],\\n    \\'prediction\\': test_pred_gpt2\\n})\\n'"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "%%time\n",
    "MAX_SEQUENCE_LENGTH = 220\n",
    "SEED = 1234\n",
    "BATCH_SIZE_gpt2 = 16\n",
    "BERT_MODEL_PATH = '../input/bert-pretrained-models/uncased_l-12_h-768_a-12/uncased_L-12_H-768_A-12/' #base ->10xx es large\n",
    "\n",
    "np.random.seed(SEED)\n",
    "torch.manual_seed(SEED)\n",
    "torch.cuda.manual_seed(SEED)\n",
    "torch.backends.cudnn.deterministic = True\n",
    "\n",
    "gpt2_config = GPT2Config('../input/gpt2-inferece-w-identity/results/'+'config.json')\n",
    "tokenizer = GPT2Tokenizer.from_pretrained('../input/gpt2-models/')#++\n",
    "\n",
    "X_test_gpt2 = convert_lines2(test_df[\"comment_text\"].fillna(\"DUMMY_VALUE\"), MAX_SEQUENCE_LENGTH, tokenizer)\n",
    "\n",
    "model = GPT2ClassificationHeadModel(gpt2_config,n_class=7)\n",
    "model.load_state_dict(torch.load(\"../input/gpt2-inferece-w-identity/results/gpt2_pytorch.bin\"))\n",
    "model.to(device)\n",
    "for param in model.parameters():\n",
    "    param.requires_grad = False\n",
    "model.eval()\n",
    "\n",
    "test_preds_gpt2 = np.zeros((len(X_test_gpt2)))\n",
    "test_gpt2 = torch.utils.data.TensorDataset(torch.tensor(X_test_gpt2, dtype=torch.long))\n",
    "test_loader_gpt2 = torch.utils.data.DataLoader(test_gpt2, batch_size=BATCH_SIZE_gpt2, shuffle=False)\n",
    "tk0 = tqdm(test_loader_gpt2)\n",
    "for i, (x_batch,) in enumerate(tk0):\n",
    "    pred = model(x_batch.to(device))\n",
    "    test_preds_gpt2[i * BATCH_SIZE_gpt2:(i + 1) * BATCH_SIZE_gpt2] = pred[:, 0].detach().cpu().squeeze().numpy()\n",
    "\n",
    "test_pred_gpt2 = torch.sigmoid(torch.tensor(test_preds_gpt2)).numpy().ravel()\n",
    "\n",
    "submission_gpt2 = pd.DataFrame.from_dict({\n",
    "    'id': test_df['id'],\n",
    "    'prediction': test_pred_gpt2\n",
    "})\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "BERT\n",
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing /kaggle/input/ppbert/pytorch-pretrained-bert/pytorch-pretrained-BERT\r\n",
      "Requirement already satisfied: torch>=0.4.1 in /opt/conda/lib/python3.6/site-packages (from pytorch-pretrained-bert==0.6.1) (1.1.0)\r\n",
      "Requirement already satisfied: numpy in /opt/conda/lib/python3.6/site-packages (from pytorch-pretrained-bert==0.6.1) (1.16.4)\r\n",
      "Requirement already satisfied: boto3 in /opt/conda/lib/python3.6/site-packages (from pytorch-pretrained-bert==0.6.1) (1.9.173)\r\n",
      "Requirement already satisfied: requests in /opt/conda/lib/python3.6/site-packages (from pytorch-pretrained-bert==0.6.1) (2.22.0)\r\n",
      "Requirement already satisfied: tqdm in /opt/conda/lib/python3.6/site-packages (from pytorch-pretrained-bert==0.6.1) (4.32.1)\r\n",
      "Requirement already satisfied: regex in /opt/conda/lib/python3.6/site-packages (from pytorch-pretrained-bert==0.6.1) (2019.6.8)\r\n",
      "Requirement already satisfied: botocore<1.13.0,>=1.12.173 in /opt/conda/lib/python3.6/site-packages (from boto3->pytorch-pretrained-bert==0.6.1) (1.12.173)\r\n",
      "Requirement already satisfied: s3transfer<0.3.0,>=0.2.0 in /opt/conda/lib/python3.6/site-packages (from boto3->pytorch-pretrained-bert==0.6.1) (0.2.1)\r\n",
      "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /opt/conda/lib/python3.6/site-packages (from boto3->pytorch-pretrained-bert==0.6.1) (0.9.4)\r\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /opt/conda/lib/python3.6/site-packages (from requests->pytorch-pretrained-bert==0.6.1) (1.24.2)\r\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.6/site-packages (from requests->pytorch-pretrained-bert==0.6.1) (2019.6.16)\r\n",
      "Requirement already satisfied: idna<2.9,>=2.5 in /opt/conda/lib/python3.6/site-packages (from requests->pytorch-pretrained-bert==0.6.1) (2.8)\r\n",
      "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /opt/conda/lib/python3.6/site-packages (from requests->pytorch-pretrained-bert==0.6.1) (3.0.4)\r\n",
      "Requirement already satisfied: docutils>=0.10 in /opt/conda/lib/python3.6/site-packages (from botocore<1.13.0,>=1.12.173->boto3->pytorch-pretrained-bert==0.6.1) (0.14)\r\n",
      "Requirement already satisfied: python-dateutil<3.0.0,>=2.1; python_version >= \"2.7\" in /opt/conda/lib/python3.6/site-packages (from botocore<1.13.0,>=1.12.173->boto3->pytorch-pretrained-bert==0.6.1) (2.8.0)\r\n",
      "Requirement already satisfied: six>=1.5 in /opt/conda/lib/python3.6/site-packages (from python-dateutil<3.0.0,>=2.1; python_version >= \"2.7\"->botocore<1.13.0,>=1.12.173->boto3->pytorch-pretrained-bert==0.6.1) (1.12.0)\r\n",
      "Building wheels for collected packages: pytorch-pretrained-bert\r\n",
      "  Building wheel for pytorch-pretrained-bert (setup.py) ... \u001b[?25l-\b \b\\\b \bdone\r\n",
      "\u001b[?25h  Stored in directory: /tmp/.cache/pip/wheels/bf/86/9a/36f2f25ad22af9ff95c8b53880174e7ed7422255a0054a7a8e\r\n",
      "Successfully built pytorch-pretrained-bert\r\n",
      "\u001b[31mERROR: allennlp 0.8.4 requires awscli>=1.11.91, which is not installed.\u001b[0m\r\n",
      "\u001b[31mERROR: allennlp 0.8.4 requires flaky, which is not installed.\u001b[0m\r\n",
      "\u001b[31mERROR: allennlp 0.8.4 requires responses>=0.7, which is not installed.\u001b[0m\r\n",
      "Installing collected packages: pytorch-pretrained-bert\r\n",
      "  Found existing installation: pytorch-pretrained-bert 0.6.2\r\n",
      "    Uninstalling pytorch-pretrained-bert-0.6.2:\r\n",
      "      Successfully uninstalled pytorch-pretrained-bert-0.6.2\r\n",
      "Successfully installed pytorch-pretrained-bert-0.6.1\r\n"
     ]
    }
   ],
   "source": [
    "#BERT\n",
    "import sys\n",
    "package_dir = \"../input/ppbert/pytorch-pretrained-bert/pytorch-pretrained-BERT\"\n",
    "sys.path.append(package_dir)\n",
    "\n",
    "! pip install ../input/ppbert/pytorch-pretrained-bert/pytorch-pretrained-BERT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pytorch_pretrained_bert import BertTokenizer, BertForSequenceClassification, BertAdam\n",
    "from pytorch_pretrained_bert import BertConfig\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['bert_pytorch.bin', 'custom.css', '__results__.html', 'pytorch_model.bin', 'bert_config.json']\n"
     ]
    }
   ],
   "source": [
    "print(os.listdir('../input/bert-inference-w-identity/results/bert-inference-w-identity'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "MAX_SEQUENCE_LENGTH = 220\n",
    "SEED = 1234\n",
    "BATCH_SIZE = 32\n",
    "BERT_MODEL_PATH = '../input/bert-pretrained-models/uncased_l-12_h-768_a-12/uncased_L-12_H-768_A-12/' #base ->10xx es large\n",
    "\n",
    "np.random.seed(SEED)\n",
    "torch.manual_seed(SEED)\n",
    "torch.cuda.manual_seed(SEED)\n",
    "torch.backends.cudnn.deterministic = True\n",
    "\n",
    "bert_config = BertConfig('../input/bert-inference-w-identity/results/bert-inference-w-identity/bert_config.json')\n",
    "bert_tokenizer = BertTokenizer.from_pretrained(BERT_MODEL_PATH, cache_dir=None,do_lower_case=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "44725da3dc744932b09cd3b7299a5e3c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=97320), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "X_test = convert_lines(test_df[\"comment_text\"].fillna(\"DUMMY_VALUE\"), MAX_SEQUENCE_LENGTH, bert_tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BertForSequenceClassification(\n",
       "  (bert): BertModel(\n",
       "    (embeddings): BertEmbeddings(\n",
       "      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
       "      (position_embeddings): Embedding(512, 768)\n",
       "      (token_type_embeddings): Embedding(2, 768)\n",
       "      (LayerNorm): BertLayerNorm()\n",
       "      (dropout): Dropout(p=0.1)\n",
       "    )\n",
       "    (encoder): BertEncoder(\n",
       "      (layer): ModuleList(\n",
       "        (0): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): BertLayerNorm()\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): BertLayerNorm()\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (1): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): BertLayerNorm()\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): BertLayerNorm()\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (2): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): BertLayerNorm()\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): BertLayerNorm()\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (3): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): BertLayerNorm()\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): BertLayerNorm()\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (4): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): BertLayerNorm()\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): BertLayerNorm()\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (5): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): BertLayerNorm()\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): BertLayerNorm()\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (6): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): BertLayerNorm()\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): BertLayerNorm()\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (7): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): BertLayerNorm()\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): BertLayerNorm()\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (8): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): BertLayerNorm()\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): BertLayerNorm()\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (9): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): BertLayerNorm()\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): BertLayerNorm()\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (10): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): BertLayerNorm()\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): BertLayerNorm()\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (11): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): BertLayerNorm()\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): BertLayerNorm()\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (pooler): BertPooler(\n",
       "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "      (activation): Tanh()\n",
       "    )\n",
       "  )\n",
       "  (dropout): Dropout(p=0.1)\n",
       "  (classifier): Linear(in_features=768, out_features=7, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = BertForSequenceClassification(bert_config, num_labels=7)\n",
    "model.load_state_dict(torch.load(\"../input/bert-inference-w-identity/results/bert-inference-w-identity/bert_pytorch.bin\"))\n",
    "model.to(device)\n",
    "for param in model.parameters():\n",
    "    param.requires_grad = False\n",
    "model.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d6d3c556d56547d183d91d3b3eae2594",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3042), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "test_preds = np.zeros((len(X_test)))\n",
    "test = torch.utils.data.TensorDataset(torch.tensor(X_test, dtype=torch.long))\n",
    "test_loader = torch.utils.data.DataLoader(test, batch_size=32, shuffle=False)\n",
    "tk0 = tqdm(test_loader)\n",
    "for i, (x_batch,) in enumerate(tk0):\n",
    "    pred = model(x_batch.to(device), attention_mask=(x_batch > 0).to(device), labels=None)\n",
    "    test_preds[i * 32:(i + 1) * 32] = pred[:, 0].detach().cpu().squeeze().numpy()\n",
    "\n",
    "bert_test_pred = torch.sigmoid(torch.tensor(test_preds)).numpy().ravel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "bert_config2 = BertConfig('../input/bert-inference-w-identity-2/results/bert_config.json')\n",
    "\n",
    "model2 = BertForSequenceClassification(bert_config2, num_labels=7)\n",
    "#model2.load_state_dict(torch.load(\"../input/bert-inference-w-identity-1/results/bert_pytorch_3.bin\"))\n",
    "model2.load_state_dict(torch.load(\"../input/bert-inference-w-identity-2/results/bert_pytorch_4.bin\"))\n",
    "model2.to(device)\n",
    "for param in model2.parameters():\n",
    "    param.requires_grad = False\n",
    "model2.eval()\n",
    "\n",
    "test_preds2 = np.zeros((len(X_test)))\n",
    "for i, (x_batch,) in enumerate(tk0):\n",
    "    pred = model2(x_batch.to(device), attention_mask=(x_batch > 0).to(device), labels=None)\n",
    "    test_preds2[i * 32:(i + 1) * 32] = pred[:, 0].detach().cpu().squeeze().numpy()\n",
    "\n",
    "test_pred2 = torch.sigmoid(torch.tensor(test_preds2)).numpy().ravel()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "bert_config3 = BertConfig('../input/bert-inference-w-identity-1/results/bert_config.json')\n",
    "\n",
    "model3 = BertForSequenceClassification(bert_config3, num_labels=7)\n",
    "model3.load_state_dict(torch.load(\"../input/bert-inference-w-identity-1/results/bert_pytorch_3.bin\"))\n",
    "#model2.load_state_dict(torch.load(\"../input/bert-inference-w-identity-2/results/bert_pytorch_4.bin\"))\n",
    "model3.to(device)\n",
    "for param in model3.parameters():\n",
    "    param.requires_grad = False\n",
    "model3.eval()\n",
    "\n",
    "test_preds3 = np.zeros((len(X_test)))\n",
    "for i, (x_batch,) in enumerate(tk0):\n",
    "    pred = model3(x_batch.to(device), attention_mask=(x_batch > 0).to(device), labels=None)\n",
    "    test_preds3[i * 32:(i + 1) * 32] = pred[:, 0].detach().cpu().squeeze().numpy()\n",
    "\n",
    "test_pred3 = torch.sigmoid(torch.tensor(test_preds3)).numpy().ravel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "bert_config4 = BertConfig('../input/bert-inference-w-identity-seed6789/results/bert_config.json')\n",
    "\n",
    "model4 = BertForSequenceClassification(bert_config4, num_labels=7)\n",
    "model4.load_state_dict(torch.load(\"../input/bert-inference-w-identity-seed6789/results/bert_pytorch.bin\"))\n",
    "#model2.load_state_dict(torch.load(\"../input/bert-inference-w-identity-2/results/bert_pytorch_4.bin\"))\n",
    "model4.to(device)\n",
    "for param in model4.parameters():\n",
    "    param.requires_grad = False\n",
    "model4.eval()\n",
    "\n",
    "test_preds4 = np.zeros((len(X_test)))\n",
    "for i, (x_batch,) in enumerate(tk0):\n",
    "    pred = model4(x_batch.to(device), attention_mask=(x_batch > 0).to(device), labels=None)\n",
    "    test_preds4[i * 32:(i + 1) * 32] = pred[:, 0].detach().cpu().squeeze().numpy()\n",
    "\n",
    "test_pred4 = torch.sigmoid(torch.tensor(test_preds4)).numpy().ravel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "bert_config5 = BertConfig('../input/bert-inference-w-identityseed5060/results/bert_config.json')\n",
    "\n",
    "model5 = BertForSequenceClassification(bert_config5, num_labels=7)\n",
    "model5.load_state_dict(torch.load(\"../input/bert-inference-w-identityseed5060/results/bert_pytorch.bin\"))\n",
    "#model2.load_state_dict(torch.load(\"../input/bert-inference-w-identity-2/results/bert_pytorch_4.bin\"))\n",
    "model5.to(device)\n",
    "for param in model5.parameters():\n",
    "    param.requires_grad = False\n",
    "model5.eval()\n",
    "\n",
    "test_preds5 = np.zeros((len(X_test)))\n",
    "for i, (x_batch,) in enumerate(tk0):\n",
    "    pred = model5(x_batch.to(device), attention_mask=(x_batch > 0).to(device), labels=None)\n",
    "    test_preds5[i * 32:(i + 1) * 32] = pred[:, 0].detach().cpu().squeeze().numpy()\n",
    "\n",
    "test_pred5 = torch.sigmoid(torch.tensor(test_preds5)).numpy().ravel()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Media Berts\n",
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "bert_test_pred = (bert_test_pred + test_pred2 + test_pred3 + test_pred4 + test_pred5) / 5\n",
    "#bert_test_pred = (test_pred2 + test_pred3 + test_pred4) / 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "submission_bert = pd.DataFrame.from_dict({\n",
    "    'id': IDS_TEST,\n",
    "    'prediction': bert_test_pred\n",
    "})\n",
    "submission_bert2 = pd.DataFrame.from_dict({\n",
    "    'id': IDS_TEST,\n",
    "    'prediction': test_pred2\n",
    "})\n",
    "submission_bert3 = pd.DataFrame.from_dict({\n",
    "    'id': IDS_TEST,\n",
    "    'prediction': test_pred3\n",
    "})\n",
    "submission_bert4 = pd.DataFrame.from_dict({\n",
    "    'id': IDS_TEST,\n",
    "    'prediction': test_pred4\n",
    "})\n",
    "submission_bert5 = pd.DataFrame.from_dict({\n",
    "    'id': IDS_TEST,\n",
    "    'prediction': test_pred5\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "416"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "JUNTAMOS TODO\n",
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "submission = pd.read_csv('../input/jigsaw-unintended-bias-in-toxicity-classification/sample_submission.csv')\n",
    "submission['id'] = IDS_TEST\n",
    "#submission['prediction'] = (submission_lstm['prediction'].apply(lambda x: x*0.4) + submission_bert['prediction'].apply(lambda x: x*0.6))\n",
    "submission['prediction'] = (submission_lstm['prediction'].apply(lambda x: x*0.22172817) + submission_bert['prediction'].apply(lambda x: x*0.10849379) +\n",
    "                           submission_bert2['prediction'].apply(lambda x: x*0.16600595) + submission_bert3['prediction'].apply(lambda x: x*0.09252766) +\n",
    "                           submission_bert4['prediction'].apply(lambda x: x*0.07704189) + submission_bert5['prediction'].apply(lambda x: x*0.11969974))\n",
    "\n",
    "submission.to_csv('submission.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "027c96a2d5204349816fc6dab856d032": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_ecd0f37608f24efd89e54deac8f288b7",
        "IPY_MODEL_16f9a5f7cd4c467597f63ad7a6cea615"
       ],
       "layout": "IPY_MODEL_d6f7cdd09c9848e6b27dddac51ff682b"
      }
     },
     "0375f83f462644fabbd8d5e986e9fb84": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "05a61cb435e8451196175d1ea9680398": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "16788c09e0874351b4985bd971024baf": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "16cda643c3bb44b2abac757d2282f868": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "16f9a5f7cd4c467597f63ad7a6cea615": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_6e40c78a0d5f461f9bde420b97e89786",
       "placeholder": "​",
       "style": "IPY_MODEL_ded2fb20fa7d423fa21b532eb1322527",
       "value": "100% 97320/97320 [00:34&lt;00:00, 2848.85it/s]"
      }
     },
     "1e3027596b6b404688ac86e3c1ea85b2": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "1fb80076a5c0430eb824af943f528d56": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "44725da3dc744932b09cd3b7299a5e3c": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_d5a93e98df674dcd8f067f40c1ef04bd",
        "IPY_MODEL_e38099bacd904a0fbe618e3f937d3fdc"
       ],
       "layout": "IPY_MODEL_16cda643c3bb44b2abac757d2282f868"
      }
     },
     "6ab41c1b2937474a85ff013d405c9973": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "6e40c78a0d5f461f9bde420b97e89786": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "7d565d60b9c349dda121fb8f7d45bb70": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "8813bd860de048ce98333ba7ef2b89c2": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "917d55fafb0242a08b038f29daac191d": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "IntProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "IntProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_0375f83f462644fabbd8d5e986e9fb84",
       "max": 3042,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_c85e1378c6dc4914b2dbe60b2bafd7d2",
       "value": 3042
      }
     },
     "a592b8d9cc9944a5b23c5a332d29d46b": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "bcda54b2cfc94816b9bd099b92bfe107": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "be581e526f0547a88a99bbec19ecfeaf": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_1fb80076a5c0430eb824af943f528d56",
       "placeholder": "​",
       "style": "IPY_MODEL_16788c09e0874351b4985bd971024baf",
       "value": "100% 3042/3042 [11:58&lt;00:00,  4.23it/s]"
      }
     },
     "c85e1378c6dc4914b2dbe60b2bafd7d2": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "d5a93e98df674dcd8f067f40c1ef04bd": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "IntProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "IntProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_6ab41c1b2937474a85ff013d405c9973",
       "max": 97320,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_7d565d60b9c349dda121fb8f7d45bb70",
       "value": 97320
      }
     },
     "d6d3c556d56547d183d91d3b3eae2594": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_917d55fafb0242a08b038f29daac191d",
        "IPY_MODEL_be581e526f0547a88a99bbec19ecfeaf"
       ],
       "layout": "IPY_MODEL_1e3027596b6b404688ac86e3c1ea85b2"
      }
     },
     "d6f7cdd09c9848e6b27dddac51ff682b": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "ded2fb20fa7d423fa21b532eb1322527": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "e38099bacd904a0fbe618e3f937d3fdc": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_8813bd860de048ce98333ba7ef2b89c2",
       "placeholder": "​",
       "style": "IPY_MODEL_05a61cb435e8451196175d1ea9680398",
       "value": "100% 97320/97320 [02:01&lt;00:00, 799.04it/s]"
      }
     },
     "ecd0f37608f24efd89e54deac8f288b7": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "IntProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "IntProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_bcda54b2cfc94816b9bd099b92bfe107",
       "max": 97320,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_a592b8d9cc9944a5b23c5a332d29d46b",
       "value": 97320
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}